1"use strict"; 2/* DokuWiki BotMon Plugin Script file */ 3/* 14.10.2025 - 0.5.0 - pre-release */ 4/* Author: Sascha Leib <ad@hominem.info> */ 5 6// enumeration of user types: 7const BM_USERTYPE = Object.freeze({ 8 'UNKNOWN': 'unknown', 9 'KNOWN_USER': 'user', 10 'PROBABLY_HUMAN': 'human', 11 'LIKELY_BOT': 'likely_bot', 12 'KNOWN_BOT': 'known_bot' 13}); 14 15// enumeration of log types: 16const BM_LOGTYPE = Object.freeze({ 17 'SERVER': 'srv', 18 'CLIENT': 'log', 19 'TICKER': 'tck' 20}); 21 22// enumeration of IP versions: 23const BM_IPVERSION = Object.freeze({ 24 'IPv4': 4, 25 'IPv6': 6 26}); 27 28/* BotMon root object */ 29const BotMon = { 30 31 init: function() { 32 //console.info('BotMon.init()'); 33 34 // find the plugin basedir: 35 this._baseDir = document.currentScript.src.substring(0, document.currentScript.src.lastIndexOf('/')+1); 36 37 // read the page language from the DOM: 38 this._lang = document.getRootNode().documentElement.lang || this._lang; 39 40 // get the time offset: 41 this._timeDiff = BotMon.t._getTimeOffset(); 42 43 // get yesterday's date: 44 let d = new Date(); 45 if (BMSettings.showday == 'yesterday') d.setDate(d.getDate() - 1); 46 this._datestr = d.toISOString().slice(0, 10); 47 48 // init the sub-objects: 49 BotMon.t._callInit(this); 50 }, 51 52 _baseDir: null, 53 _lang: 'en', 54 _datestr: '', 55 _timeDiff: '', 56 57 /* internal tools */ 58 t: { 59 /* helper function to call inits of sub-objects */ 60 _callInit: function(obj) { 61 //console.info('BotMon.t._callInit(obj=',obj,')'); 62 63 /* call init / _init on each sub-object: */ 64 Object.keys(obj).forEach( (key,i) => { 65 const sub = obj[key]; 66 let init = null; 67 if (typeof sub === 'object' && sub.init) { 68 init = sub.init; 69 } 70 71 // bind to object 72 if (typeof init == 'function') { 73 const init2 = init.bind(sub); 74 init2(obj); 75 } 76 }); 77 }, 78 79 /* helper function to calculate the time difference to UTC: */ 80 _getTimeOffset: function() { 81 const now = new Date(); 82 let offset = now.getTimezoneOffset(); // in minutes 83 const sign = Math.sign(offset); // +1 or -1 84 offset = Math.abs(offset); // always positive 85 86 let hours = 0; 87 while (offset >= 60) { 88 hours += 1; 89 offset -= 60; 90 } 91 return ( hours > 0 ? sign * hours + ' h' : '') + (offset > 0 ? ` ${offset} min` : ''); 92 }, 93 94 /* helper function to create a new element with all attributes and text content */ 95 _makeElement: function(name, atlist = undefined, text = undefined) { 96 var r = null; 97 try { 98 r = document.createElement(name); 99 if (atlist) { 100 for (let attr in atlist) { 101 r.setAttribute(attr, atlist[attr]); 102 } 103 } 104 if (text) { 105 r.textContent = text.toString(); 106 } 107 } catch(e) { 108 console.error(e); 109 } 110 return r; 111 }, 112 113 /* helper to convert an ip address string to a normalised format: */ 114 _ip2Num: function(ip) { 115 if (!ip) { 116 return 'null'; 117 } else if (ip.indexOf(':') > 0) { /* IP6 */ 118 return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(':')); 119 } else { /* IP4 */ 120 return ip.split('.').map(d => ('000'+d).slice(-3) ).join('.'); 121 } 122 }, 123 124 /* helper function to format a Date object to show only the time. */ 125 /* returns String */ 126 _formatTime: function(date) { 127 128 if (date) { 129 return date.getHours() + ':' + ('0'+date.getMinutes()).slice(-2) + ':' + ('0'+date.getSeconds()).slice(-2); 130 } else { 131 return null; 132 } 133 134 }, 135 136 /* helper function to show a time difference in seconds or minutes */ 137 /* returns String */ 138 _formatTimeDiff: function(dateA, dateB) { 139 140 // if the second date is ealier, swap them: 141 if (dateA > dateB) dateB = [dateA, dateA = dateB][0]; 142 143 // get the difference in milliseconds: 144 let ms = dateB - dateA; 145 146 if (ms > 50) { /* ignore small time spans */ 147 const h = Math.floor((ms / (1000 * 60 * 60)) % 24); 148 const m = Math.floor((ms / (1000 * 60)) % 60); 149 const s = Math.floor((ms / 1000) % 60); 150 151 return ( h>0 ? h + 'h ': '') + ( m>0 ? m + 'm ': '') + ( s>0 ? s + 's': ''); 152 } 153 154 return null; 155 156 }, 157 158 // calcualte a reduced ration between two numbers 159 // adapted from https://stackoverflow.com/questions/3946373/math-in-js-how-do-i-get-a-ratio-from-a-percentage 160 _getRatio: function(a, b, tolerance) { 161 162 var bg = b; 163 var sm = a; 164 if (a > b) { 165 var bg = a; 166 var sm = b; 167 } 168 169 for (var i = 1; i < 1000000; i++) { 170 var d = sm / i; 171 var res = bg / d; 172 var howClose = Math.abs(res - res.toFixed(0)); 173 if (howClose < tolerance) { 174 if (a > b) { 175 return res.toFixed(0) + ':' + i; 176 } else { 177 return i + ':' + res.toFixed(0); 178 } 179 } 180 } 181 } 182 } 183}; 184 185/* everything specific to the 'Latest' tab is self-contained in the 'live' object: */ 186BotMon.live = { 187 init: function() { 188 //console.info('BotMon.live.init()'); 189 190 // set the title: 191 const tDiff = '<abbr title="Coordinated Universal Time">UTC</abbr> ' + (BotMon._timeDiff != '' ? ` (offset: ${BotMon._timeDiff}` : '' ) + ')'; 192 BotMon.live.gui.status.setTitle(`Data for <time datetime="${BotMon._datestr}">${BotMon._datestr}</time> ${tDiff}`); 193 194 // init sub-objects: 195 BotMon.t._callInit(this); 196 }, 197 198 data: { 199 init: function() { 200 //console.info('BotMon.live.data.init()'); 201 202 // call sub-inits: 203 BotMon.t._callInit(this); 204 }, 205 206 // this will be called when the known json files are done loading: 207 _dispatch: function(file) { 208 //console.info('BotMon.live.data._dispatch(,',file,')'); 209 210 // shortcut to make code more readable: 211 const data = BotMon.live.data; 212 213 // set the flags: 214 switch(file) { 215 case 'rules': 216 data._dispatchRulesLoaded = true; 217 break; 218 case 'ipranges': 219 data._dispatchIPRangesLoaded = true; 220 break; 221 case 'bots': 222 data._dispatchBotsLoaded = true; 223 break; 224 case 'clients': 225 data._dispatchClientsLoaded = true; 226 break; 227 case 'platforms': 228 data._dispatchPlatformsLoaded = true; 229 break; 230 default: 231 // ignore 232 } 233 234 // are all the flags set? 235 if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded && data._dispatchIPRangesLoaded) { 236 // chain the log files loading: 237 BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded); 238 } 239 }, 240 // flags to track which data files have been loaded: 241 _dispatchBotsLoaded: false, 242 _dispatchClientsLoaded: false, 243 _dispatchPlatformsLoaded: false, 244 _dispatchIPRangesLoaded: false, 245 _dispatchRulesLoaded: false, 246 247 // event callback, after the server log has been loaded: 248 _onServerLogLoaded: function() { 249 //console.info('BotMon.live.data._onServerLogLoaded()'); 250 251 // chain the client log file to load: 252 BotMon.live.data.loadLogFile(BM_LOGTYPE.CLIENT, BotMon.live.data._onClientLogLoaded); 253 }, 254 255 // event callback, after the client log has been loaded: 256 _onClientLogLoaded: function() { 257 //console.info('BotMon.live.data._onClientLogLoaded()'); 258 259 // chain the ticks file to load: 260 BotMon.live.data.loadLogFile(BM_LOGTYPE.TICKER, BotMon.live.data._onTicksLogLoaded); 261 262 }, 263 264 // event callback, after the tiker log has been loaded: 265 _onTicksLogLoaded: function() { 266 //console.info('BotMon.live.data._onTicksLogLoaded()'); 267 268 // analyse the data: 269 BotMon.live.data.analytics.analyseAll(); 270 271 // sort the data: 272 // #TODO 273 274 // display the data: 275 BotMon.live.gui.overview.make(); 276 277 //console.log(BotMon.live.data.model._visitors); 278 279 }, 280 281 // the data model: 282 model: { 283 // visitors storage: 284 _visitors: [], 285 286 // find an already existing visitor record: 287 findVisitor: function(visitor, type) { 288 //console.info('BotMon.live.data.model.findVisitor()', type); 289 //console.log(visitor); 290 291 // shortcut to make code more readable: 292 const model = BotMon.live.data.model; 293 294 //const timeout = 60 * 60 * 1000; // session timeout: One hour 295 296 if (visitor._type == BM_USERTYPE.KNOWN_BOT) { // known bots match by their bot ID: 297 298 for (let i=0; i<model._visitors.length; i++) { 299 const v = model._visitors[i]; 300 301 // bots match when their ID matches: 302 if (v._bot && v._bot.id == visitor._bot.id) { 303 return v; 304 } 305 } 306 307 } else { // other types match by their DW/PHPIDs: 308 309 // loop over all visitors already registered and check for ID matches: 310 for (let i=0; i<model._visitors.length; i++) { 311 const v = model._visitors[i]; 312 313 if ( v.id.trim() !== '' && v.id == visitor.id) { // match the DW/PHP IDs 314 return v; 315 } 316 } 317 318 // if not found, try to match IP address and user agent: 319 for (let i=0; i<model._visitors.length; i++) { 320 const v = model._visitors[i]; 321 if ( v.ip == visitor.ip && v.agent == visitor.agent) { 322 return v; 323 } 324 } 325 } 326 327 return null; // nothing found 328 }, 329 330 /* if there is already this visit registered, return the page view item */ 331 _getPageView: function(visit, view) { 332 333 // shortcut to make code more readable: 334 const model = BotMon.live.data.model; 335 336 for (let i=0; i<visit._pageViews.length; i++) { 337 const pv = visit._pageViews[i]; 338 if (pv.pg == view.pg) { 339 return pv; 340 } 341 } 342 return null; // not found 343 }, 344 345 // register a new visitor (or update if already exists) 346 registerVisit: function(nv, type) { 347 //console.info('registerVisit', nv, type); 348 349 // shortcut to make code more readable: 350 const model = BotMon.live.data.model; 351 352 // is it a known bot? 353 const bot = BotMon.live.data.bots.match(nv.agent); 354 355 // enrich new visitor with relevant data: 356 if (!nv._bot) nv._bot = bot ?? null; // bot info 357 nv._type = ( bot ? BM_USERTYPE.KNOWN_BOT : ( nv.usr && nv.usr !== '' ? BM_USERTYPE.KNOWN_USER : BM_USERTYPE.UNKNOWN ) ); // user type 358 if (bot && bot.geo) { 359 if (!nv.geo || nv.geo == '' || nv.geo == 'ZZ') nv.geo = bot.geo; 360 } else if (!nv.geo ||nv.geo == '') { 361 nv.geo = 'ZZ'; 362 } 363 364 // update first and last seen: 365 if (!nv._firstSeen) nv._firstSeen = nv.ts; // first-seen 366 nv._lastSeen = nv.ts; // last-seen 367 368 // country name: 369 try { 370 nv._country = ( nv.geo == 'local' ? "localhost" : "Unknown" ); 371 if (nv.geo && nv.geo !== '' && nv.geo !== 'ZZ' && nv.geo !== 'local') { 372 const countryName = new Intl.DisplayNames(['en', BotMon._lang], {type: 'region'}); 373 nv._country = countryName.of(nv.geo.substring(0,2)) ?? nv.geo; 374 } 375 } catch (err) { 376 console.error(err); 377 nv._country = 'Error'; 378 } 379 380 // check if it already exists: 381 let visitor = model.findVisitor(nv, type); 382 if (!visitor) { 383 visitor = {...nv, ...{ 384 _seenBy: [type], 385 _viewCount: 0, // number of page views 386 _loadCount: 0, // number of page loads (not necessarily views!) 387 _pageViews: [], // array of page views 388 _hasReferrer: false, // has at least one referrer 389 _jsClient: false, // visitor has been seen logged by client js as well 390 _client: BotMon.live.data.clients.match(nv.agent) ?? null, // client info 391 _platform: BotMon.live.data.platforms.match(nv.agent), // platform info 392 _captcha: {'-': 0, 'Y': 0, 'N': 0, 'W':0} // captcha counter 393 }}; 394 model._visitors.push(visitor); 395 }; 396 397 // update first and last seen: 398 if (visitor._firstSeen > nv.ts) { 399 visitor._firstSeen = nv.ts; 400 } 401 if (visitor._lastSeen < nv.ts) { 402 visitor._lastSeen = nv.ts; 403 } 404 405 // update total loads and views (not the same!): 406 visitor._loadCount += 1; 407 visitor._viewCount += (nv.captcha == 'Y' ? 0 : 1); 408 409 // ...because also a captcha is a "load", but not a "view". 410 // let's count the captcha statuses as well: 411 if (nv.captcha) visitor._captcha[nv.captcha] += 1; 412 413 // is this visit already registered? 414 let prereg = model._getPageView(visitor, nv); 415 if (!prereg) { 416 // add new page view: 417 prereg = model._makePageView(nv, type); 418 visitor._pageViews.push(prereg); 419 } 420 421 // update last seen date 422 prereg._lastSeen = nv.ts; 423 424 // increase view count: 425 prereg._loadCount += (visitor.captcha == 'Y' ? 0 : 1); 426 //prereg._tickCount += 1; 427 428 // update referrer state: 429 visitor._hasReferrer = visitor._hasReferrer || 430 (prereg.ref !== undefined && prereg.ref !== ''); 431 432 // update time stamp for last-seen: 433 if (visitor._lastSeen < nv.ts) { 434 visitor._lastSeen = nv.ts; 435 } 436 437 // if needed: 438 return visitor; 439 }, 440 441 // updating visit data from the client-side log: 442 updateVisit: function(dat) { 443 //console.info('updateVisit', dat); 444 445 // shortcut to make code more readable: 446 const model = BotMon.live.data.model; 447 448 const type = BM_LOGTYPE.CLIENT; 449 450 let visitor = BotMon.live.data.model.findVisitor(dat, type); 451 if (!visitor) { 452 visitor = model.registerVisit(dat, type); 453 } 454 if (visitor) { 455 456 if (visitor._lastSeen < dat.ts) { 457 visitor._lastSeen = dat.ts; 458 } 459 if (!visitor._seenBy.includes(type)) { 460 visitor._seenBy.push(type); 461 } 462 visitor._jsClient = true; // seen by client js 463 } 464 465 // find the page view: 466 let prereg = BotMon.live.data.model._getPageView(visitor, dat); 467 if (prereg) { 468 // update the page view: 469 prereg._lastSeen = dat.ts; 470 if (!prereg._seenBy.includes(type)) prereg._seenBy.push(type); 471 prereg._jsClient = true; // seen by client js 472 } else { 473 // add the page view to the visitor: 474 prereg = model._makePageView(dat, type); 475 visitor._pageViews.push(prereg); 476 } 477 prereg._tickCount += 1; 478 }, 479 480 // updating visit data from the ticker log: 481 updateTicks: function(dat) { 482 //console.info('updateTicks', dat); 483 484 // shortcut to make code more readable: 485 const model = BotMon.live.data.model; 486 487 const type = BM_LOGTYPE.TICKER; 488 489 // find the visit info: 490 let visitor = model.findVisitor(dat, type); 491 if (!visitor) { 492 console.info(`No visitor with ID “${dat.id}” found, registering as a new one.`); 493 visitor = model.registerVisit(dat, type); 494 } 495 if (visitor) { 496 // update visitor: 497 if (visitor._lastSeen < dat.ts) visitor._lastSeen = dat.ts; 498 if (!visitor._seenBy.includes(type)) visitor._seenBy.push(type); 499 500 // get the page view info: 501 let pv = model._getPageView(visitor, dat); 502 if (!pv) { 503 console.info(`No page view for visit ID “${dat.id}”, page “${dat.pg}”, registering a new one.`); 504 pv = model._makePageView(dat, type); 505 visitor._pageViews.push(pv); 506 } 507 508 // update the page view info: 509 if (!pv._seenBy.includes(type)) pv._seenBy.push(type); 510 if (pv._lastSeen.getTime() < dat.ts.getTime()) pv._lastSeen = dat.ts; 511 pv._tickCount += 1; 512 513 } 514 }, 515 516 // helper function to create a new "page view" item: 517 _makePageView: function(data, type) { 518 // console.info('_makePageView', data); 519 520 // try to parse the referrer: 521 let rUrl = null; 522 try { 523 rUrl = ( data.ref && data.ref !== '' ? new URL(data.ref) : null ); 524 } catch (e) { 525 console.warn(`Invalid referer: “${data.ref}”.`); 526 console.info(data); 527 } 528 529 return { 530 _by: type, 531 ip: data.ip, 532 pg: data.pg, 533 lang: data.lang || '??', 534 _ref: rUrl, 535 _firstSeen: data.ts, 536 _lastSeen: data.ts, 537 _seenBy: [type], 538 _jsClient: ( type !== BM_LOGTYPE.SERVER), 539 _viewCount: 0, 540 _loadCount: 0, 541 _tickCount: 0 542 }; 543 } 544 }, 545 546 // functions to analyse the data: 547 analytics: { 548 549 /** 550 * Initializes the analytics data storage object: 551 */ 552 init: function() { 553 //console.info('BotMon.live.data.analytics.init()'); 554 }, 555 556 // data storage: 557 data: { 558 totalVisits: 0, 559 totalPageViews: 0, 560 humanPageViews: 0, 561 bots: { 562 known: 0, 563 suspected: 0, 564 human: 0, 565 users: 0 566 } 567 }, 568 569 // sort the visits by type: 570 groups: { 571 knownBots: [], 572 suspectedBots: [], 573 humans: [], 574 users: [] 575 }, 576 577 // all analytics 578 analyseAll: function() { 579 //console.info('BotMon.live.data.analytics.analyseAll()'); 580 581 // shortcut to make code more readable: 582 const model = BotMon.live.data.model; 583 const me = BotMon.live.data.analytics; 584 585 BotMon.live.gui.status.showBusy("Analysing data …"); 586 587 // loop over all visitors: 588 model._visitors.forEach( (v) => { 589 590 // count visits and page views: 591 this.data.totalVisits += 1; 592 this.data.totalPageViews += v._viewCount; 593 594 // check for typical bot aspects: 595 let botScore = 0; 596 597 if (v._type == BM_USERTYPE.KNOWN_BOT) { // known bots 598 599 this.data.bots.known += v._viewCount; 600 this.groups.knownBots.push(v); 601 602 } else if (v._type == BM_USERTYPE.KNOWN_USER) { // known users */ 603 604 this.data.bots.users += v._viewCount; 605 this.groups.users.push(v); 606 607 } else { 608 609 // get evaluation: 610 const e = BotMon.live.data.rules.evaluate(v); 611 v._eval = e.rules; 612 v._botVal = e.val; 613 614 if (e.isBot) { // likely bots 615 v._type = BM_USERTYPE.LIKELY_BOT; 616 this.data.bots.suspected += v._viewCount; 617 this.groups.suspectedBots.push(v); 618 } else { // probably humans 619 v._type = BM_USERTYPE.PROBABLY_HUMAN; 620 this.data.bots.human += v._viewCount; 621 this.groups.humans.push(v); 622 } 623 } 624 625 // perform actions depending on the visitor type: 626 if (v._type == BM_USERTYPE.KNOWN_BOT ) { /* known bots only */ 627 628 } else if (v._type == BM_USERTYPE.LIKELY_BOT) { /* probable bots only */ 629 630 // add bot views to IP range information: 631 me.addToIpRanges(v); 632 633 } else { /* humans only */ 634 635 // add browser and platform statistics: 636 me.addBrowserPlatform(v); 637 638 // add 639 v._pageViews.forEach( pv => { 640 me.addToRefererList(pv._ref); 641 me.addToPagesList(pv.pg); 642 }); 643 } 644 645 // add to the country lists: 646 me.addToCountries(v.geo, v._country, v._type); 647 648 }); 649 650 BotMon.live.gui.status.hideBusy('Done.'); 651 }, 652 653 // get a list of known bots: 654 getTopBots: function(max) { 655 //console.info('BotMon.live.data.analytics.getTopBots('+max+')'); 656 657 //console.log(BotMon.live.data.analytics.groups.knownBots); 658 659 let botsList = BotMon.live.data.analytics.groups.knownBots.toSorted( (a, b) => { 660 return b._viewCount - a._viewCount; 661 }); 662 663 const other = { 664 'id': 'other', 665 'name': "Others", 666 'count': 0 667 }; 668 669 const rList = []; 670 const max2 = ( botsList.length > max ? max-1 : botsList.length ); 671 let total = 0; // adding up the items 672 for (let i=0; i<botsList.length; i++) { 673 const it = botsList[i]; 674 if (it && it._bot) { 675 if (i < max2) { 676 rList.push({ 677 id: it._bot.id, 678 name: (it._bot.n ? it._bot.n : it._bot.id), 679 count: it._viewCount 680 }); 681 } else { 682 other.count += it._pageViews.length; 683 }; 684 total += it._viewCount; 685 } 686 }; 687 688 // add the "other" item, if needed: 689 if (botsList.length > max2) { 690 rList.push(other); 691 }; 692 693 rList.forEach( it => { 694 it.pct = (it.count * 100 / total); 695 }); 696 697 return rList; 698 }, 699 700 // most visited pages list: 701 _pagesList: [], 702 703 /** 704 * Add a page view to the list of most visited pages. 705 * @param {string} pageId - The page ID to add to the list. 706 * @example 707 * BotMon.live.data.analytics.addToPagesList('1234567890'); 708 */ 709 addToPagesList: function(pageId) { 710 //console.log('BotMon.live.data.analytics.addToPagesList', pageId); 711 712 const me = BotMon.live.data.analytics; 713 714 // already exists? 715 let pgObj = null; 716 for (let i = 0; i < me._pagesList.length; i++) { 717 if (me._pagesList[i].id == pageId) { 718 pgObj = me._pagesList[i]; 719 break; 720 } 721 } 722 723 // if not exists, create it: 724 if (!pgObj) { 725 pgObj = { 726 id: pageId, 727 count: 1 728 }; 729 me._pagesList.push(pgObj); 730 } else { 731 pgObj.count += 1; 732 } 733 }, 734 735 getTopPages: function(max) { 736 //console.info('BotMon.live.data.analytics.getTopPages('+max+')'); 737 const me = BotMon.live.data.analytics; 738 return me._pagesList.toSorted( (a, b) => { 739 return b.count - a.count; 740 }).slice(0,max); 741 }, 742 743 // Referer List: 744 _refererList: [], 745 746 addToRefererList: function(ref) { 747 //console.log('BotMon.live.data.analytics.addToRefererList',ref); 748 749 const me = BotMon.live.data.analytics; 750 751 // ignore internal references: 752 if (ref && ref.host == window.location.host) { 753 return; 754 } 755 756 const refInfo = me.getRefererInfo(ref); 757 758 // already exists? 759 let refObj = null; 760 for (let i = 0; i < me._refererList.length; i++) { 761 if (me._refererList[i].id == refInfo.id) { 762 refObj = me._refererList[i]; 763 break; 764 } 765 } 766 767 // if not exists, create it: 768 if (!refObj) { 769 refObj = refInfo; 770 refObj.count = 1; 771 me._refererList.push(refObj); 772 } else { 773 refObj.count += 1; 774 } 775 }, 776 777 getRefererInfo: function(url) { 778 //console.log('BotMon.live.data.analytics.getRefererInfo',url); 779 try { 780 url = new URL(url); 781 } catch (e) { 782 return { 783 'id': 'null', 784 'n': 'Invalid Referer' 785 }; 786 } 787 788 // find the referer ID: 789 let refId = 'null'; 790 let refName = 'No Referer'; 791 if (url && url.host) { 792 const hArr = url.host.split('.'); 793 const tld = hArr[hArr.length-1]; 794 refId = ( tld == 'localhost' ? tld : hArr[hArr.length-2]); 795 refName = hArr[hArr.length-2] + '.' + tld; 796 } 797 798 return { 799 'id': refId, 800 'n': refName 801 }; 802 }, 803 804 /** 805 * Get a sorted list of the top referers. 806 * The list is sorted in descending order of count. 807 * If the array has more items than the given maximum, the rest of the items are added to an "other" item. 808 * Each item in the list has a "pct" property, which is the percentage of the total count. 809 * @param {number} max - The maximum number of items to return. 810 * @return {Array} The sorted list of top referers. 811 */ 812 getTopReferers: function(max) { 813 //console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')')); 814 815 const me = BotMon.live.data.analytics; 816 817 return me._makeTopList(me._refererList, max); 818 }, 819 820 /** 821 * Create a sorted list of top items from a given array. 822 * The list is sorted in descending order of count. 823 * If the array has more items than the given maximum, the rest of the items are added to an "other" item. 824 * Each item in the list has a "pct" property, which is the percentage of the total count. 825 * @param {Array} arr - The array to sort and truncate. 826 * @param {number} max - The maximum number of items to return. 827 * @return {Array} The sorted list of top items. 828 */ 829 _makeTopList: function(arr, max) { 830 //console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')')); 831 832 const me = BotMon.live.data.analytics; 833 834 // sort the list: 835 arr.sort( (a,b) => { 836 return b.count - a.count; 837 }); 838 839 const rList = []; // return array 840 const max2 = ( arr.length >= max ? max-1 : arr.length ); 841 const other = { 842 'id': 'other', 843 'name': "Others", 844 'typ': 'other', 845 'count': 0 846 }; 847 let total = 0; // adding up the items 848 for (let i=0; Math.min(max, arr.length) > i; i++) { 849 const it = arr[i]; 850 if (it) { 851 if (i < max2) { 852 const rIt = { 853 id: it.id, 854 name: (it.n ? it.n : it.id), 855 typ: it.typ || it.id, 856 count: it.count 857 }; 858 rList.push(rIt); 859 } else { 860 other.count += it.count; 861 } 862 total += it.count; 863 } 864 } 865 866 // add the "other" item, if needed: 867 if (arr.length > max2) { 868 rList.push(other); 869 }; 870 871 rList.forEach( it => { 872 it.pct = (it.count * 100 / total); 873 }); 874 875 return rList; 876 }, 877 878 /* countries of visits */ 879 _countries: { 880 'human': [], 881 'bot': [] 882 }, 883 /** 884 * Adds a country code to the statistics. 885 * 886 * @param {string} iso The ISO 3166-1 alpha-2 country code. 887 */ 888 addToCountries: function(iso, name, type) { 889 890 const me = BotMon.live.data.analytics; 891 892 // find the correct array: 893 let arr = null; 894 switch (type) { 895 896 case BM_USERTYPE.KNOWN_USER: 897 case BM_USERTYPE.PROBABLY_HUMAN: 898 arr = me._countries.human; 899 break; 900 case BM_USERTYPE.LIKELY_BOT: 901 case BM_USERTYPE.KNOWN_BOT: 902 arr = me._countries.bot; 903 break; 904 default: 905 console.warn(`Unknown user type ${type} in function addToCountries.`); 906 } 907 908 if (arr) { 909 let cRec = arr.find( it => it.id == iso); 910 if (!cRec) { 911 cRec = { 912 'id': iso, 913 'n': name, 914 'count': 1 915 }; 916 arr.push(cRec); 917 } else { 918 cRec.count += 1; 919 } 920 } 921 }, 922 923 /** 924 * Returns a list of countries with visit counts, sorted by visit count in descending order. 925 * 926 * @param {BM_USERTYPE} type array of types type of visitors to return. 927 * @param {number} max The maximum number of entries to return. 928 * @return {Array} A list of objects with properties 'iso' (ISO 3166-1 alpha-2 country code) and 'count' (visit count). 929 */ 930 getCountryList: function(type, max) { 931 932 const me = BotMon.live.data.analytics; 933 934 // find the correct array: 935 let arr = null; 936 switch (type) { 937 938 case 'human': 939 arr = me._countries.human; 940 break; 941 case 'bot': 942 arr = me._countries.bot; 943 break; 944 default: 945 console.warn(`Unknown user type ${type} in function getCountryList.`); 946 return; 947 } 948 949 return me._makeTopList(arr, max); 950 }, 951 952 /* browser and platform of human visitors */ 953 _browsers: [], 954 _platforms: [], 955 956 addBrowserPlatform: function(visitor) { 957 //console.info('addBrowserPlatform', visitor); 958 959 const me = BotMon.live.data.analytics; 960 961 // add to browsers list: 962 let browserRec = ( visitor._client ? visitor._client : {'id': 'unknown'}); 963 if (visitor._client) { 964 let bRec = me._browsers.find( it => it.id == browserRec.id); 965 if (!bRec) { 966 bRec = { 967 id: browserRec.id, 968 n: browserRec.n, 969 count: 1 970 }; 971 me._browsers.push(bRec); 972 } else { 973 bRec.count += 1; 974 } 975 } 976 977 // add to platforms list: 978 let platformRec = ( visitor._platform ? visitor._platform : {'id': 'unknown'}); 979 if (visitor._platform) { 980 let pRec = me._platforms.find( it => it.id == platformRec.id); 981 if (!pRec) { 982 pRec = { 983 id: platformRec.id, 984 n: platformRec.n, 985 count: 1 986 }; 987 me._platforms.push(pRec); 988 } else { 989 pRec.count += 1; 990 } 991 } 992 993 }, 994 995 getTopBrowsers: function(max) { 996 997 const me = BotMon.live.data.analytics; 998 999 return me._makeTopList(me._browsers, max); 1000 }, 1001 1002 getTopPlatforms: function(max) { 1003 1004 const me = BotMon.live.data.analytics; 1005 1006 return me._makeTopList(me._platforms, max); 1007 }, 1008 1009 /* bounces are counted, not calculates: */ 1010 getBounceCount: function(type) { 1011 1012 const me = BotMon.live.data.analytics; 1013 var bounces = 0; 1014 const list = me.groups[type]; 1015 1016 list.forEach(it => { 1017 bounces += (it._viewCount <= 1 ? 1 : 0); 1018 }); 1019 1020 return bounces; 1021 }, 1022 1023 _ipRanges: [], 1024 1025 /* adds a visit to the ip ranges arrays */ 1026 addToIpRanges: function(v) { 1027 //console.info('addToIpRanges', v.ip); 1028 1029 const me = BotMon.live.data.analytics; 1030 const ipRanges = BotMon.live.data.ipRanges; 1031 1032 // Number of IP address segments to look at: 1033 const kIP4Segments = 1; 1034 const kIP6Segments = 3; 1035 1036 let ipGroup = ''; // group name 1037 let ipSeg = []; // IP segment array 1038 let rawIP = ''; // raw ip prefix 1039 let ipName = ''; // IP group display name 1040 1041 const ipType = v.ip.indexOf(':') > 0 ? BM_IPVERSION.IPv6 : BM_IPVERSION.IPv4; 1042 const ipAddr = BotMon.t._ip2Num(v.ip); 1043 1044 // is there already a known IP range assigned? 1045 if (v._ipRange) { 1046 1047 ipGroup = v._ipRange.g; // group name 1048 ipName = ipRanges.getOwner( v._ipRange.g ) || "Unknown"; 1049 1050 } else { // no known IP range, let's collect necessary information: 1051 1052 1053 // collect basic IP address info: 1054 if (ipType == BM_IPVERSION.IPv6) { 1055 ipSeg = ipAddr.split(':'); 1056 const prefix = v.ip.split(':').slice(0, kIP6Segments).join(':'); 1057 rawIP = ipSeg.slice(0, kIP6Segments).join(':'); 1058 ipGroup = 'ip6-' + rawIP.replaceAll(':', '-'); 1059 ipName = prefix + '::'; // + '/' + (16 * kIP6Segments); 1060 } else { 1061 ipSeg = ipAddr.split('.'); 1062 const prefix = v.ip.split('.').slice(0, kIP4Segments).join('.'); 1063 rawIP = ipSeg.slice(0, kIP4Segments).join('.') ; 1064 ipGroup = 'ip4-' + rawIP.replaceAll('.', '-'); 1065 ipName = prefix + '.x.x.x'.substring(0, 1+(4-kIP4Segments)*2); // + '/' + (8 * kIP4Segments); 1066 } 1067 } 1068 1069 // check if record already exists: 1070 let ipRec = me._ipRanges.find( it => it.g == ipGroup); 1071 if (!ipRec) { 1072 1073 // ip info record initialised: 1074 ipRec = { 1075 g: ipGroup, 1076 n: ipName, 1077 count: 0 1078 } 1079 1080 // existing record? 1081 if (v._ipRange) { 1082 1083 ipRec.from = v._ipRange.from; 1084 ipRec.to = v._ipRange.to; 1085 ipRec.typ = 'net'; 1086 1087 } else { // no known IP range, let's collect necessary information: 1088 1089 // complete the ip info record: 1090 if (ipType == BM_IPVERSION.IPv6) { 1091 ipRec.from = rawIP + ':0000:0000:0000:0000:0000:0000:0000'.substring(0, (8-kIP6Segments)*5); 1092 ipRec.to = rawIP + ':FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF'.substring(0, (8-kIP6Segments)*5); 1093 ipRec.typ = '6'; 1094 } else { 1095 ipRec.from = rawIP + '.000.000.000.000'.substring(0, (4-kIP4Segments)*4); 1096 ipRec.to = rawIP + '.255.255.255.254'.substring(0, (4-kIP4Segments)*4); 1097 ipRec.typ = '4'; 1098 } 1099 } 1100 1101 me._ipRanges.push(ipRec); 1102 } 1103 1104 // add to counter: 1105 ipRec.count += v._viewCount; 1106 1107 }, 1108 1109 getTopBotISPs: function(max) { 1110 1111 const me = BotMon.live.data.analytics; 1112 1113 return me._makeTopList(me._ipRanges, max); 1114 }, 1115 }, 1116 1117 // information on "known bots": 1118 bots: { 1119 // loads the list of known bots from a JSON file: 1120 init: async function() { 1121 //console.info('BotMon.live.data.bots.init()'); 1122 1123 // Load the list of known bots: 1124 BotMon.live.gui.status.showBusy("Loading known bots …"); 1125 const url = BotMon._baseDir + 'config/known-bots.json'; 1126 try { 1127 const response = await fetch(url); 1128 if (!response.ok) { 1129 throw new Error(`${response.status} ${response.statusText}`); 1130 } 1131 1132 this._list = await response.json(); 1133 this._ready = true; 1134 1135 } catch (error) { 1136 BotMon.live.gui.status.setError("Error while loading the known bots file:", error.message); 1137 } finally { 1138 BotMon.live.gui.status.hideBusy("Status: Done."); 1139 BotMon.live.data._dispatch('bots') 1140 } 1141 }, 1142 1143 // returns bot info if the clientId matches a known bot, null otherwise: 1144 match: function(agent) { 1145 //console.info('BotMon.live.data.bots.match(',agent,')'); 1146 1147 const BotList = BotMon.live.data.bots._list; 1148 1149 // default is: not found! 1150 let botInfo = null; 1151 1152 if (!agent) return null; 1153 1154 // check for known bots: 1155 BotList.find(bot => { 1156 let r = false; 1157 for (let j=0; j<bot.rx.length; j++) { 1158 const rxr = agent.match(new RegExp(bot.rx[j])); 1159 if (rxr) { 1160 botInfo = { 1161 n : bot.n, 1162 id: bot.id, 1163 geo: (bot.geo ? bot.geo : null), 1164 url: bot.url, 1165 v: (rxr.length > 1 ? rxr[1] : -1) 1166 }; 1167 r = true; 1168 break; 1169 }; 1170 }; 1171 return r; 1172 }); 1173 1174 // check for unknown bots: 1175 if (!botInfo) { 1176 const botmatch = agent.match(/([\s\d\w\-]*bot|[\s\d\w\-]*crawler|[\s\d\w\-]*spider)[\/\s\w\-;\),\\.$]/i); 1177 if(botmatch) { 1178 botInfo = {'id': ( botmatch[1] || "other_" ), 'n': "Other" + ( botmatch[1] ? " (" + botmatch[1] + ")" : "" ) , "bot": botmatch[1] }; 1179 } 1180 } 1181 1182 //console.log("botInfo:", botInfo); 1183 return botInfo; 1184 }, 1185 1186 1187 // indicates if the list is loaded and ready to use: 1188 _ready: false, 1189 1190 // the actual bot list is stored here: 1191 _list: [] 1192 }, 1193 1194 // information on known clients (browsers): 1195 clients: { 1196 // loads the list of known clients from a JSON file: 1197 init: async function() { 1198 //console.info('BotMon.live.data.clients.init()'); 1199 1200 // Load the list of known bots: 1201 BotMon.live.gui.status.showBusy("Loading known clients"); 1202 const url = BotMon._baseDir + 'config/known-clients.json'; 1203 try { 1204 const response = await fetch(url); 1205 if (!response.ok) { 1206 throw new Error(`${response.status} ${response.statusText}`); 1207 } 1208 1209 BotMon.live.data.clients._list = await response.json(); 1210 BotMon.live.data.clients._ready = true; 1211 1212 } catch (error) { 1213 BotMon.live.gui.status.setError("Error while loading the known clients file: " + error.message); 1214 } finally { 1215 BotMon.live.gui.status.hideBusy("Status: Done."); 1216 BotMon.live.data._dispatch('clients') 1217 } 1218 }, 1219 1220 // returns bot info if the user-agent matches a known bot, null otherwise: 1221 match: function(agent) { 1222 //console.info('BotMon.live.data.clients.match(',agent,')'); 1223 1224 let match = {"n": "Unknown", "v": -1, "id": 'null'}; 1225 1226 if (agent) { 1227 BotMon.live.data.clients._list.find(client => { 1228 let r = false; 1229 for (let j=0; j<client.rx.length; j++) { 1230 const rxr = agent.match(new RegExp(client.rx[j])); 1231 if (rxr) { 1232 match.n = client.n; 1233 match.v = (rxr.length > 1 ? rxr[1] : -1); 1234 match.id = client.id || null; 1235 r = true; 1236 break; 1237 } 1238 } 1239 return r; 1240 }); 1241 } 1242 1243 //console.log(match) 1244 return match; 1245 }, 1246 1247 // return the browser name for a browser ID: 1248 getName: function(id) { 1249 const it = BotMon.live.data.clients._list.find(client => client.id == id); 1250 return ( it && it.n ? it.n : "Unknown"); //it.n; 1251 }, 1252 1253 // indicates if the list is loaded and ready to use: 1254 _ready: false, 1255 1256 // the actual bot list is stored here: 1257 _list: [] 1258 1259 }, 1260 1261 // information on known platforms (operating systems): 1262 platforms: { 1263 // loads the list of known platforms from a JSON file: 1264 init: async function() { 1265 //console.info('BotMon.live.data.platforms.init()'); 1266 1267 // Load the list of known bots: 1268 BotMon.live.gui.status.showBusy("Loading known platforms"); 1269 const url = BotMon._baseDir + 'config/known-platforms.json'; 1270 try { 1271 const response = await fetch(url); 1272 if (!response.ok) { 1273 throw new Error(`${response.status} ${response.statusText}`); 1274 } 1275 1276 BotMon.live.data.platforms._list = await response.json(); 1277 BotMon.live.data.platforms._ready = true; 1278 1279 } catch (error) { 1280 BotMon.live.gui.status.setError("Error while loading the known platforms file: " + error.message); 1281 } finally { 1282 BotMon.live.gui.status.hideBusy("Status: Done."); 1283 BotMon.live.data._dispatch('platforms') 1284 } 1285 }, 1286 1287 // returns bot info if the browser id matches a known platform: 1288 match: function(cid) { 1289 //console.info('BotMon.live.data.platforms.match(',cid,')'); 1290 1291 let match = {"n": "Unknown", "id": 'null'}; 1292 1293 if (cid) { 1294 BotMon.live.data.platforms._list.find(platform => { 1295 let r = false; 1296 for (let j=0; j<platform.rx.length; j++) { 1297 const rxr = cid.match(new RegExp(platform.rx[j])); 1298 if (rxr) { 1299 match.n = platform.n; 1300 match.v = (rxr.length > 1 ? rxr[1] : -1); 1301 match.id = platform.id || null; 1302 r = true; 1303 break; 1304 } 1305 } 1306 return r; 1307 }); 1308 } 1309 1310 return match; 1311 }, 1312 1313 // return the platform name for a given ID: 1314 getName: function(id) { 1315 const it = BotMon.live.data.platforms._list.find( pf => pf.id == id); 1316 return ( it ? it.n : 'Unknown' ); 1317 }, 1318 1319 1320 // indicates if the list is loaded and ready to use: 1321 _ready: false, 1322 1323 // the actual bot list is stored here: 1324 _list: [] 1325 1326 }, 1327 1328 // storage and functions for the known bot IP-Ranges: 1329 ipRanges: { 1330 1331 init: function() { 1332 //console.log('BotMon.live.data.ipRanges.init()'); 1333 // #TODO: Load from separate IP-Ranges file 1334 // load the rules file: 1335 const me = BotMon.live.data; 1336 1337 try { 1338 BotMon.live.data._loadSettingsFile(['user-ipranges', 'known-ipranges'], 1339 (json) => { 1340 1341 // groups can be just saved in the data structure: 1342 if (json.groups && json.groups.constructor.name == 'Array') { 1343 me.ipRanges._groups = json.groups; 1344 } 1345 1346 // groups can be just saved in the data structure: 1347 if (json.ranges && json.ranges.constructor.name == 'Array') { 1348 json.ranges.forEach(range => { 1349 me.ipRanges.add(range); 1350 }) 1351 } 1352 1353 // finished loading 1354 BotMon.live.gui.status.hideBusy("Status: Done."); 1355 BotMon.live.data._dispatch('ipranges') 1356 }); 1357 } catch (error) { 1358 BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); 1359 } 1360 }, 1361 1362 // the actual bot list is stored here: 1363 _list: [], 1364 _groups: [], 1365 1366 add: function(data) { 1367 //console.log('BotMon.live.data.ipRanges.add(',data,')'); 1368 1369 const me = BotMon.live.data.ipRanges; 1370 1371 // convert IP address to easier comparable form: 1372 const ip2Num = BotMon.t._ip2Num; 1373 1374 let item = { 1375 'cidr': data.from.replaceAll(/::+/g, '::') + '/' + ( data.m ? data.m : '??' ), 1376 'from': ip2Num(data.from), 1377 'to': ip2Num(data.to), 1378 'm': data.m, 1379 'g': data.g 1380 }; 1381 me._list.push(item); 1382 1383 }, 1384 1385 getOwner: function(gid) { 1386 1387 const me = BotMon.live.data.ipRanges; 1388 1389 for (let i=0; i < me._groups.length; i++) { 1390 const it = me._groups[i]; 1391 if (it.id == gid) { 1392 return it.name; 1393 } 1394 } 1395 return null; 1396 }, 1397 1398 match: function(ip) { 1399 //console.log('BotMon.live.data.ipRanges.match(',ip,')'); 1400 1401 const me = BotMon.live.data.ipRanges; 1402 1403 // convert IP address to easier comparable form: 1404 const ipNum = BotMon.t._ip2Num(ip); 1405 1406 for (let i=0; i < me._list.length; i++) { 1407 const ipRange = me._list[i]; 1408 1409 if (ipNum >= ipRange.from && ipNum <= ipRange.to) { 1410 return ipRange; 1411 } 1412 1413 }; 1414 return null; 1415 } 1416 }, 1417 1418 // storage for the rules and related functions 1419 rules: { 1420 1421 /** 1422 * Initializes the rules data. 1423 * 1424 * Loads the default config file and the user config file (if present). 1425 * The default config file is used if the user config file does not have a certain setting. 1426 * The user config file can override settings from the default config file. 1427 * 1428 * The rules are loaded from the `rules` property of the config files. 1429 * The IP ranges are loaded from the `ipRanges` property of the config files. 1430 * 1431 * If an error occurs while loading the config file, it is displayed in the status bar. 1432 * After the config file is loaded, the status bar is hidden. 1433 */ 1434 init: async function() { 1435 //console.info('BotMon.live.data.rules.init()'); 1436 1437 // Load the list of known bots: 1438 BotMon.live.gui.status.showBusy("Loading list of rules …"); 1439 1440 // load the rules file: 1441 const me = BotMon.live.data; 1442 1443 try { 1444 BotMon.live.data._loadSettingsFile(['user-config', 'default-config'], 1445 (json) => { 1446 1447 // override the threshold? 1448 if (json.threshold) me._threshold = json.threshold; 1449 1450 // set the rules list: 1451 if (json.rules && json.rules.constructor.name == 'Array') { 1452 me.rules._rulesList = json.rules; 1453 } 1454 1455 BotMon.live.gui.status.hideBusy("Status: Done."); 1456 BotMon.live.data._dispatch('rules') 1457 } 1458 ); 1459 } catch (error) { 1460 BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); 1461 } 1462 }, 1463 1464 _rulesList: [], // list of rules to find out if a visitor is a bot 1465 _threshold: 100, // above this, it is considered a bot. 1466 1467 // returns a descriptive text for a rule id 1468 getRuleInfo: function(ruleId) { 1469 // console.info('getRuleInfo', ruleId); 1470 1471 // shortcut for neater code: 1472 const me = BotMon.live.data.rules; 1473 1474 for (let i=0; i<me._rulesList.length; i++) { 1475 const rule = me._rulesList[i]; 1476 if (rule.id == ruleId) { 1477 return rule; 1478 } 1479 } 1480 return null; 1481 1482 }, 1483 1484 // evaluate a visitor for lkikelihood of being a bot 1485 evaluate: function(visitor) { 1486 1487 // shortcut for neater code: 1488 const me = BotMon.live.data.rules; 1489 1490 let r = { // evaluation result 1491 'val': 0, 1492 'rules': [], 1493 'isBot': false 1494 }; 1495 1496 for (let i=0; i<me._rulesList.length; i++) { 1497 const rule = me._rulesList[i]; 1498 const params = ( rule.params ? rule.params : [] ); 1499 1500 if (rule.func) { // rule is calling a function 1501 if (me.func[rule.func]) { 1502 if(me.func[rule.func](visitor, ...params)) { 1503 r.val += rule.bot; 1504 r.rules.push(rule.id) 1505 } 1506 } else { 1507 //console.warn("Unknown rule function: “${rule.func}”. Ignoring rule.") 1508 } 1509 } 1510 } 1511 1512 // is a bot? 1513 r.isBot = (r.val >= me._threshold); 1514 1515 return r; 1516 }, 1517 1518 // list of functions that can be called by the rules list to evaluate a visitor: 1519 func: { 1520 1521 // check if client is on the list passed as parameter: 1522 matchesClient: function(visitor, ...clients) { 1523 1524 const clientId = ( visitor._client ? visitor._client.id : ''); 1525 return clients.includes(clientId); 1526 }, 1527 1528 // check if OS/Platform is one of the obsolete ones: 1529 matchesPlatform: function(visitor, ...platforms) { 1530 1531 const pId = ( visitor._platform ? visitor._platform.id : ''); 1532 1533 if (visitor._platform.id == null) console.log(visitor._platform); 1534 1535 return platforms.includes(pId); 1536 }, 1537 1538 // are there at lest num pages loaded? 1539 smallPageCount: function(visitor, num) { 1540 return (visitor._viewCount <= Number(num)); 1541 }, 1542 1543 // There was no entry in a specific log file for this visitor: 1544 // note that this will also trigger the "noJavaScript" rule: 1545 noRecord: function(visitor, type) { 1546 if (!visitor._seenBy.includes('srv')) return false; // only if 'srv' is also specified! 1547 return !visitor._seenBy.includes(type); 1548 }, 1549 1550 // there are no referrers in any of the page visits: 1551 noReferrer: function(visitor) { 1552 1553 let r = false; // return value 1554 for (let i = 0; i < visitor._pageViews.length; i++) { 1555 if (!visitor._pageViews[i]._ref) { 1556 r = true; 1557 break; 1558 } 1559 } 1560 return r; 1561 }, 1562 1563 // test for specific client identifiers: 1564 matchesUserAgent: function(visitor, ...list) { 1565 1566 for (let i=0; i<list.length; i++) { 1567 if (visitor.agent == list[i]) { 1568 return true 1569 } 1570 }; 1571 return false; 1572 }, 1573 1574 // unusual combinations of Platform and Client: 1575 combinationTest: function(visitor, ...combinations) { 1576 1577 for (let i=0; i<combinations.length; i++) { 1578 1579 if (visitor._platform.id == combinations[i][0] 1580 && visitor._client.id == combinations[i][1]) { 1581 return true 1582 } 1583 }; 1584 1585 return false; 1586 }, 1587 1588 // is the IP address from a known bot network? 1589 fromKnownBotIP: function(visitor) { 1590 //console.info('fromKnownBotIP()', visitor.ip); 1591 1592 const ipInfo = BotMon.live.data.ipRanges.match(visitor.ip); 1593 1594 if (ipInfo) { 1595 visitor._ipInKnownBotRange = true; 1596 visitor._ipRange = ipInfo; 1597 } 1598 1599 return (ipInfo !== null); 1600 }, 1601 1602 // is the page language mentioned in the client's accepted languages? 1603 // the parameter holds an array of exceptions, i.e. page languages that should be ignored. 1604 matchLang: function(visitor, ...exceptions) { 1605 1606 if (visitor.lang && visitor.accept && exceptions.indexOf(visitor.lang) < 0) { 1607 return (visitor.accept.split(',').indexOf(visitor.lang) < 0); 1608 } 1609 return false; 1610 }, 1611 1612 // the "Accept language" header contains certain entries: 1613 clientAccepts: function(visitor, ...languages) { 1614 //console.info('clientAccepts', visitor.accept, languages); 1615 1616 if (visitor.accept && languages) {; 1617 return ( visitor.accept.split(',').filter(lang => languages.includes(lang)).length > 0 ); 1618 } 1619 return false; 1620 }, 1621 1622 // Is there an accept-language field defined at all? 1623 noAcceptLang: function(visitor) { 1624 1625 if (!visitor.accept || visitor.accept.length <= 0) { // no accept-languages header 1626 return true; 1627 } 1628 // TODO: parametrize this! 1629 return false; 1630 }, 1631 // At least x page views were recorded, but they come within less than y seconds 1632 loadSpeed: function(visitor, minItems, maxTime) { 1633 1634 if (visitor._viewCount >= minItems) { 1635 //console.log('loadSpeed', visitor._pageViews.length, minItems, maxTime); 1636 1637 const pvArr = visitor._pageViews.map(pv => pv._lastSeen).sort(); 1638 1639 let totalTime = 0; 1640 for (let i=1; i < pvArr.length; i++) { 1641 totalTime += (pvArr[i] - pvArr[i-1]); 1642 } 1643 1644 //console.log(' ', totalTime , Math.round(totalTime / (pvArr.length * 1000)), (( totalTime / pvArr.length ) <= maxTime * 1000), visitor.ip); 1645 1646 return (( totalTime / pvArr.length ) <= maxTime * 1000); 1647 } 1648 }, 1649 1650 // Country code matches one of those in the list: 1651 matchesCountry: function(visitor, ...countries) { 1652 1653 // ingore if geoloc is not set or unknown: 1654 if (visitor.geo) { 1655 return (countries.indexOf(visitor.geo) >= 0); 1656 } 1657 return false; 1658 }, 1659 1660 // Country does not match one of the given codes. 1661 notFromCountry: function(visitor, ...countries) { 1662 1663 // ingore if geoloc is not set or unknown: 1664 if (visitor.geo && visitor.geo !== 'ZZ') { 1665 return (countries.indexOf(visitor.geo) < 0); 1666 } 1667 return false; 1668 } 1669 } 1670 }, 1671 1672 /** 1673 * Loads a settings file from the specified list of filenames. 1674 * If the file is successfully loaded, it will call the callback function 1675 * with the loaded JSON data. 1676 * If no file can be loaded, it will display an error message. 1677 * 1678 * @param {string[]} fns - list of filenames to load 1679 * @param {function} callback - function to call with the loaded JSON data 1680 */ 1681 _loadSettingsFile: async function(fns, callback) { 1682 //console.info('BotMon.live.data._loadSettingsFile()', fns); 1683 1684 const kJsonExt = '.json'; 1685 let loaded = false; // if successfully loaded file 1686 1687 for (let i=0; i<fns.length; i++) { 1688 const filename = fns[i] +kJsonExt; 1689 try { 1690 const response = await fetch(DOKU_BASE + 'lib/plugins/botmon/config/' + filename); 1691 if (!response.ok) { 1692 continue; 1693 } else { 1694 loaded = true; 1695 } 1696 const json = await response.json(); 1697 if (callback && typeof callback === 'function') { 1698 callback(json); 1699 } 1700 break; 1701 } catch (e) { 1702 BotMon.live.gui.status.setError("Error while loading the config file: " + filename); 1703 } 1704 } 1705 1706 if (!loaded) { 1707 BotMon.live.gui.status.setError("Could not load a config file."); 1708 } 1709 }, 1710 1711 /** 1712 * Loads a log file (server, page load, or ticker) and parses it. 1713 * @param {String} type - the type of the log file to load (srv, log, or tck) 1714 * @param {Function} [onLoaded] - an optional callback function to call after loading is finished. 1715 */ 1716 loadLogFile: async function(type, onLoaded = undefined) { 1717 //console.info('BotMon.live.data.loadLogFile(',type,')'); 1718 1719 let typeName = ''; 1720 let columns = []; 1721 1722 switch (type) { 1723 case "srv": 1724 typeName = "Server"; 1725 columns = ['ts','ip','pg','id','typ','usr','agent','ref','lang','accept','geo','captcha']; 1726 break; 1727 case "log": 1728 typeName = "Page load"; 1729 columns = ['ts','ip','pg','id','usr','lt','ref','agent']; 1730 break; 1731 case "tck": 1732 typeName = "Ticker"; 1733 columns = ['ts','ip','pg','id','agent']; 1734 break; 1735 default: 1736 console.warn(`Unknown log type ${type}.`); 1737 return; 1738 } 1739 1740 // Show the busy indicator and set the visible status: 1741 BotMon.live.gui.status.showBusy(`Loading ${typeName} log file …`); 1742 1743 // compose the URL from which to load: 1744 const url = BotMon._baseDir + `logs/${BotMon._datestr}.${type}.txt`; 1745 //console.log("Loading:",url); 1746 1747 // fetch the data: 1748 try { 1749 const response = await fetch(url); 1750 if (!response.ok) { 1751 1752 throw new Error(`${response.status} ${response.statusText}`); 1753 1754 } else { 1755 1756 // parse the data: 1757 const logtxt = await response.text(); 1758 if (logtxt.length <= 0) { 1759 throw new Error(`Empty log file ${url}.`); 1760 } 1761 1762 logtxt.split('\n').forEach((line) => { 1763 if (line.trim() === '') return; // skip empty lines 1764 const cols = line.split('\t'); 1765 1766 // assign the columns to an object: 1767 const data = {}; 1768 cols.forEach( (colVal,i) => { 1769 const colName = columns[i] || `col${i}`; 1770 const colValue = (colName == 'ts' ? new Date(colVal) : colVal.trim()); 1771 data[colName] = colValue; 1772 }); 1773 1774 // register the visit in the model: 1775 switch(type) { 1776 case BM_LOGTYPE.SERVER: 1777 BotMon.live.data.model.registerVisit(data, type); 1778 break; 1779 case BM_LOGTYPE.CLIENT: 1780 data.typ = 'js'; 1781 BotMon.live.data.model.updateVisit(data); 1782 break; 1783 case BM_LOGTYPE.TICKER: 1784 data.typ = 'js'; 1785 BotMon.live.data.model.updateTicks(data); 1786 break; 1787 default: 1788 console.warn(`Unknown log type ${type}.`); 1789 return; 1790 } 1791 }); 1792 } 1793 1794 } catch (error) { 1795 BotMon.live.gui.status.setError(`Error while loading the ${typeName} log file: ${error.message} – data may be incomplete.`); 1796 } finally { 1797 BotMon.live.gui.status.hideBusy("Status: Done."); 1798 if (onLoaded) { 1799 onLoaded(); // callback after loading is finished. 1800 } 1801 } 1802 } 1803 }, 1804 1805 gui: { 1806 init: function() { 1807 //console.log('BotMon.live.gui.init()'); 1808 1809 // init sub-objects: 1810 BotMon.t._callInit(this); 1811 }, 1812 1813 tabs: { 1814 init: function() { 1815 //console.log('BotMon.live.gui.tabs.init()'); 1816 1817 /* find and add all existing tabs */ 1818 document.querySelectorAll('#botmon__admin *[role=tablist]') 1819 .forEach((tablist) => { 1820 tablist.querySelectorAll('*[role=tab]') 1821 .forEach( t => t.addEventListener('click', this._onTabClick) ) 1822 }); 1823 }, 1824 1825 /* callback for tab click */ 1826 _onTabClick: function(e) { 1827 //console.log('BotMon.live.gui.tabs._onTabClick()'); 1828 1829 /* reusable constants: */ 1830 const kAriaSelected = 'aria-selected'; 1831 const kAriaControls = 'aria-controls'; 1832 const kTrue = 'true'; 1833 const kFalse = 'false'; 1834 const kHidden = 'hidden'; 1835 1836 /* cancel default action */ 1837 e.preventDefault(); 1838 1839 /* if the active tab is clicked, do nothing: */ 1840 let selState = this.getAttribute(kAriaSelected); 1841 if ( selState && selState == kTrue ) { 1842 return; 1843 } 1844 1845 /* find the active tab element: */ 1846 var aItem = null; 1847 let tablist = this.parentNode; 1848 while (tablist.getAttribute('role') !== 'tablist') { 1849 tablist = tablist.parentNode; 1850 } 1851 1852 if (tablist.getAttribute('role') == 'tablist') { 1853 let lis = tablist.querySelectorAll('*[role=tab]'); 1854 lis.forEach( (it) => { 1855 let selected = it.getAttribute(kAriaSelected); 1856 if ( selected && selected == kTrue ) { 1857 aItem = it; 1858 } 1859 }); 1860 } 1861 1862 /* swap the active states: */ 1863 this.setAttribute(kAriaSelected, kTrue); 1864 if (aItem) { 1865 aItem.setAttribute(kAriaSelected, kFalse); 1866 let aId = aItem.getAttribute(kAriaControls); 1867 let aObj = document.getElementById(aId); 1868 if (aObj) aObj.hidden = true; 1869 } 1870 1871 /* show the new panel: */ 1872 let nId = this.getAttribute(kAriaControls); 1873 let nObj = document.getElementById(nId); 1874 if (nObj) nObj.hidden = false; 1875 } 1876 }, 1877 1878 /* The Overview / web metrics section of the live tab */ 1879 overview: { 1880 /** 1881 * Populates the overview part of the today tab with the analytics data. 1882 * 1883 * @method make 1884 * @memberof BotMon.live.gui.overview 1885 */ 1886 make: function() { 1887 1888 const data = BotMon.live.data.analytics.data; 1889 1890 const maxItemsPerList = 5; // how many list items to show? 1891 1892 const kNoData = '–'; // shown when data is missing 1893 1894 // shortcut for neater code: 1895 const makeElement = BotMon.t._makeElement; 1896 1897 const botsVsHumans = document.getElementById('botmon__today__botsvshumans'); 1898 if (botsVsHumans) { 1899 botsVsHumans.appendChild(makeElement('dt', {}, "Page views")); 1900 1901 for (let i = 0; i <= 5; i++) { 1902 const dd = makeElement('dd'); 1903 let title = ''; 1904 let value = ''; 1905 switch(i) { 1906 case 0: 1907 title = "Known bots:"; 1908 value = data.bots.known || kNoData; 1909 break; 1910 case 1: 1911 title = "Suspected bots:"; 1912 value = data.bots.suspected || kNoData; 1913 break; 1914 case 2: 1915 title = "Probably humans:"; 1916 value = data.bots.human || kNoData; 1917 break; 1918 case 3: 1919 title = "Registered users:"; 1920 value = data.bots.users || kNoData; 1921 break; 1922 case 4: 1923 title = "Total:"; 1924 value = data.totalPageViews || kNoData; 1925 break; 1926 case 5: 1927 title = "Bots-humans ratio:"; 1928 value = BotMon.t._getRatio(data.bots.suspected + data.bots.known, data.bots.users + data.bots.human, 100); 1929 break; 1930 default: 1931 console.warn(`Unknown list type ${i}.`); 1932 } 1933 dd.appendChild(makeElement('span', {}, title)); 1934 dd.appendChild(makeElement('strong', {}, value)); 1935 botsVsHumans.appendChild(dd); 1936 } 1937 } 1938 1939 // update known bots list: 1940 const botElement = document.getElementById('botmon__botslist'); /* Known bots */ 1941 if (botElement) { 1942 botElement.appendChild(makeElement('dt', {}, `Top known bots`)); 1943 1944 let botList = BotMon.live.data.analytics.getTopBots(maxItemsPerList); 1945 botList.forEach( (botInfo) => { 1946 const bli = makeElement('dd'); 1947 bli.appendChild(makeElement('span', {'class': 'has_icon bot bot_' + botInfo.id }, botInfo.name)); 1948 bli.appendChild(makeElement('span', {'class': 'count' }, botInfo.count)); 1949 botElement.append(bli) 1950 }); 1951 } 1952 1953 // update the suspected bot IP ranges list: 1954 const botIps = document.getElementById('botmon__botips'); 1955 if (botIps) { 1956 botIps.appendChild(makeElement('dt', {}, "Top bot Networks")); 1957 1958 const ispList = BotMon.live.data.analytics.getTopBotISPs(5); 1959 //console.log(ispList); 1960 ispList.forEach( (netInfo) => { 1961 const li = makeElement('dd'); 1962 li.appendChild(makeElement('span', {'class': 'has_icon ipaddr ip' + netInfo.typ }, netInfo.name)); 1963 li.appendChild(makeElement('span', {'class': 'count' }, netInfo.count)); 1964 botIps.append(li) 1965 }); 1966 } 1967 1968 // update the top bot countries list: 1969 const botCountries = document.getElementById('botmon__botcountries'); 1970 if (botCountries) { 1971 botCountries.appendChild(makeElement('dt', {}, `Top bot Countries`)); 1972 const countryList = BotMon.live.data.analytics.getCountryList('bot', 5); 1973 countryList.forEach( (cInfo) => { 1974 const cLi = makeElement('dd'); 1975 cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name)); 1976 cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count)); 1977 botCountries.appendChild(cLi); 1978 }); 1979 } 1980 1981 // update the webmetrics overview: 1982 const wmoverview = document.getElementById('botmon__today__wm_overview'); 1983 if (wmoverview) { 1984 1985 const humanVisits = BotMon.live.data.analytics.groups.users.length + BotMon.live.data.analytics.groups.humans.length; 1986 const bounceRate = Math.round(100 * (BotMon.live.data.analytics.getBounceCount('users') + BotMon.live.data.analytics.getBounceCount('humans')) / humanVisits); 1987 1988 wmoverview.appendChild(makeElement('dt', {}, "Humans’ metrics")); 1989 for (let i = 0; i <= 4; i++) { 1990 const dd = makeElement('dd'); 1991 let title = ''; 1992 let value = ''; 1993 switch(i) { 1994 case 0: 1995 title = "Registered users’ page views:"; 1996 value = data.bots.users || kNoData; 1997 break; 1998 case 1: 1999 title = "“Probably humans” page views:"; 2000 value = data.bots.human || kNoData; 2001 break; 2002 case 2: 2003 title = "Total human page views:"; 2004 value = (data.bots.users + data.bots.human) || kNoData; 2005 break; 2006 case 3: 2007 title = "Total human visits:"; 2008 value = humanVisits || kNoData; 2009 break; 2010 case 4: 2011 title = "Humans’ bounce rate:"; 2012 value = bounceRate + '%'; 2013 break; 2014 default: 2015 console.warn(`Unknown list type ${i}.`); 2016 } 2017 dd.appendChild(makeElement('span', {}, title)); 2018 dd.appendChild(makeElement('strong', {}, value)); 2019 wmoverview.appendChild(dd); 2020 } 2021 } 2022 2023 // update the webmetrics clients list: 2024 const wmclients = document.getElementById('botmon__today__wm_clients'); 2025 if (wmclients) { 2026 2027 wmclients.appendChild(makeElement('dt', {}, "Browsers")); 2028 2029 const clientList = BotMon.live.data.analytics.getTopBrowsers(maxItemsPerList); 2030 if (clientList) { 2031 clientList.forEach( (cInfo) => { 2032 const cDd = makeElement('dd'); 2033 cDd.appendChild(makeElement('span', {'class': 'has_icon client cl_' + cInfo.id }, ( cInfo.name ? cInfo.name : cInfo.id))); 2034 cDd.appendChild(makeElement('span', { 2035 'class': 'count', 2036 'title': cInfo.count + " page views" 2037 }, cInfo.pct.toFixed(1) + '%')); 2038 wmclients.appendChild(cDd); 2039 }); 2040 } 2041 } 2042 2043 // update the webmetrics platforms list: 2044 const wmplatforms = document.getElementById('botmon__today__wm_platforms'); 2045 if (wmplatforms) { 2046 2047 wmplatforms.appendChild(makeElement('dt', {}, "Platforms")); 2048 2049 const pfList = BotMon.live.data.analytics.getTopPlatforms(maxItemsPerList); 2050 if (pfList) { 2051 pfList.forEach( (pInfo) => { 2052 const pDd = makeElement('dd'); 2053 pDd.appendChild(makeElement('span', {'class': 'has_icon platform pf_' + pInfo.id }, ( pInfo.name ? pInfo.name : pInfo.id))); 2054 pDd.appendChild(makeElement('span', { 2055 'class': 'count', 2056 'title': pInfo.count + " page views" 2057 }, pInfo.pct.toFixed(1) + '%')); 2058 wmplatforms.appendChild(pDd); 2059 }); 2060 } 2061 } 2062 2063 // update the top bot countries list: 2064 const usrCountries = document.getElementById('botmon__today__wm_countries'); 2065 if (usrCountries) { 2066 usrCountries.appendChild(makeElement('dt', {}, `Top visitor Countries:`)); 2067 const usrCtryList = BotMon.live.data.analytics.getCountryList('human', 5); 2068 usrCtryList.forEach( (cInfo) => { 2069 const cLi = makeElement('dd'); 2070 cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name)); 2071 cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count)); 2072 usrCountries.appendChild(cLi); 2073 }); 2074 } 2075 2076 // update the top pages; 2077 const wmpages = document.getElementById('botmon__today__wm_pages'); 2078 if (wmpages) { 2079 2080 wmpages.appendChild(makeElement('dt', {}, "Top pages")); 2081 2082 const pgList = BotMon.live.data.analytics.getTopPages(maxItemsPerList); 2083 if (pgList) { 2084 pgList.forEach( (pgInfo) => { 2085 const pgDd = makeElement('dd'); 2086 pgDd.appendChild(makeElement('a', { 2087 'class': 'page_icon', 2088 'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(pgInfo.id), 2089 'target': 'preview', 2090 'title': "PageID: " + pgInfo.id 2091 }, pgInfo.id)); 2092 pgDd.appendChild(makeElement('span', { 2093 'class': 'count', 2094 'title': pgInfo.count + " page views" 2095 }, pgInfo.count)); 2096 wmpages.appendChild(pgDd); 2097 }); 2098 } 2099 } 2100 2101 // update the top referrers; 2102 const wmreferers = document.getElementById('botmon__today__wm_referers'); 2103 if (wmreferers) { 2104 2105 wmreferers.appendChild(makeElement('dt', {}, "Referers")); 2106 2107 const refList = BotMon.live.data.analytics.getTopReferers(maxItemsPerList); 2108 if (refList) { 2109 refList.forEach( (rInfo) => { 2110 const rDd = makeElement('dd'); 2111 rDd.appendChild(makeElement('span', {'class': 'has_icon referer ref_' + rInfo.id }, rInfo.name)); 2112 rDd.appendChild(makeElement('span', { 2113 'class': 'count', 2114 'title': rInfo.count + " references" 2115 }, rInfo.pct.toFixed(1) + '%')); 2116 wmreferers.appendChild(rDd); 2117 }); 2118 } 2119 } 2120 } 2121 }, 2122 2123 status: { 2124 setText: function(txt) { 2125 const el = document.getElementById('botmon__today__status'); 2126 if (el && BotMon.live.gui.status._errorCount <= 0) { 2127 el.innerText = txt; 2128 } 2129 }, 2130 2131 setTitle: function(html) { 2132 const el = document.getElementById('botmon__today__title'); 2133 if (el) { 2134 el.innerHTML = html; 2135 } 2136 }, 2137 2138 setError: function(txt) { 2139 console.error(txt); 2140 BotMon.live.gui.status._errorCount += 1; 2141 const el = document.getElementById('botmon__today__status'); 2142 if (el) { 2143 el.innerText = "Data may be incomplete."; 2144 el.classList.add('error'); 2145 } 2146 }, 2147 _errorCount: 0, 2148 2149 showBusy: function(txt = null) { 2150 BotMon.live.gui.status._busyCount += 1; 2151 const el = document.getElementById('botmon__today__busy'); 2152 if (el) { 2153 el.style.display = 'inline-block'; 2154 } 2155 if (txt) BotMon.live.gui.status.setText(txt); 2156 }, 2157 _busyCount: 0, 2158 2159 hideBusy: function(txt = null) { 2160 const el = document.getElementById('botmon__today__busy'); 2161 BotMon.live.gui.status._busyCount -= 1; 2162 if (BotMon.live.gui.status._busyCount <= 0) { 2163 if (el) el.style.display = 'none'; 2164 if (txt) BotMon.live.gui.status.setText(txt); 2165 } 2166 } 2167 }, 2168 2169 lists: { 2170 init: function() { 2171 2172 // function shortcut: 2173 const makeElement = BotMon.t._makeElement; 2174 2175 const parent = document.getElementById('botmon__today__visitorlists'); 2176 if (parent) { 2177 2178 for (let i=0; i < 4; i++) { 2179 2180 // change the id and title by number: 2181 let listTitle = ''; 2182 let listId = ''; 2183 let infolink = null; 2184 switch (i) { 2185 case 0: 2186 listTitle = "Registered users"; 2187 listId = 'users'; 2188 break; 2189 case 1: 2190 listTitle = "Probably humans"; 2191 listId = 'humans'; 2192 break; 2193 case 2: 2194 listTitle = "Suspected bots"; 2195 listId = 'suspectedBots'; 2196 infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/suspected_bots'; 2197 break; 2198 case 3: 2199 listTitle = "Known bots"; 2200 listId = 'knownBots'; 2201 infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/known_bots'; 2202 break; 2203 default: 2204 console.warn('Unknown list number.'); 2205 } 2206 2207 const details = makeElement('details', { 2208 'data-group': listId, 2209 'data-loaded': false 2210 }); 2211 const title = details.appendChild(makeElement('summary')); 2212 title.appendChild(makeElement('span', {'class': 'title'}, listTitle)); 2213 if (infolink) { 2214 title.appendChild(makeElement('a', { 2215 'class': 'ext_info', 2216 'target': '_blank', 2217 'href': infolink, 2218 'title': "More information" 2219 }, "Info")); 2220 } 2221 details.addEventListener("toggle", this._onDetailsToggle); 2222 2223 parent.appendChild(details); 2224 2225 } 2226 } 2227 }, 2228 2229 _onDetailsToggle: function(e) { 2230 //console.info('BotMon.live.gui.lists._onDetailsToggle()'); 2231 2232 const target = e.target; 2233 2234 if (target.getAttribute('data-loaded') == 'false') { // only if not loaded yet 2235 target.setAttribute('data-loaded', 'loading'); 2236 2237 const fillType = target.getAttribute('data-group'); 2238 const fillList = BotMon.live.data.analytics.groups[fillType]; 2239 if (fillList && fillList.length > 0) { 2240 2241 const ul = BotMon.t._makeElement('ul'); 2242 2243 fillList.forEach( (it) => { 2244 ul.appendChild(BotMon.live.gui.lists._makeVisitorItem(it, fillType)); 2245 }); 2246 2247 target.appendChild(ul); 2248 target.setAttribute('data-loaded', 'true'); 2249 } else { 2250 target.setAttribute('data-loaded', 'false'); 2251 } 2252 2253 } 2254 }, 2255 2256 _makeVisitorItem: function(data, type) { 2257 2258 // shortcut for neater code: 2259 const make = BotMon.t._makeElement; 2260 2261 let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' ); 2262 if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0'; 2263 2264 const platformName = (data._platform ? data._platform.n : 'Unknown'); 2265 const clientName = (data._client ? data._client.n: 'Unknown'); 2266 2267 const sumClass = ( !data._seenBy || data._seenBy.indexOf(BM_LOGTYPE.SERVER) < 0 ? 'noServer' : 'hasServer'); 2268 2269 const li = make('li'); // root list item 2270 const details = make('details'); 2271 const summary = make('summary', { 2272 'class': sumClass 2273 }); 2274 details.appendChild(summary); 2275 2276 const span1 = make('span'); /* left-hand group */ 2277 2278 if (data._type !== BM_USERTYPE.KNOWN_BOT) { /* No platform/client for bots */ 2279 span1.appendChild(make('span', { /* Platform */ 2280 'class': 'icon_only platform pf_' + (data._platform ? data._platform.id : 'unknown'), 2281 'title': "Platform: " + platformName 2282 }, platformName)); 2283 2284 span1.appendChild(make('span', { /* Client */ 2285 'class': 'icon_only client client cl_' + (data._client ? data._client.id : 'unknown'), 2286 'title': "Client: " + clientName 2287 }, clientName)); 2288 } 2289 2290 // identifier: 2291 if (data._type == BM_USERTYPE.KNOWN_BOT) { /* Bot only */ 2292 2293 const botName = ( data._bot && data._bot.n ? data._bot.n : "Unknown"); 2294 span1.appendChild(make('span', { /* Bot */ 2295 'class': 'has_icon bot bot_' + (data._bot ? data._bot.id : 'unknown'), 2296 'title': "Bot: " + botName 2297 }, botName)); 2298 2299 } else if (data._type == BM_USERTYPE.KNOWN_USER) { /* User only */ 2300 2301 span1.appendChild(make('span', { /* User */ 2302 'class': 'has_icon user_known', 2303 'title': "User: " + data.usr 2304 }, data.usr)); 2305 2306 } else { /* others */ 2307 2308 2309 span1.appendChild(make('span', { // IP-Address 2310 'class': 'has_icon ipaddr ip' + ipType, 2311 'title': "IP-Address: " + data.ip 2312 }, data.ip)); 2313 2314 /*span1.appendChild(make('span', { // Internal ID 2315 'class': 'has_icon session typ_' + data.typ, 2316 'title': "ID: " + data.id 2317 }, data.id));*/ 2318 } 2319 2320 span1.appendChild(make('span', { /* page views */ 2321 'class': 'has_icon pageseen', 2322 'title': data._pageViews.length + " page load(s)" 2323 }, data._pageViews.length)); 2324 2325 2326 // referer icons: 2327 if ((data._type == BM_USERTYPE.PROBABLY_HUMAN || data._type == BM_USERTYPE.LIKELY_BOT) && data.ref) { 2328 const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref); 2329 span1.appendChild(make('span', { 2330 'class': 'icon_only referer ref_' + refInfo.id, 2331 'title': "Referer: " + data.ref 2332 }, refInfo.n)); 2333 } 2334 2335 summary.appendChild(span1); 2336 const span2 = make('span'); /* right-hand group */ 2337 2338 // country flag: 2339 if (data.geo && data.geo !== 'ZZ') { 2340 span2.appendChild(make('span', { 2341 'class': 'icon_only country ctry_' + data.geo.toLowerCase(), 2342 'data-ctry': data.geo, 2343 'title': "Country: " + ( data._country || "Unknown") 2344 }, ( data._country || "Unknown") )); 2345 } 2346 2347 span2.appendChild(make('span', { // seen-by icon: 2348 'class': 'icon_only seenby sb_' + data._seenBy.join(''), 2349 'title': "Seen by: " + data._seenBy.join('+') 2350 }, data._seenBy.join(', '))); 2351 2352 const captchaCode = '' + ( data._captcha['Y'] > 0 ? 'Y' : '' ) + ( data._captcha['N'] > 0 ? 'N' : '' ) + ( data._captcha['W'] > 0 ? 'W' : '' ); 2353 if (captchaCode !== '') { 2354 span2.appendChild(make('span', { // captcha status 2355 'class': 'icon_only captcha cap_' + captchaCode, 2356 }, captchaCode)); 2357 } 2358 2359 summary.appendChild(span2); 2360 2361 // add details expandable section: 2362 details.appendChild(BotMon.live.gui.lists._makeVisitorDetails(data, type)); 2363 2364 li.appendChild(details); 2365 return li; 2366 }, 2367 2368 _makeVisitorDetails: function(data, type) { 2369 2370 // shortcut for neater code: 2371 const make = BotMon.t._makeElement; 2372 2373 let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' ); 2374 if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0'; 2375 const platformName = (data._platform ? data._platform.n : 'Unknown'); 2376 const clientName = (data._client ? data._client.n: 'Unknown'); 2377 2378 const dl = make('dl', {'class': 'visitor_details'}); 2379 2380 if (data._type == BM_USERTYPE.KNOWN_BOT) { 2381 2382 dl.appendChild(make('dt', {}, "Bot name:")); /* bot info */ 2383 dl.appendChild(make('dd', {'class': 'icon_only bot bot_' + (data._bot ? data._bot.id : 'unknown')}, 2384 (data._bot ? data._bot.n : 'Unknown'))); 2385 2386 if (data._bot && data._bot.url) { 2387 dl.appendChild(make('dt', {}, "Bot info:")); /* bot info */ 2388 const botInfoDd = dl.appendChild(make('dd')); 2389 botInfoDd.appendChild(make('a', { 2390 'href': data._bot.url, 2391 'target': '_blank' 2392 }, data._bot.url)); /* bot info link*/ 2393 2394 } 2395 2396 } else { /* not for bots */ 2397 2398 dl.appendChild(make('dt', {}, "Client:")); /* client */ 2399 dl.appendChild(make('dd', {'class': 'has_icon client cl_' + (data._client ? data._client.id : 'unknown')}, 2400 clientName + ( data._client.v > 0 ? ' (' + data._client.v + ')' : '' ) )); 2401 2402 dl.appendChild(make('dt', {}, "Platform:")); /* platform */ 2403 dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')}, 2404 platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); 2405 2406 /*dl.appendChild(make('dt', {}, "ID:")); 2407 dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/ 2408 } 2409 2410 dl.appendChild(make('dt', {}, "IP-Address:")); 2411 const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType}); 2412 ipItem.appendChild(make('span', {'class': 'address'} , data.ip)); 2413 ipItem.appendChild(make('a', { 2414 'class': 'icon_only extlink ipinfo', 2415 'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`, 2416 'target': 'ipinfo', 2417 'title': "View this address on IPInfo.io" 2418 } , "DNS Info")); 2419 ipItem.appendChild(make('a', { 2420 'class': 'icon_only extlink abuseipdb', 2421 'href': `https://www.abuseipdb.com/check/${encodeURIComponent(data.ip)}`, 2422 'target': 'abuseipdb', 2423 'title': "Check this address on AbuseIPDB.com" 2424 } , "Check on AbuseIPDB")); 2425 dl.appendChild(ipItem); 2426 2427 if (Math.abs(data._lastSeen - data._firstSeen) < 100) { 2428 dl.appendChild(make('dt', {}, "Seen:")); 2429 dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString())); 2430 } else { 2431 dl.appendChild(make('dt', {}, "First seen:")); 2432 dl.appendChild(make('dd', {'class': 'firstSeen'}, data._firstSeen.toLocaleString())); 2433 dl.appendChild(make('dt', {}, "Last seen:")); 2434 dl.appendChild(make('dd', {'class': 'lastSeen'}, data._lastSeen.toLocaleString())); 2435 } 2436 2437 dl.appendChild(make('dt', {}, "Actions:")); 2438 dl.appendChild(make('dd', {'class': 'views'}, 2439 "Page loads: " + data._loadCount.toString() + 2440 ( data._captcha['Y'] > 0 || data._captcha['W'] || data._captcha['-'] > 0 ? ", captchas: " + (data._captcha['Y']+data._captcha['W']+data._captcha['-']).toString() : '') + 2441 ", views: " + data._viewCount.toString() 2442 )); 2443 2444 dl.appendChild(make('dt', {}, "User-Agent:")); 2445 dl.appendChild(make('dd', {'class': 'agent'}, data.agent)); 2446 2447 if (data.ref && data.ref !== '') { 2448 dl.appendChild(make('dt', {}, "Referrer:")); 2449 2450 const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref); 2451 const refDd = dl.appendChild(make('dd', { 2452 'class': 'has_icon referer ref_' + refInfo.id 2453 })); 2454 refDd.appendChild(make('a', { 2455 'href': data.ref, 2456 'target': 'refTarget' 2457 }, data.ref)); 2458 } 2459 2460 dl.appendChild(make('dt', {}, "Languages:")); 2461 dl.appendChild(make('dd', {'class': 'langs'}, ` [${data.accept}]`)); 2462 2463 if (data.geo && data.geo !=='') { 2464 dl.appendChild(make('dt', {}, "Location:")); 2465 dl.appendChild(make('dd', { 2466 'class': 'has_icon country ctry_' + data.geo.toLowerCase(), 2467 'data-ctry': data.geo, 2468 'title': "Country: " + data._country 2469 }, data._country + ' (' + data.geo + ')')); 2470 } 2471 if (data.captcha && data.captcha !=='') { 2472 dl.appendChild(make('dt', {}, "Captcha-status:")); 2473 dl.appendChild(make('dd', { 2474 'class': 'captcha' 2475 }, data.captcha)); 2476 } 2477 2478 dl.appendChild(make('dt', {}, "Session ID:")); 2479 dl.appendChild(make('dd', {'class': 'has_icon session typ_' + data.typ}, data.id)); 2480 2481 dl.appendChild(make('dt', {}, "Seen by:")); 2482 dl.appendChild(make('dd', {'class': 'has_icon seenby sb_' + data._seenBy.join('')}, data._seenBy.join(', ') )); 2483 2484 dl.appendChild(make('dt', {}, "Visited pages:")); 2485 const pagesDd = make('dd', {'class': 'pages'}); 2486 const pageList = make('ul'); 2487 2488 /* list all page views */ 2489 data._pageViews.sort( (a, b) => a._firstSeen - b._firstSeen ); 2490 data._pageViews.forEach( (page) => { 2491 pageList.appendChild(BotMon.live.gui.lists._makePageViewItem(page)); 2492 }); 2493 pagesDd.appendChild(pageList); 2494 dl.appendChild(pagesDd); 2495 2496 /* bot evaluation rating */ 2497 if (data._type !== BM_USERTYPE.KNOWN_BOT && data._type !== BM_USERTYPE.KNOWN_USER) { 2498 dl.appendChild(make('dt', undefined, "Bot rating:")); 2499 dl.appendChild(make('dd', {'class': 'bot-rating'}, ( data._botVal ? data._botVal : '–' ) + ' (of ' + BotMon.live.data.rules._threshold + ')')); 2500 2501 /* add bot evaluation details: */ 2502 if (data._eval) { 2503 dl.appendChild(make('dt', {}, "Bot evaluation:")); 2504 const evalDd = make('dd', {'class': 'eval'}); 2505 const testList = make('ul'); 2506 data._eval.forEach( test => { 2507 2508 const tObj = BotMon.live.data.rules.getRuleInfo(test); 2509 let tDesc = tObj ? tObj.desc : test; 2510 2511 // special case for Bot IP range test: 2512 if (tObj.func == 'fromKnownBotIP') { 2513 const rangeInfo = BotMon.live.data.ipRanges.match(data.ip); 2514 if (rangeInfo) { 2515 const owner = BotMon.live.data.ipRanges.getOwner(rangeInfo.g); 2516 tDesc += ' (range: “' + rangeInfo.cidr + '”, ' + owner + ')'; 2517 } 2518 } 2519 2520 // create the entry field 2521 const tstLi = make('li'); 2522 tstLi.appendChild(make('span', { 2523 'data-testid': test 2524 }, tDesc)); 2525 tstLi.appendChild(make('span', {}, ( tObj ? tObj.bot : '—') )); 2526 testList.appendChild(tstLi); 2527 }); 2528 2529 // add total row 2530 const tst2Li = make('li', { 2531 'class': 'total' 2532 }); 2533 /*tst2Li.appendChild(make('span', {}, "Total:")); 2534 tst2Li.appendChild(make('span', {}, data._botVal)); 2535 testList.appendChild(tst2Li);*/ 2536 2537 evalDd.appendChild(testList); 2538 dl.appendChild(evalDd); 2539 } 2540 } 2541 // return the element to add to the UI: 2542 return dl; 2543 }, 2544 2545 // make a page view item: 2546 _makePageViewItem: function(page) { 2547 //console.log("makePageViewItem:",page); 2548 2549 // shortcut for neater code: 2550 const make = BotMon.t._makeElement; 2551 2552 // the actual list item: 2553 const pgLi = make('li'); 2554 2555 const row1 = make('div', {'class': 'row'}); 2556 2557 row1.appendChild(make('a', { // page id is the left group 2558 'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(page.pg), 2559 'target': 'preview', 2560 'hreflang': page.lang, 2561 'title': "PageID: " + page.pg 2562 }, page.pg)); /* DW Page ID */ 2563 2564 const rightGroup = row1.appendChild(make('div')); // right-hand group 2565 2566 // get the time difference: 2567 rightGroup.appendChild(make('span', { 2568 'class': 'first-seen', 2569 'title': "First visited: " + page._firstSeen.toLocaleString() + " UTC" 2570 }, BotMon.t._formatTime(page._firstSeen))); 2571 2572 rightGroup.appendChild(make('span', { /* page loads */ 2573 'class': 'has_icon pageviews', 2574 'title': page._viewCount.toString() + " page load(s)" 2575 }, page._viewCount.toString())); 2576 2577 pgLi.appendChild(row1); 2578 2579 /* LINE 2 */ 2580 2581 const row2 = make('div', {'class': 'row'}); 2582 2583 // page referrer: 2584 if (page._ref) { 2585 row2.appendChild(make('span', { 2586 'class': 'referer', 2587 'title': "Referrer: " + page._ref.href 2588 }, page._ref.hostname)); 2589 } else { 2590 row2.appendChild(make('span', { 2591 'class': 'referer' 2592 }, "No referer")); 2593 } 2594 2595 // visit duration: 2596 let visitTimeStr = "Bounce"; 2597 const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime(); 2598 if (visitDuration > 0) { 2599 visitTimeStr = Math.floor(visitDuration / 1000) + "s"; 2600 } 2601 const tDiff = BotMon.t._formatTimeDiff(page._firstSeen, page._lastSeen); 2602 if (tDiff) { 2603 row2.appendChild(make('span', {'class': 'visit-length', 'title': 'Last seen: ' + page._lastSeen.toLocaleString()}, tDiff)); 2604 } else { 2605 row2.appendChild(make('span', { 2606 'class': 'bounce', 2607 'title': "Visitor bounced"}, "Bounce")); 2608 } 2609 2610 pgLi.appendChild(row2); 2611 2612 return pgLi; 2613 } 2614 } 2615 } 2616}; 2617 2618/* launch only if the BotMon admin panel is open: */ 2619if (document.getElementById('botmon__admin')) { 2620 BotMon.init(); 2621}