xref: /plugin/botmon/admin.js (revision 6b6cd387da613838df43ec1dc8194b7d3a057341) !
1"use strict";
2/* DokuWiki BotMon Plugin Script file */
3/* 14.10.2025 - 0.5.0 - pre-release */
4/* Author: Sascha Leib <ad@hominem.info> */
5
6// enumeration of user types:
7const BM_USERTYPE = Object.freeze({
8	'UNKNOWN': 'unknown',
9	'KNOWN_USER': 'user',
10	'PROBABLY_HUMAN': 'human',
11	'LIKELY_BOT': 'likely_bot',
12	'KNOWN_BOT': 'known_bot'
13});
14
15// enumeration of log types:
16const BM_LOGTYPE = Object.freeze({
17	'SERVER': 'srv',
18	'CLIENT': 'log',
19	'TICKER': 'tck'
20});
21
22// enumeration of IP versions:
23const BM_IPVERSION = Object.freeze({
24	'IPv4': 4,
25	'IPv6': 6
26});
27
28/* BotMon root object */
29const BotMon = {
30
31	init: function() {
32		//console.info('BotMon.init()');
33
34		// find the plugin basedir:
35		this._baseDir = document.currentScript.src.substring(0, document.currentScript.src.lastIndexOf('/')+1);
36
37		// read the page language from the DOM:
38		this._lang = document.getRootNode().documentElement.lang || this._lang;
39
40		// get the time offset:
41		this._timeDiff = BotMon.t._getTimeOffset();
42
43		// get yesterday's date:
44		let d = new Date();
45		if (BMSettings.showday == 'yesterday') d.setDate(d.getDate() - 1);
46		this._datestr = d.toISOString().slice(0, 10);
47
48		// init the sub-objects:
49		BotMon.t._callInit(this);
50	},
51
52	_baseDir: null,
53	_lang: 'en',
54	_datestr: '',
55	_timeDiff: '',
56
57	/* internal tools */
58	t: {
59		/* helper function to call inits of sub-objects */
60		_callInit: function(obj) {
61			//console.info('BotMon.t._callInit(obj=',obj,')');
62
63			/* call init / _init on each sub-object: */
64			Object.keys(obj).forEach( (key,i) => {
65				const sub = obj[key];
66				let init = null;
67				if (typeof sub === 'object' && sub.init) {
68					init = sub.init;
69				}
70
71				// bind to object
72				if (typeof init == 'function') {
73					const init2 = init.bind(sub);
74					init2(obj);
75				}
76			});
77		},
78
79		/* helper function to calculate the time difference to UTC: */
80		_getTimeOffset: function() {
81			const now = new Date();
82			let offset = now.getTimezoneOffset(); // in minutes
83			const sign = Math.sign(offset); // +1 or -1
84			offset = Math.abs(offset); // always positive
85
86			let hours = 0;
87			while (offset >= 60) {
88				hours += 1;
89				offset -= 60;
90			}
91			return ( hours > 0 ? sign * hours + ' h' : '') + (offset > 0 ? ` ${offset} min` : '');
92		},
93
94		/* helper function to create a new element with all attributes and text content */
95		_makeElement: function(name, atlist = undefined, text = undefined) {
96			var r = null;
97			try {
98				r = document.createElement(name);
99				if (atlist) {
100					for (let attr in atlist) {
101						r.setAttribute(attr, atlist[attr]);
102					}
103				}
104				if (text) {
105					r.textContent = text.toString();
106				}
107			} catch(e) {
108				console.error(e);
109			}
110			return r;
111		},
112
113		/* helper to convert an ip address string to a normalised format: */
114		_ip2Num: function(ip) {
115			if (!ip) {
116				return 'null';
117			} else if (ip.indexOf(':') > 0) { /* IP6 */
118				return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(':'));
119			} else { /* IP4 */
120				return ip.split('.').map(d => ('000'+d).slice(-3) ).join('.');
121			}
122		},
123
124		/* helper function to format a Date object to show only the time. */
125		/* returns String */
126		_formatTime: function(date) {
127
128			if (date) {
129				return date.getHours() + ':' + ('0'+date.getMinutes()).slice(-2) + ':' + ('0'+date.getSeconds()).slice(-2);
130			} else {
131				return null;
132			}
133
134		},
135
136		/* helper function to show a time difference in seconds or minutes */
137		/* returns String */
138		_formatTimeDiff: function(dateA, dateB) {
139
140			// if the second date is ealier, swap them:
141			if (dateA > dateB) dateB = [dateA, dateA = dateB][0];
142
143			// get the difference in milliseconds:
144			let ms = dateB - dateA;
145
146			if (ms > 50) { /* ignore small time spans */
147				const h = Math.floor((ms / (1000 * 60 * 60)) % 24);
148				const m = Math.floor((ms / (1000 * 60)) % 60);
149				const s = Math.floor((ms / 1000) % 60);
150
151				return ( h>0 ? h + 'h ': '') + ( m>0 ? m + 'm ': '') + ( s>0 ? s + 's': '');
152			}
153
154			return null;
155
156		},
157
158		// calcualte a reduced ration between two numbers
159		// adapted from https://stackoverflow.com/questions/3946373/math-in-js-how-do-i-get-a-ratio-from-a-percentage
160		_getRatio: function(a, b, tolerance) {
161
162			var bg = b;
163			var sm = a;
164			if (a == 0 || b == 0) return '—';
165			if (a > b) {
166				var bg = a;
167				var sm = b;
168			}
169
170			for (var i = 1; i < 1000000; i++) {
171				var d = sm / i;
172				var res = bg / d;
173				var howClose = Math.abs(res - res.toFixed(0));
174				if (howClose < tolerance) {
175					if (a > b) {
176						return res.toFixed(0) + ':' + i;
177					} else {
178						return i + ':' + res.toFixed(0);
179					}
180				}
181			}
182		}
183	}
184};
185
186/* everything specific to the 'Latest' tab is self-contained in the 'live' object: */
187BotMon.live = {
188	init: function() {
189		//console.info('BotMon.live.init()');
190
191		// set the title:
192		const tDiff = '<abbr title="Coordinated Universal Time">UTC</abbr> ' + (BotMon._timeDiff != '' ? ` (offset: ${BotMon._timeDiff}` : '' ) + ')';
193		BotMon.live.gui.status.setTitle(`Data for <time datetime="${BotMon._datestr}">${BotMon._datestr}</time> ${tDiff}`);
194
195		// init sub-objects:
196		BotMon.t._callInit(this);
197	},
198
199	data: {
200		init: function() {
201			//console.info('BotMon.live.data.init()');
202
203			// call sub-inits:
204			BotMon.t._callInit(this);
205		},
206
207		// this will be called when the known json files are done loading:
208		_dispatch: function(file) {
209			//console.info('BotMon.live.data._dispatch(,',file,')');
210
211			// shortcut to make code more readable:
212			const data = BotMon.live.data;
213
214			// set the flags:
215			switch(file) {
216				case 'rules':
217					data._dispatchRulesLoaded = true;
218					break;
219				case 'ipranges':
220					data._dispatchIPRangesLoaded = true;
221					break;
222				case 'bots':
223					data._dispatchBotsLoaded = true;
224					break;
225				case 'clients':
226					data._dispatchClientsLoaded = true;
227					break;
228				case 'platforms':
229					data._dispatchPlatformsLoaded = true;
230					break;
231				default:
232					// ignore
233			}
234
235			// are all the flags set?
236			if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded && data._dispatchIPRangesLoaded) {
237				// chain the log files loading:
238				BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded);
239			}
240		},
241		// flags to track which data files have been loaded:
242		_dispatchBotsLoaded: false,
243		_dispatchClientsLoaded: false,
244		_dispatchPlatformsLoaded: false,
245		_dispatchIPRangesLoaded: false,
246		_dispatchRulesLoaded: false,
247
248		// event callback, after the server log has been loaded:
249		_onServerLogLoaded: function() {
250			//console.info('BotMon.live.data._onServerLogLoaded()');
251
252			// chain the client log file to load:
253			BotMon.live.data.loadLogFile(BM_LOGTYPE.CLIENT, BotMon.live.data._onClientLogLoaded);
254		},
255
256		// event callback, after the client log has been loaded:
257		_onClientLogLoaded: function() {
258			//console.info('BotMon.live.data._onClientLogLoaded()');
259
260			// chain the ticks file to load:
261			BotMon.live.data.loadLogFile(BM_LOGTYPE.TICKER, BotMon.live.data._onTicksLogLoaded);
262
263		},
264
265		// event callback, after the tiker log has been loaded:
266		_onTicksLogLoaded: function() {
267			//console.info('BotMon.live.data._onTicksLogLoaded()');
268
269			// analyse the data:
270			BotMon.live.data.analytics.analyseAll();
271
272			// sort the data:
273			// #TODO
274
275			// display the data:
276			BotMon.live.gui.overview.make();
277
278			//console.log(BotMon.live.data.model._visitors);
279
280		},
281
282		// the data model:
283		model: {
284			// visitors storage:
285			_visitors: [],
286
287			// find an already existing visitor record:
288			findVisitor: function(visitor, type) {
289				//console.info('BotMon.live.data.model.findVisitor()', type);
290				//console.log(visitor);
291
292				// shortcut to make code more readable:
293				const model = BotMon.live.data.model;
294
295				// combine Bot networks to one visitor?
296				const combineNets = (BMSettings.hasOwnProperty('combineNets') ? BMSettings['combineNets'] : true);;
297
298				if (visitor._type == BM_USERTYPE.KNOWN_BOT) { // known bots match by their bot ID:
299
300					for (let i=0; i<model._visitors.length; i++) {
301						const v = model._visitors[i];
302
303						// bots match when their ID matches:
304						if (v._bot && v._bot.id == visitor._bot.id) {
305							return v;
306						}
307					}
308
309				} else if (combineNets && visitor.hasOwnProperty('_ipRange')) { // combine with other visits from the same range
310
311					let nonRangeVisitor = null;
312
313					for (let i=0; i<model._visitors.length; i++) {
314						const v = model._visitors[i];
315
316						if ( v.hasOwnProperty('_ipRange') && v._ipRange.g == visitor._ipRange.g ) { // match the IPRange Group IDs
317							return v;
318						} else if ( v.id.trim() !== '' && v.id == visitor.id) { // match the DW/PHP IDs
319							nonRangeVisitor = v;
320						}
321					}
322
323					// if no ip range was found, return the non-range visitor instead
324					if (nonRangeVisitor) return nonRangeVisitor;
325
326				} else { // other types match by their DW/PHPIDs:
327
328					// loop over all visitors already registered and check for ID matches:
329					for (let i=0; i<model._visitors.length; i++) {
330						const v = model._visitors[i];
331
332						if ( v.id.trim() !== '' && v.id == visitor.id) { // match the DW/PHP IDs
333							return v;
334						}
335					}
336
337					// if not found, try to match IP address and user agent:
338					for (let i=0; i<model._visitors.length; i++) {
339						const v = model._visitors[i];
340						if (  v.ip == visitor.ip && v.agent == visitor.agent) {
341							return v;
342						}
343					}
344				}
345
346				return null; // nothing found
347			},
348
349			/* if there is already this visit registered, return the page view item */
350			_getPageView: function(visit, view) {
351
352				// shortcut to make code more readable:
353				const model = BotMon.live.data.model;
354
355				for (let i=0; i<visit._pageViews.length; i++) {
356					const pv = visit._pageViews[i];
357					if (pv.pg == view.pg) {
358						return pv;
359					}
360				}
361				return null; // not found
362			},
363
364			// register a new visitor (or update if already exists)
365			registerVisit: function(nv, type) {
366				//console.info('registerVisit', nv, type);
367
368
369				// shortcut to make code more readable:
370				const model = BotMon.live.data.model;
371
372				// is it a known bot?
373				const bot = BotMon.live.data.bots.match(nv.agent);
374
375				// enrich new visitor with relevant data:
376				if (!nv._bot) nv._bot = bot ?? null; // bot info
377				nv._type = ( bot ? BM_USERTYPE.KNOWN_BOT : ( nv.usr && nv.usr !== '' ? BM_USERTYPE.KNOWN_USER : BM_USERTYPE.UNKNOWN ) ); // user type
378				if (bot && bot.geo) {
379					if (!nv.geo || nv.geo == '' || nv.geo == 'ZZ') nv.geo = bot.geo;
380				} else if (!nv.geo ||nv.geo == '') {
381					nv.geo = 'ZZ';
382				}
383
384				// update first and last seen:
385				if (!nv._firstSeen) nv._firstSeen = nv.ts; // first-seen
386				nv._lastSeen = nv.ts; // last-seen
387
388				// known bot IP range?
389				if (nv._type == BM_USERTYPE.UNKNOWN) { // only for unknown visitors
390					const ipInfo = BotMon.live.data.ipRanges.match(nv.ip);
391					if (ipInfo) nv._ipRange = ipInfo;
392				}
393
394				// country name:
395				try {
396					nv._country = ( nv.geo == 'local' ? "localhost" : "Unknown" );
397					if (nv.geo && nv.geo !== '' && nv.geo !== 'ZZ' && nv.geo !== 'local') {
398						const countryName = new Intl.DisplayNames(['en', BotMon._lang], {type: 'region'});
399						nv._country = countryName.of(nv.geo.substring(0,2)) ?? nv.geo;
400					}
401				} catch (err) {
402					console.error(err);
403					nv._country = 'Error';
404				}
405
406				// check if it already exists:
407				let visitor = model.findVisitor(nv, type);
408				if (!visitor) {
409					visitor = {...nv, ...{
410						_seenBy: [type],
411						_viewCount: 0, // number of page views
412						_loadCount: 0, // number of page loads (not necessarily views!)
413						_pageViews: [], // array of page views
414						_hasReferrer: false, // has at least one referrer
415						_jsClient: false, // visitor has been seen logged by client js as well
416						_client: BotMon.live.data.clients.match(nv.agent) ?? null, // client info
417						_platform: BotMon.live.data.platforms.match(nv.agent), // platform info
418						_captcha: {'X': 0, 'Y': 0, 'N': 0, 'W':0, 'H': 0,
419							_str: function() { return (this.X > 0 ? 'X' : '') + (this.Y > 0 ? 'Y' : '') + (this.N > 0 ? 'N' : '') + (this.W > 0 ? 'W' : '') + (this.H > 0 ? 'H' : ''); }
420						} // captcha counter
421					}};
422					model._visitors.push(visitor);
423				};
424
425				// update first and last seen:
426				if (visitor._firstSeen > nv.ts) {
427					visitor._firstSeen = nv.ts;
428				}
429				if (visitor._lastSeen < nv.ts) {
430					visitor._lastSeen = nv.ts;
431				}
432
433				// update total loads and views (not the same!):
434				visitor._loadCount += 1;
435				visitor._viewCount += (nv.captcha == 'Y' ? 0 : 1);
436
437				// ...because also a captcha is a "load", but not a "view".
438				// let's count the captcha statuses as well:
439				if (nv.captcha) visitor._captcha[nv.captcha] += 1;
440
441				// is this visit already registered?
442				let prereg = model._getPageView(visitor, nv);
443				if (!prereg) {
444					// add new page view:
445					prereg = model._makePageView(nv, type);
446					visitor._pageViews.push(prereg);
447				}
448
449				// update last seen date
450				prereg._lastSeen = nv.ts;
451
452				// increase view count:
453				prereg._loadCount += (visitor.captcha == 'Y' ? 0 : 1);
454				//prereg._tickCount += 1;
455
456				// update referrer state:
457				visitor._hasReferrer = visitor._hasReferrer ||
458					(prereg.ref !== undefined && prereg.ref !== '');
459
460				// update time stamp for last-seen:
461				if (visitor._lastSeen < nv.ts) {
462					visitor._lastSeen = nv.ts;
463				}
464
465				// if needed:
466				return visitor;
467			},
468
469			// updating visit data from the client-side log:
470			updateVisit: function(dat) {
471				//console.info('updateVisit', dat);
472
473				// shortcut to make code more readable:
474				const model = BotMon.live.data.model;
475
476				const type = BM_LOGTYPE.CLIENT;
477
478				let visitor = BotMon.live.data.model.findVisitor(dat, type);
479				if (!visitor) {
480					visitor = model.registerVisit(dat, type);
481				}
482				if (visitor) {
483
484					if (visitor._lastSeen < dat.ts) {
485						visitor._lastSeen = dat.ts;
486					}
487					if (!visitor._seenBy.includes(type)) {
488						visitor._seenBy.push(type);
489					}
490					visitor._jsClient = true; // seen by client js
491				}
492
493				// find the page view:
494				let prereg = BotMon.live.data.model._getPageView(visitor, dat);
495				if (prereg) {
496					// update the page view:
497					prereg._lastSeen = dat.ts;
498					if (!prereg._seenBy.includes(type)) prereg._seenBy.push(type);
499					prereg._jsClient = true; // seen by client js
500				} else {
501					// add the page view to the visitor:
502					prereg = model._makePageView(dat, type);
503					visitor._pageViews.push(prereg);
504				}
505				prereg._tickCount += 1;
506			},
507
508			// updating visit data from the ticker log:
509			updateTicks: function(dat) {
510				//console.info('updateTicks', dat);
511
512				// shortcut to make code more readable:
513				const model = BotMon.live.data.model;
514
515				const type = BM_LOGTYPE.TICKER;
516
517				// find the visit info:
518				let visitor = model.findVisitor(dat, type);
519				if (!visitor) {
520					console.info(`No visitor with ID “${dat.id}” found, registering as a new one.`);
521					visitor = model.registerVisit(dat, type);
522				}
523				if (visitor) {
524					// update visitor:
525					if (visitor._lastSeen < dat.ts) visitor._lastSeen = dat.ts;
526					if (!visitor._seenBy.includes(type)) visitor._seenBy.push(type);
527
528					// get the page view info:
529					let pv = model._getPageView(visitor, dat);
530					if (!pv) {
531						console.info(`No page view for visit ID “${dat.id}”, page “${dat.pg}”, registering a new one.`);
532						pv = model._makePageView(dat, type);
533						visitor._pageViews.push(pv);
534					}
535
536					// update the page view info:
537					if (!pv._seenBy.includes(type)) pv._seenBy.push(type);
538					if (pv._lastSeen.getTime() < dat.ts.getTime()) pv._lastSeen = dat.ts;
539					pv._tickCount += 1;
540
541				}
542			},
543
544			// helper function to create a new "page view" item:
545			_makePageView: function(data, type) {
546				// console.info('_makePageView', data);
547
548				// try to parse the referrer:
549				let rUrl = null;
550				try {
551					rUrl = ( data.ref && data.ref !== '' ? new URL(data.ref) : null );
552				} catch (e) {
553					console.warn(`Invalid referer: “${data.ref}”.`);
554					console.info(data);
555				}
556
557				return {
558					_by: type,
559					ip: data.ip,
560					pg: data.pg,
561					lang: data.lang || '??',
562					_ref: rUrl,
563					_firstSeen: data.ts,
564					_lastSeen: data.ts,
565					_seenBy: [type],
566					_jsClient: ( type !== BM_LOGTYPE.SERVER),
567					_viewCount: 0,
568					_loadCount: 0,
569					_tickCount: 0
570				};
571			},
572
573			// helper function to make a human-readable title from the Captcha statuses:
574			_makeCaptchaTitle: function(cObj) {
575				const cStr = cObj._str();
576				switch (cStr) {
577					case 'Y':
578					case 'NY': return "Blocked.";
579					case 'YN': return "Solved";
580					case 'W': return "Whitelisted";
581					case 'H': return "HEAD request, no captcha";
582					default: return "Undefined: " + cStr;
583				}
584			}
585		},
586
587		// functions to analyse the data:
588		analytics: {
589
590			/**
591			 * Initializes the analytics data storage object:
592			 */
593			init: function() {
594				//console.info('BotMon.live.data.analytics.init()');
595			},
596
597			// data storage:
598			data: {
599				visits: {
600					bots: 0,
601					suspected: 0,
602					humans: 0,
603					users: 0,
604					total: 0
605				},
606				views: {
607					bots: 0,
608					suspected: 0,
609					humans: 0,
610					users: 0,
611					total: 0
612				},
613				loads: {
614					bots: 0,
615					suspected: 0,
616					humans: 0,
617					users: 0,
618					total: 0
619				},
620				captcha: {
621					bots_blocked: 0,
622					bots_passed: 0,
623					bots_whitelisted: 0,
624					humans_blocked: 0,
625					humans_passed: 0,
626					sus_blocked: 0,
627					sus_passed: 0,
628					sus_whitelisted: 0
629				}
630			},
631
632			// sort the visits by type:
633			groups: {
634				knownBots: [],
635				suspectedBots: [],
636				humans: [],
637				users: []
638			},
639
640			// all analytics
641			analyseAll: function() {
642				//console.info('BotMon.live.data.analytics.analyseAll()');
643
644				// shortcut to make code more readable:
645				const model = BotMon.live.data.model;
646				const data = BotMon.live.data.analytics.data;
647				const me = BotMon.live.data.analytics;
648
649				BotMon.live.gui.status.showBusy("Analysing data …");
650
651				// loop over all visitors:
652				model._visitors.forEach( (v) => {
653
654					const captchaStr = v._captcha._str();
655
656					// count total visits and page views:
657					data.visits.total += 1;
658					data.loads.total += v._loadCount;
659					data.views.total += v._viewCount;
660
661					// check for typical bot aspects:
662					let botScore = 0;
663
664					if (v._type == BM_USERTYPE.KNOWN_BOT) { // known bots
665
666						data.visits.bots += 1;
667						data.views.bots += v._viewCount;
668						this.groups.knownBots.push(v);
669
670						// captcha counter
671						if (captchaStr == 'Y') {
672							data.captcha.bots_blocked += 1;
673						} else if (captchaStr == 'YN') {
674							data.captcha.bots_passed += 1;
675						} else if (captchaStr == 'W') {
676							data.captcha.bots_whitelisted += 1;
677						}
678
679					} else if (v._type == BM_USERTYPE.KNOWN_USER) { // known users */
680
681						data.visits.users += 1;
682						data.views.users += v._viewCount;
683						this.groups.users.push(v);
684
685					} else {
686
687						// get evaluation:
688						const e = BotMon.live.data.rules.evaluate(v);
689						v._eval = e.rules;
690						v._botVal = e.val;
691
692						if (e.isBot) { // likely bots
693
694							v._type = BM_USERTYPE.LIKELY_BOT;
695							data.visits.suspected += 1;
696							data.views.suspected += v._viewCount;
697							this.groups.suspectedBots.push(v);
698
699							// captcha counter
700							if (captchaStr == 'Y') {
701								data.captcha.sus_blocked += 1;
702							} else if (captchaStr == 'YN') {
703								data.captcha.sus_passed += 1;
704							} else if (captchaStr == 'W') {
705								data.captcha.sus_whitelisted += 1;
706							}
707
708						} else { // probably humans
709
710							v._type = BM_USERTYPE.PROBABLY_HUMAN;
711							data.visits.humans += 1;
712							data.views.humans += v._viewCount;
713
714							this.groups.humans.push(v);
715
716							// captcha counter
717							if (captchaStr == 'Y') {
718								data.captcha.humans_blocked += 1;
719							} else if (captchaStr == 'YN') {
720								data.captcha.humans_passed += 1;
721							}
722						}
723					}
724
725					// perform actions depending on the visitor type:
726					if (v._type == BM_USERTYPE.KNOWN_BOT ) { /* known bots only */
727
728						// no specific actions here.
729
730					} else if (v._type == BM_USERTYPE.LIKELY_BOT) { /* probable bots only */
731
732						// add bot views to IP range information:
733						me.addToIpRanges(v);
734
735					} else { /* registered users and probable humans */
736
737						// add browser and platform statistics:
738						me.addBrowserPlatform(v);
739
740						// add to referrer and pages lists:
741						v._pageViews.forEach( pv => {
742							me.addToRefererList(pv._ref);
743							me.addToPagesList(pv.pg);
744						});
745					}
746
747					// add to the country lists:
748					me.addToCountries(v.geo, v._country, v._type);
749
750				});
751
752				BotMon.live.gui.status.hideBusy('Done.');
753			},
754
755			// get a list of known bots:
756			getTopBots: function(max) {
757				//console.info('BotMon.live.data.analytics.getTopBots('+max+')');
758
759				//console.log(BotMon.live.data.analytics.groups.knownBots);
760
761				let botsList = BotMon.live.data.analytics.groups.knownBots.toSorted( (a, b) => {
762					return b._viewCount - a._viewCount;
763				});
764
765				const other = {
766					'id': 'other',
767					'name': "Others",
768					'count': 0
769				};
770
771				const rList = [];
772				const max2 = ( botsList.length > max ? max-1 : botsList.length );
773				let total = 0; // adding up the items
774				for (let i=0; i<botsList.length; i++) {
775					const it = botsList[i];
776					if (it && it._bot) {
777						if (i < max2) {
778							rList.push({
779								id: it._bot.id,
780								name: (it._bot.n ? it._bot.n : it._bot.id),
781								count: it._viewCount
782							});
783						} else {
784							other.count += it._pageViews.length;
785						};
786						total += it._viewCount;
787					}
788				};
789
790				// add the "other" item, if needed:
791				if (botsList.length > max2) {
792					rList.push(other);
793				};
794
795				rList.forEach( it => {
796					it.pct = (it.count * 100 / total);
797				});
798
799				return rList;
800			},
801
802			// most visited pages list:
803			_pagesList: [],
804
805			/**
806			 * Add a page view to the list of most visited pages.
807			 * @param {string} pageId - The page ID to add to the list.
808			 * @example
809			 * BotMon.live.data.analytics.addToPagesList('1234567890');
810			 */
811			addToPagesList: function(pageId) {
812				//console.log('BotMon.live.data.analytics.addToPagesList', pageId);
813
814				const me = BotMon.live.data.analytics;
815
816				// already exists?
817				let pgObj = null;
818				for (let i = 0; i < me._pagesList.length; i++) {
819					if (me._pagesList[i].id == pageId) {
820						pgObj = me._pagesList[i];
821						break;
822					}
823				}
824
825				// if not exists, create it:
826				if (!pgObj) {
827					pgObj = {
828						id: pageId,
829						count: 1
830					};
831					me._pagesList.push(pgObj);
832				} else {
833					pgObj.count += 1;
834				}
835			},
836
837			getTopPages: function(max) {
838				//console.info('BotMon.live.data.analytics.getTopPages('+max+')');
839				const me = BotMon.live.data.analytics;
840				return me._pagesList.toSorted( (a, b) => {
841					return b.count - a.count;
842				}).slice(0,max);
843			},
844
845			// Referer List:
846			_refererList: [],
847
848			addToRefererList: function(ref) {
849				//console.log('BotMon.live.data.analytics.addToRefererList',ref);
850
851				const me = BotMon.live.data.analytics;
852
853				// ignore internal references:
854				if (ref && ref.host == window.location.host) {
855					return;
856				}
857
858				const refInfo = me.getRefererInfo(ref);
859
860				// already exists?
861				let refObj = null;
862				for (let i = 0; i < me._refererList.length; i++) {
863					if (me._refererList[i].id == refInfo.id) {
864						refObj = me._refererList[i];
865						break;
866					}
867				}
868
869				// if not exists, create it:
870				if (!refObj) {
871					refObj = refInfo;
872					refObj.count = 1;
873					me._refererList.push(refObj);
874				} else {
875					refObj.count += 1;
876				}
877			},
878
879			getRefererInfo: function(url) {
880				//console.log('BotMon.live.data.analytics.getRefererInfo',url);
881				try {
882					url = new URL(url);
883				} catch (e) {
884					return {
885						'id': 'null',
886						'n': 'Invalid Referer'
887					};
888				}
889
890				// find the referer ID:
891				let refId = 'null';
892				let refName = 'No Referer';
893				if (url && url.host) {
894					const hArr = url.host.split('.');
895					const tld = hArr[hArr.length-1];
896					refId = ( tld == 'localhost' ? tld : hArr[hArr.length-2]);
897					refName = hArr[hArr.length-2] + '.' + tld;
898				}
899
900				return {
901					'id': refId,
902					'n': refName
903				};
904			},
905
906			/**
907			 * Get a sorted list of the top referers.
908			 * The list is sorted in descending order of count.
909			 * If the array has more items than the given maximum, the rest of the items are added to an "other" item.
910			 * Each item in the list has a "pct" property, which is the percentage of the total count.
911			 * @param {number} max - The maximum number of items to return.
912			 * @return {Array} The sorted list of top referers.
913			 */
914			getTopReferers: function(max) {
915				//console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')'));
916
917				const me = BotMon.live.data.analytics;
918
919				return me._makeTopList(me._refererList, max);
920			},
921
922			/**
923			 * Create a sorted list of top items from a given array.
924			 * The list is sorted in descending order of count.
925			 * If the array has more items than the given maximum, the rest of the items are added to an "other" item.
926			 * Each item in the list has a "pct" property, which is the percentage of the total count.
927			 * @param {Array} arr - The array to sort and truncate.
928			 * @param {number} max - The maximum number of items to return.
929			 * @return {Array} The sorted list of top items.
930			 */
931			_makeTopList: function(arr, max) {
932				//console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')'));
933
934				const me = BotMon.live.data.analytics;
935
936				// sort the list:
937				arr.sort( (a,b) => {
938					return b.count - a.count;
939				});
940
941				const rList = []; // return array
942				const max2 = ( arr.length >= max ? max-1 : arr.length );
943				const other = {
944					'id': 'other',
945					'name': "Others",
946					'typ': 'other',
947					'count': 0
948				};
949				let total = 0; // adding up the items
950				for (let i=0; Math.min(max, arr.length) > i; i++) {
951					const it = arr[i];
952					if (it) {
953						if (i < max2) {
954							const rIt = {
955								id: it.id,
956								name: (it.n ? it.n : it.id),
957								typ: it.typ || it.id,
958								count: it.count
959							};
960							rList.push(rIt);
961						} else {
962							other.count += it.count;
963						}
964						total += it.count;
965					}
966				}
967
968				// add the "other" item, if needed:
969				if (arr.length > max2) {
970					rList.push(other);
971				};
972
973				rList.forEach( it => {
974					it.pct = (it.count * 100 / total);
975				});
976
977				return rList;
978			},
979
980			/* countries of visits */
981			_countries: {
982				'human': [],
983				'bot': []
984			},
985			/**
986			 * Adds a country code to the statistics.
987			 *
988			 * @param {string} iso The ISO 3166-1 alpha-2 country code.
989			 */
990			addToCountries: function(iso, name, type) {
991
992				const me = BotMon.live.data.analytics;
993
994				// find the correct array:
995				let arr = null;
996				switch (type) {
997
998					case BM_USERTYPE.KNOWN_USER:
999					case BM_USERTYPE.PROBABLY_HUMAN:
1000						arr = me._countries.human;
1001						break;
1002					case BM_USERTYPE.LIKELY_BOT:
1003					case BM_USERTYPE.KNOWN_BOT:
1004						arr = me._countries.bot;
1005						break;
1006					default:
1007						console.warn(`Unknown user type ${type} in function addToCountries.`);
1008				}
1009
1010				if (arr) {
1011					let cRec = arr.find( it => it.id == iso);
1012					if (!cRec) {
1013						cRec = {
1014							'id': iso,
1015							'n': name,
1016							'count': 1
1017						};
1018						arr.push(cRec);
1019					} else {
1020						cRec.count += 1;
1021					}
1022				}
1023			},
1024
1025			/**
1026			 * Returns a list of countries with visit counts, sorted by visit count in descending order.
1027			 *
1028			 * @param {BM_USERTYPE} type array of types type of visitors to return.
1029			 * @param {number} max The maximum number of entries to return.
1030			 * @return {Array} A list of objects with properties 'iso' (ISO 3166-1 alpha-2 country code) and 'count' (visit count).
1031			 */
1032			getCountryList: function(type, max) {
1033
1034				const me = BotMon.live.data.analytics;
1035
1036				// find the correct array:
1037				let arr = null;
1038				switch (type) {
1039
1040					case 'human':
1041						arr = me._countries.human;
1042						break;
1043					case 'bot':
1044						arr = me._countries.bot;
1045						break;
1046					default:
1047						console.warn(`Unknown user type ${type} in function getCountryList.`);
1048						return;
1049				}
1050
1051				return me._makeTopList(arr, max);
1052			},
1053
1054			/* browser and platform of human visitors */
1055			_browsers: [],
1056			_platforms: [],
1057
1058			addBrowserPlatform: function(visitor) {
1059				//console.info('addBrowserPlatform', visitor);
1060
1061				const me = BotMon.live.data.analytics;
1062
1063				// add to browsers list:
1064				let browserRec = ( visitor._client ? visitor._client : {'id': 'unknown'});
1065				if (visitor._client) {
1066					let bRec = me._browsers.find( it => it.id == browserRec.id);
1067					if (!bRec) {
1068						bRec = {
1069							id: browserRec.id,
1070							n: browserRec.n,
1071							count: 1
1072						};
1073						me._browsers.push(bRec);
1074					} else {
1075						bRec.count += 1;
1076					}
1077				}
1078
1079				// add to platforms list:
1080				let platformRec = ( visitor._platform ? visitor._platform : {'id': 'unknown'});
1081				if (visitor._platform) {
1082					let pRec = me._platforms.find( it => it.id == platformRec.id);
1083					if (!pRec) {
1084						pRec = {
1085							id: platformRec.id,
1086							n: platformRec.n,
1087							count: 1
1088						};
1089						me._platforms.push(pRec);
1090					} else {
1091						pRec.count += 1;
1092					}
1093				}
1094
1095			},
1096
1097			getTopBrowsers: function(max) {
1098
1099				const me = BotMon.live.data.analytics;
1100
1101				return me._makeTopList(me._browsers, max);
1102			},
1103
1104			getTopPlatforms: function(max) {
1105
1106				const me = BotMon.live.data.analytics;
1107
1108				return me._makeTopList(me._platforms, max);
1109			},
1110
1111			/* bounces are counted, not calculates: */
1112			getBounceCount: function(type) {
1113
1114				const me = BotMon.live.data.analytics;
1115				var bounces = 0;
1116				const list = me.groups[type];
1117
1118				list.forEach(it => {
1119					bounces += (it._viewCount <= 1 ? 1 : 0);
1120				});
1121
1122				return bounces;
1123			},
1124
1125			_ipRanges: [],
1126
1127			/* adds a visit to the ip ranges arrays */
1128			addToIpRanges: function(v) {
1129				//console.info('addToIpRanges', v.ip);
1130
1131				const me = BotMon.live.data.analytics;
1132				const ipRanges = BotMon.live.data.ipRanges;
1133
1134				// Number of IP address segments to look at:
1135				const kIP4Segments = 1;
1136				const kIP6Segments = 3;
1137
1138				let ipGroup = ''; // group name
1139				let ipSeg = []; // IP segment array
1140				let rawIP = ''; // raw ip prefix
1141				let ipName = ''; // IP group display name
1142
1143				const ipType = v.ip.indexOf(':') > 0 ? BM_IPVERSION.IPv6 : BM_IPVERSION.IPv4;
1144				const ipAddr = BotMon.t._ip2Num(v.ip);
1145
1146				// is there already a known IP range assigned?
1147				if (v._ipRange) {
1148
1149					ipGroup = v._ipRange.g; // group name
1150					ipName = ipRanges.getOwner( v._ipRange.g ) || "Unknown";
1151
1152				} else { // no known IP range, let's collect necessary information:
1153
1154					// collect basic IP address info:
1155					if (ipType == BM_IPVERSION.IPv6) {
1156						ipSeg = ipAddr.split(':');
1157						const prefix = v.ip.split(':').slice(0, kIP6Segments).join(':');
1158						rawIP = ipSeg.slice(0, kIP6Segments).join(':');
1159						ipGroup = 'ip6-' + rawIP.replaceAll(':', '-');
1160						ipName = prefix + '::'; // + '/' + (16 * kIP6Segments);
1161					} else {
1162						ipSeg = ipAddr.split('.');
1163						const prefix = v.ip.split('.').slice(0, kIP4Segments).join('.');
1164						rawIP = ipSeg.slice(0, kIP4Segments).join('.') ;
1165						ipGroup = 'ip4-' + rawIP.replaceAll('.', '-');
1166						ipName = prefix + '.x.x.x'.substring(0, 1+(4-kIP4Segments)*2); // + '/' + (8 * kIP4Segments);
1167					}
1168				}
1169
1170				// check if record already exists:
1171				let ipRec = me._ipRanges.find( it => it.g == ipGroup);
1172				if (!ipRec) {
1173
1174					// ip info record initialised:
1175					ipRec = {
1176						g: ipGroup,
1177						n: ipName,
1178						count: 0
1179					}
1180
1181					// existing record?
1182					if (v._ipRange) {
1183
1184						ipRec.from = v._ipRange.from;
1185						ipRec.to = v._ipRange.to;
1186						ipRec.typ = 'net';
1187
1188					} else { // no known IP range, let's collect necessary information:
1189
1190						// complete the ip info record:
1191						if (ipType == BM_IPVERSION.IPv6) {
1192							ipRec.from = rawIP + ':0000:0000:0000:0000:0000:0000:0000'.substring(0, (8-kIP6Segments)*5);
1193							ipRec.to = rawIP + ':FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF'.substring(0, (8-kIP6Segments)*5);
1194							ipRec.typ = '6';
1195						} else {
1196							ipRec.from = rawIP + '.000.000.000.000'.substring(0, (4-kIP4Segments)*4);
1197							ipRec.to = rawIP + '.255.255.255.254'.substring(0, (4-kIP4Segments)*4);
1198							ipRec.typ = '4';
1199						}
1200					}
1201
1202					me._ipRanges.push(ipRec);
1203				}
1204
1205				// add to counter:
1206				ipRec.count += v._viewCount;
1207
1208			},
1209
1210			getTopBotISPs: function(max) {
1211
1212				const me = BotMon.live.data.analytics;
1213
1214				return me._makeTopList(me._ipRanges, max);
1215			},
1216		},
1217
1218		// information on "known bots":
1219		bots: {
1220			// loads the list of known bots from a JSON file:
1221			init: async function() {
1222				//console.info('BotMon.live.data.bots.init()');
1223
1224				// Load the list of known bots:
1225				BotMon.live.gui.status.showBusy("Loading known bots …");
1226				const url = BotMon._baseDir + 'config/known-bots.json';
1227				try {
1228					const response = await fetch(url);
1229					if (!response.ok) {
1230						throw new Error(`${response.status} ${response.statusText}`);
1231					}
1232
1233					this._list = await response.json();
1234					this._ready = true;
1235
1236				} catch (error) {
1237					BotMon.live.gui.status.setError("Error while loading the known bots file:", error.message);
1238				} finally {
1239					BotMon.live.gui.status.hideBusy("Status: Done.");
1240					BotMon.live.data._dispatch('bots')
1241				}
1242			},
1243
1244			// returns bot info if the clientId matches a known bot, null otherwise:
1245			match: function(agent) {
1246				//console.info('BotMon.live.data.bots.match(',agent,')');
1247
1248				const BotList = BotMon.live.data.bots._list;
1249
1250				// default is: not found!
1251				let botInfo = null;
1252
1253				if (!agent) return null;
1254
1255				// check for known bots:
1256				BotList.find(bot => {
1257					let r = false;
1258					for (let j=0; j<bot.rx.length; j++) {
1259						const rxr = agent.match(new RegExp(bot.rx[j]));
1260						if (rxr) {
1261							botInfo = {
1262								n : bot.n,
1263								id: bot.id,
1264								geo: (bot.geo ? bot.geo : null),
1265								url: bot.url,
1266								v: (rxr.length > 1 ? rxr[1] : -1)
1267							};
1268							r = true;
1269							break;
1270						};
1271					};
1272					return r;
1273				});
1274
1275				// check for unknown bots:
1276				if (!botInfo) {
1277					const botmatch = agent.match(/([\s\d\w\-]*bot|[\s\d\w\-]*crawler|[\s\d\w\-]*spider)[\/\s\w\-;\),\\.$]/i);
1278					if(botmatch) {
1279						botInfo = {'id': ( botmatch[1] || "other_" ), 'n': "Other" + ( botmatch[1] ? " (" + botmatch[1] + ")" : "" ) , "bot": botmatch[1] };
1280					}
1281				}
1282
1283				//console.log("botInfo:", botInfo);
1284				return botInfo;
1285			},
1286
1287
1288			// indicates if the list is loaded and ready to use:
1289			_ready: false,
1290
1291			// the actual bot list is stored here:
1292			_list: []
1293		},
1294
1295		// information on known clients (browsers):
1296		clients: {
1297			// loads the list of known clients from a JSON file:
1298			init: async function() {
1299				//console.info('BotMon.live.data.clients.init()');
1300
1301				// Load the list of known bots:
1302				BotMon.live.gui.status.showBusy("Loading known clients");
1303				const url = BotMon._baseDir + 'config/known-clients.json';
1304				try {
1305					const response = await fetch(url);
1306					if (!response.ok) {
1307						throw new Error(`${response.status} ${response.statusText}`);
1308					}
1309
1310					BotMon.live.data.clients._list = await response.json();
1311					BotMon.live.data.clients._ready = true;
1312
1313				} catch (error) {
1314					BotMon.live.gui.status.setError("Error while loading the known clients file: " + error.message);
1315				} finally {
1316					BotMon.live.gui.status.hideBusy("Status: Done.");
1317					BotMon.live.data._dispatch('clients')
1318				}
1319			},
1320
1321			// returns bot info if the user-agent matches a known bot, null otherwise:
1322			match: function(agent) {
1323				//console.info('BotMon.live.data.clients.match(',agent,')');
1324
1325				let match = {"n": "Unknown", "v": -1, "id": 'null'};
1326
1327				if (agent) {
1328					BotMon.live.data.clients._list.find(client => {
1329						let r = false;
1330						for (let j=0; j<client.rx.length; j++) {
1331							const rxr = agent.match(new RegExp(client.rx[j]));
1332							if (rxr) {
1333								match.n = client.n;
1334								match.v = (rxr.length > 1 ? rxr[1] : -1);
1335								match.id = client.id || null;
1336								r = true;
1337								break;
1338							}
1339						}
1340						return r;
1341					});
1342				}
1343
1344				//console.log(match)
1345				return match;
1346			},
1347
1348			// return the browser name for a browser ID:
1349			getName: function(id) {
1350				const it = BotMon.live.data.clients._list.find(client => client.id == id);
1351				return ( it && it.n ? it.n : "Unknown"); //it.n;
1352			},
1353
1354			// indicates if the list is loaded and ready to use:
1355			_ready: false,
1356
1357			// the actual bot list is stored here:
1358			_list: []
1359
1360		},
1361
1362		// information on known platforms (operating systems):
1363		platforms: {
1364			// loads the list of known platforms from a JSON file:
1365			init: async function() {
1366				//console.info('BotMon.live.data.platforms.init()');
1367
1368				// Load the list of known bots:
1369				BotMon.live.gui.status.showBusy("Loading known platforms");
1370				const url = BotMon._baseDir + 'config/known-platforms.json';
1371				try {
1372					const response = await fetch(url);
1373					if (!response.ok) {
1374						throw new Error(`${response.status} ${response.statusText}`);
1375					}
1376
1377					BotMon.live.data.platforms._list = await response.json();
1378					BotMon.live.data.platforms._ready = true;
1379
1380				} catch (error) {
1381					BotMon.live.gui.status.setError("Error while loading the known platforms file: " + error.message);
1382				} finally {
1383					BotMon.live.gui.status.hideBusy("Status: Done.");
1384					BotMon.live.data._dispatch('platforms')
1385				}
1386			},
1387
1388			// returns bot info if the browser id matches a known platform:
1389			match: function(cid) {
1390				//console.info('BotMon.live.data.platforms.match(',cid,')');
1391
1392				let match = {"n": "Unknown", "id": 'null'};
1393
1394				if (cid) {
1395					BotMon.live.data.platforms._list.find(platform => {
1396						let r = false;
1397						for (let j=0; j<platform.rx.length; j++) {
1398							const rxr = cid.match(new RegExp(platform.rx[j]));
1399							if (rxr) {
1400								match.n = platform.n;
1401								match.v = (rxr.length > 1 ? rxr[1] : -1);
1402								match.id = platform.id || null;
1403								r = true;
1404								break;
1405							}
1406						}
1407						return r;
1408					});
1409				}
1410
1411				return match;
1412			},
1413
1414			// return the platform name for a given ID:
1415			getName: function(id) {
1416				const it = BotMon.live.data.platforms._list.find( pf => pf.id == id);
1417				return ( it ? it.n : 'Unknown' );
1418			},
1419
1420
1421			// indicates if the list is loaded and ready to use:
1422			_ready: false,
1423
1424			// the actual bot list is stored here:
1425			_list: []
1426
1427		},
1428
1429		// storage and functions for the known bot IP-Ranges:
1430		ipRanges: {
1431
1432			init: function() {
1433				//console.log('BotMon.live.data.ipRanges.init()');
1434				// #TODO: Load from separate IP-Ranges file
1435				// load the rules file:
1436				const me = BotMon.live.data;
1437
1438				try {
1439					BotMon.live.data._loadSettingsFile(['user-ipranges', 'known-ipranges'],
1440						(json) => {
1441
1442						// groups can be just saved in the data structure:
1443						if (json.groups && json.groups.constructor.name == 'Array') {
1444							me.ipRanges._groups = json.groups;
1445						}
1446
1447						// groups can be just saved in the data structure:
1448						if (json.ranges && json.ranges.constructor.name == 'Array') {
1449							json.ranges.forEach(range => {
1450								me.ipRanges.add(range);
1451							})
1452						}
1453
1454						// finished loading
1455						BotMon.live.gui.status.hideBusy("Status: Done.");
1456						BotMon.live.data._dispatch('ipranges')
1457					});
1458				} catch (error) {
1459					BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
1460				}
1461			},
1462
1463			// the actual bot list is stored here:
1464			_list: [],
1465			_groups: [],
1466
1467			add: function(data) {
1468				//console.log('BotMon.live.data.ipRanges.add(',data,')');
1469
1470				const me = BotMon.live.data.ipRanges;
1471
1472				// convert IP address to easier comparable form:
1473				const ip2Num = BotMon.t._ip2Num;
1474
1475				let item = {
1476					'cidr': data.from.replaceAll(/::+/g, '::') + '/' + ( data.m ? data.m : '??' ),
1477					'from': ip2Num(data.from),
1478					'to': ip2Num(data.to),
1479					'm': data.m,
1480					'g': data.g
1481				};
1482				me._list.push(item);
1483
1484			},
1485
1486			getOwner: function(gid) {
1487
1488				const me = BotMon.live.data.ipRanges;
1489
1490				for (let i=0; i < me._groups.length; i++) {
1491					const it = me._groups[i];
1492					if (it.id == gid) {
1493						return it.name;
1494					}
1495				}
1496				return null;
1497			},
1498
1499			match: function(ip) {
1500				//console.log('BotMon.live.data.ipRanges.match(',ip,')');
1501
1502				const me = BotMon.live.data.ipRanges;
1503
1504				// convert IP address to easier comparable form:
1505				const ipNum = BotMon.t._ip2Num(ip);
1506
1507				for (let i=0; i < me._list.length; i++) {
1508					const ipRange = me._list[i];
1509
1510					if (ipNum >= ipRange.from && ipNum <= ipRange.to) {
1511						return ipRange;
1512					}
1513
1514				};
1515				return null;
1516			}
1517		},
1518
1519		// storage for the rules and related functions
1520		rules: {
1521
1522			/**
1523			 * Initializes the rules data.
1524			 *
1525			 * Loads the default config file and the user config file (if present).
1526			 * The default config file is used if the user config file does not have a certain setting.
1527			 * The user config file can override settings from the default config file.
1528			 *
1529			 * The rules are loaded from the `rules` property of the config files.
1530			 * The IP ranges are loaded from the `ipRanges` property of the config files.
1531			 *
1532			 * If an error occurs while loading the config file, it is displayed in the status bar.
1533			 * After the config file is loaded, the status bar is hidden.
1534			 */
1535			init: async function() {
1536				//console.info('BotMon.live.data.rules.init()');
1537
1538				// Load the list of known bots:
1539				BotMon.live.gui.status.showBusy("Loading list of rules …");
1540
1541				// load the rules file:
1542				const me = BotMon.live.data;
1543
1544				try {
1545					BotMon.live.data._loadSettingsFile(['user-config', 'default-config'],
1546						(json) => {
1547
1548							// override the threshold?
1549							if (json.threshold) me._threshold = json.threshold;
1550
1551							// set the rules list:
1552							if (json.rules && json.rules.constructor.name == 'Array') {
1553								me.rules._rulesList = json.rules;
1554							}
1555
1556							BotMon.live.gui.status.hideBusy("Status: Done.");
1557							BotMon.live.data._dispatch('rules')
1558						}
1559					);
1560				} catch (error) {
1561					BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
1562				}
1563			},
1564
1565			_rulesList: [], // list of rules to find out if a visitor is a bot
1566			_threshold: 100, // above this, it is considered a bot.
1567
1568			// returns a descriptive text for a rule id
1569			getRuleInfo: function(ruleId) {
1570				// console.info('getRuleInfo', ruleId);
1571
1572				// shortcut for neater code:
1573				const me = BotMon.live.data.rules;
1574
1575				for (let i=0; i<me._rulesList.length; i++) {
1576					const rule = me._rulesList[i];
1577					if (rule.id == ruleId) {
1578						return rule;
1579					}
1580				}
1581				return null;
1582
1583			},
1584
1585			// evaluate a visitor for lkikelihood of being a bot
1586			evaluate: function(visitor) {
1587
1588				// shortcut for neater code:
1589				const me = BotMon.live.data.rules;
1590
1591				let r =  {	// evaluation result
1592					'val': 0,
1593					'rules': [],
1594					'isBot': false
1595				};
1596
1597				for (let i=0; i<me._rulesList.length; i++) {
1598					const rule = me._rulesList[i];
1599					const params = ( rule.params ? rule.params : [] );
1600
1601					if (rule.func) { // rule is calling a function
1602						if (me.func[rule.func]) {
1603							if(me.func[rule.func](visitor, ...params)) {
1604								r.val += rule.bot;
1605								r.rules.push(rule.id)
1606							}
1607						} else {
1608							//console.warn("Unknown rule function: “${rule.func}”. Ignoring rule.")
1609						}
1610					}
1611				}
1612
1613				// is a bot?
1614				r.isBot = (r.val >= me._threshold);
1615
1616				return r;
1617			},
1618
1619			// list of functions that can be called by the rules list to evaluate a visitor:
1620			func: {
1621
1622				// check if client is on the list passed as parameter:
1623				matchesClient: function(visitor, ...clients) {
1624
1625					const clientId = ( visitor._client ? visitor._client.id : '');
1626					return clients.includes(clientId);
1627				},
1628
1629				// check if OS/Platform is one of the obsolete ones:
1630				matchesPlatform: function(visitor, ...platforms) {
1631
1632					const pId = ( visitor._platform ? visitor._platform.id : '');
1633
1634					if (visitor._platform.id == null) console.log(visitor._platform);
1635
1636					return platforms.includes(pId);
1637				},
1638
1639				// are there at lest num pages loaded?
1640				smallPageCount: function(visitor, num) {
1641					return (visitor._viewCount <= Number(num));
1642				},
1643
1644				// There was no entry in a specific log file for this visitor:
1645				// note that this will also trigger the "noJavaScript" rule:
1646				noRecord: function(visitor, type) {
1647					if (!visitor._seenBy.includes('srv')) return false; // only if 'srv' is also specified!
1648					return !visitor._seenBy.includes(type);
1649				},
1650
1651				// there are no referrers in any of the page visits:
1652				noReferrer: function(visitor) {
1653
1654					let r = false; // return value
1655					for (let i = 0; i < visitor._pageViews.length; i++) {
1656						if (!visitor._pageViews[i]._ref) {
1657							r = true;
1658							break;
1659						}
1660					}
1661					return r;
1662				},
1663
1664				// test for specific client identifiers:
1665				matchesUserAgent: function(visitor, ...list) {
1666
1667					for (let i=0; i<list.length; i++) {
1668						if (visitor.agent == list[i]) {
1669							return true
1670						}
1671					};
1672					return false;
1673				},
1674
1675				// unusual combinations of Platform and Client:
1676				combinationTest: function(visitor, ...combinations) {
1677
1678					for (let i=0; i<combinations.length; i++) {
1679
1680						if (visitor._platform.id == combinations[i][0]
1681							&& visitor._client.id == combinations[i][1]) {
1682								return true
1683						}
1684					};
1685
1686					return false;
1687				},
1688
1689				// is the IP address from a known bot network?
1690				fromKnownBotIP: function(visitor) {
1691					//console.info('fromKnownBotIP()', visitor.ip);
1692
1693					return visitor.hasOwnProperty('_ipRange');
1694				},
1695
1696				// is the page language mentioned in the client's accepted languages?
1697				// the parameter holds an array of exceptions, i.e. page languages that should be ignored.
1698				matchLang: function(visitor, ...exceptions) {
1699
1700					if (visitor.lang && visitor.accept && exceptions.indexOf(visitor.lang) < 0) {
1701						return (visitor.accept.split(',').indexOf(visitor.lang) < 0);
1702					}
1703					return false;
1704				},
1705
1706				// the "Accept language" header contains certain entries:
1707				clientAccepts: function(visitor, ...languages) {
1708					//console.info('clientAccepts', visitor.accept, languages);
1709
1710					if (visitor.accept && languages) {;
1711						return ( visitor.accept.split(',').filter(lang => languages.includes(lang)).length > 0 );
1712					}
1713					return false;
1714				},
1715
1716				// Is there an accept-language field defined at all?
1717				noAcceptLang: function(visitor) {
1718
1719					if (!visitor.accept || visitor.accept.length <= 0) { // no accept-languages header
1720						return true;
1721					}
1722					// TODO: parametrize this!
1723					return false;
1724				},
1725				// At least x page views were recorded, but they come within less than y seconds
1726				loadSpeed: function(visitor, minItems, maxTime) {
1727
1728					if (visitor._viewCount >= minItems) {
1729						//console.log('loadSpeed', visitor._pageViews.length, minItems, maxTime);
1730
1731						const pvArr = visitor._pageViews.map(pv => pv._lastSeen).sort();
1732
1733						let totalTime = 0;
1734						for (let i=1; i < pvArr.length; i++) {
1735							totalTime += (pvArr[i] - pvArr[i-1]);
1736						}
1737
1738						//console.log('     ', totalTime , Math.round(totalTime / (pvArr.length * 1000)), (( totalTime / pvArr.length ) <= maxTime * 1000), visitor.ip);
1739
1740						return (( totalTime / pvArr.length ) <= maxTime * 1000);
1741					}
1742				},
1743
1744				// Country code matches one of those in the list:
1745				matchesCountry: function(visitor, ...countries) {
1746
1747					// ingore if geoloc is not set or unknown:
1748					if (visitor.geo) {
1749						return (countries.indexOf(visitor.geo) >= 0);
1750					}
1751					return false;
1752				},
1753
1754				// Country does not match one of the given codes.
1755				notFromCountry: function(visitor, ...countries) {
1756
1757					// ingore if geoloc is not set or unknown:
1758					if (visitor.geo && visitor.geo !== 'ZZ') {
1759						return (countries.indexOf(visitor.geo) < 0);
1760					}
1761					return false;
1762				},
1763
1764				// Check if visitor never went beyond a captcha
1765				blockedByCaptcha: function(visitor) {
1766					return (visitor._captcha.Y > 0 && visitor._captcha.N === 0);
1767				},
1768
1769				// Check if visitor came from a whitelisted IP-address
1770				whitelistedByCaptcha: function(visitor) {
1771					return (visitor._captcha.W > 0);
1772				}
1773			}
1774		},
1775
1776		/**
1777		 * Loads a settings file from the specified list of filenames.
1778		 * If the file is successfully loaded, it will call the callback function
1779		 * with the loaded JSON data.
1780		 * If no file can be loaded, it will display an error message.
1781		 *
1782		 * @param {string[]} fns - list of filenames to load
1783		 * @param {function} callback - function to call with the loaded JSON data
1784		 */
1785		_loadSettingsFile: async function(fns, callback) {
1786			//console.info('BotMon.live.data._loadSettingsFile()', fns);
1787
1788			const kJsonExt = '.json';
1789			let loaded = false; // if successfully loaded file
1790
1791			for (let i=0; i<fns.length; i++) {
1792				const filename = fns[i] +kJsonExt;
1793				try {
1794					const response = await fetch(DOKU_BASE + 'lib/plugins/botmon/config/' + filename);
1795					if (!response.ok) {
1796						continue;
1797					} else {
1798						loaded = true;
1799					}
1800					const json = await response.json();
1801					if (callback && typeof callback === 'function') {
1802						callback(json);
1803					}
1804					break;
1805				} catch (e) {
1806					BotMon.live.gui.status.setError("Error while loading the config file: " + filename);
1807				}
1808			}
1809
1810			if (!loaded) {
1811				BotMon.live.gui.status.setError("Could not load a config file.");
1812			}
1813		},
1814
1815		/**
1816		 * Loads a log file (server, page load, or ticker) and parses it.
1817		 * @param {String} type - the type of the log file to load (srv, log, or tck)
1818		 * @param {Function} [onLoaded] - an optional callback function to call after loading is finished.
1819		 */
1820		loadLogFile: async function(type, onLoaded = undefined) {
1821			//console.info('BotMon.live.data.loadLogFile(',type,')');
1822
1823			let typeName = '';
1824			let columns = [];
1825
1826			switch (type) {
1827				case "srv":
1828					typeName = "Server";
1829					columns = ['ts','ip','pg','id','typ','usr','agent','ref','lang','accept','geo','captcha'];
1830					break;
1831				case "log":
1832					typeName = "Page load";
1833					columns = ['ts','ip','pg','id','usr','lt','ref','agent'];
1834					break;
1835				case "tck":
1836					typeName = "Ticker";
1837					columns = ['ts','ip','pg','id','agent'];
1838					break;
1839				default:
1840					console.warn(`Unknown log type ${type}.`);
1841					return;
1842			}
1843
1844			// Show the busy indicator and set the visible status:
1845			BotMon.live.gui.status.showBusy(`Loading ${typeName} log file …`);
1846
1847			// compose the URL from which to load:
1848			const url = BotMon._baseDir + `logs/${BotMon._datestr}.${type}.txt`;
1849			//console.log("Loading:",url);
1850
1851			// fetch the data:
1852			try {
1853				const response = await fetch(url);
1854				if (!response.ok) {
1855
1856					throw new Error(`${response.status} ${response.statusText}`);
1857
1858				} else {
1859
1860					// parse the data:
1861					const logtxt = await response.text();
1862					if (logtxt.length <= 0) {
1863						throw new Error(`Empty log file ${url}.`);
1864					}
1865
1866					logtxt.split('\n').forEach((line) => {
1867						if (line.trim() === '') return; // skip empty lines
1868						const cols = line.split('\t');
1869
1870						// assign the columns to an object:
1871						const data = {};
1872						cols.forEach( (colVal,i) => {
1873							const colName = columns[i] || `col${i}`;
1874							const colValue = (colName == 'ts' ? new Date(colVal) : colVal.trim());
1875							data[colName] = colValue;
1876						});
1877
1878						// register the visit in the model:
1879						switch(type) {
1880							case BM_LOGTYPE.SERVER:
1881								BotMon.live.data.model.registerVisit(data, type);
1882								break;
1883							case BM_LOGTYPE.CLIENT:
1884								data.typ = 'js';
1885								BotMon.live.data.model.updateVisit(data);
1886								break;
1887							case BM_LOGTYPE.TICKER:
1888								data.typ = 'js';
1889								BotMon.live.data.model.updateTicks(data);
1890								break;
1891							default:
1892								console.warn(`Unknown log type ${type}.`);
1893								return;
1894						}
1895					});
1896				}
1897
1898			} catch (error) {
1899				BotMon.live.gui.status.setError(`Error while loading the ${typeName} log file: ${error.message} – data may be incomplete.`);
1900			} finally {
1901				BotMon.live.gui.status.hideBusy("Status: Done.");
1902				if (onLoaded) {
1903					onLoaded(); // callback after loading is finished.
1904				}
1905			}
1906		}
1907	},
1908
1909	gui: {
1910		init: function() {
1911			//console.log('BotMon.live.gui.init()');
1912
1913			// init sub-objects:
1914			BotMon.t._callInit(this);
1915		},
1916
1917		tabs: {
1918			init: function() {
1919				//console.log('BotMon.live.gui.tabs.init()');
1920
1921				/* find and add all existing tabs */
1922				document.querySelectorAll('#botmon__admin *[role=tablist]')
1923					.forEach((tablist) => {
1924						tablist.querySelectorAll('*[role=tab]')
1925					.forEach( t => t.addEventListener('click', this._onTabClick) )
1926					});
1927			},
1928
1929			/* callback for tab click */
1930			_onTabClick: function(e) {
1931				//console.log('BotMon.live.gui.tabs._onTabClick()');
1932
1933				/* reusable constants: */
1934				const kAriaSelected = 'aria-selected';
1935				const kAriaControls = 'aria-controls';
1936				const kTrue = 'true';
1937				const kFalse = 'false';
1938				const kHidden = 'hidden';
1939
1940				/* cancel default action */
1941				e.preventDefault();
1942
1943				/* if the active tab is clicked, do nothing: */
1944				let selState = this.getAttribute(kAriaSelected);
1945				if ( selState && selState == kTrue ) {
1946					return;
1947				}
1948
1949				/* find the active tab element: */
1950				var aItem = null;
1951				let tablist = this.parentNode;
1952				while (tablist.getAttribute('role') !== 'tablist') {
1953					tablist = tablist.parentNode;
1954				}
1955
1956				if (tablist.getAttribute('role') == 'tablist') {
1957					let lis = tablist.querySelectorAll('*[role=tab]');
1958					lis.forEach( (it) => {
1959						let selected = it.getAttribute(kAriaSelected);
1960						if ( selected && selected == kTrue ) {
1961							aItem = it;
1962						}
1963					});
1964				}
1965
1966				/* swap the active states: */
1967				this.setAttribute(kAriaSelected, kTrue);
1968				if (aItem) {
1969					aItem.setAttribute(kAriaSelected, kFalse);
1970					let aId = aItem.getAttribute(kAriaControls);
1971					let aObj = document.getElementById(aId);
1972					if (aObj) aObj.hidden = true;
1973				}
1974
1975				/* show the new panel: */
1976				let nId = this.getAttribute(kAriaControls);
1977				let nObj = document.getElementById(nId);
1978				if (nObj) nObj.hidden = false;
1979			}
1980		},
1981
1982		/* The Overview / web metrics section of the live tab */
1983		overview: {
1984			/**
1985			 * Populates the overview part of the today tab with the analytics data.
1986			 *
1987			 * @method make
1988			 * @memberof BotMon.live.gui.overview
1989			 */
1990			make: function() {
1991
1992				const maxItemsPerList = 5; // how many list items to show?
1993				const useCaptcha = BMSettings.useCaptcha || false;
1994
1995				const kNoData = '–'; // shown when data is missing
1996				const kSeparator = ' / ';
1997
1998				// shortcuts for neater code:
1999				const makeElement = BotMon.t._makeElement;
2000				const data = BotMon.live.data.analytics.data;
2001
2002				const botsVsHumans = document.getElementById('botmon__today__botsvshumans');
2003				if (botsVsHumans) {
2004					botsVsHumans.appendChild(makeElement('dt', {}, "Bot statistics"));
2005
2006					for (let i = 0; i <= ( useCaptcha ? 5 : 3 ); i++) {
2007						const dd = makeElement('dd');
2008						let title = '';
2009						let value = '';
2010						switch(i) {
2011							case 0:
2012								title = "Total (loads / views / visits):";
2013								value = (data.loads.total || kNoData) + kSeparator + (data.views.total || kNoData) + kSeparator + (data.visits.total || kNoData);
2014								break;
2015							case 1:
2016								title = "Known bots (views / visits):";
2017								value = (data.views.bots || kNoData) + kSeparator + (data.visits.bots || kNoData);
2018								break;
2019							case 2:
2020								title = "Suspected bots (views / visits):";
2021								value = (data.visits.suspected || kNoData) + kSeparator + (data.views.suspected || kNoData)
2022								break;
2023							case 3:
2024								title = "Bots-humans ratio (views / visits):";
2025								value = BotMon.t._getRatio(data.views.suspected + data.views.bots, data.views.users + data.views.humans, 100) + kSeparator + BotMon.t._getRatio(data.visits.suspected + data.visits.bots, data.visits.users + data.visits.humans, 100);
2026								break;
2027							case 4:
2028								title = "Known bots blocked / passed / whitelisted:";
2029								value = data.captcha.bots_blocked + kSeparator + data.captcha.bots_passed + kSeparator + data.captcha.bots_whitelisted;
2030								break;
2031							case 5:
2032								title = "Suspected bots blocked / passed / whitelisted:";
2033								value = data.captcha.sus_blocked + kSeparator + data.captcha.sus_passed + kSeparator + data.captcha.sus_whitelisted;
2034								break;
2035							default:
2036								console.warn(`Unknown list type ${i}.`);
2037						}
2038						dd.appendChild(makeElement('span', {}, title));
2039						dd.appendChild(makeElement('strong', {}, value));
2040						botsVsHumans.appendChild(dd);
2041					}
2042				}
2043
2044				// update known bots list:
2045				const botElement = document.getElementById('botmon__botslist'); /* Known bots */
2046				if (botElement) {
2047					botElement.appendChild(makeElement('dt', {}, `Top known bots`));
2048
2049					let botList = BotMon.live.data.analytics.getTopBots(maxItemsPerList);
2050					botList.forEach( (botInfo) => {
2051						const bli = makeElement('dd');
2052						bli.appendChild(makeElement('span', {'class': 'has_icon bot bot_' + botInfo.id }, botInfo.name));
2053						bli.appendChild(makeElement('span', {'class': 'count' }, botInfo.count));
2054						botElement.append(bli)
2055					});
2056				}
2057
2058				// update the suspected bot IP ranges list:
2059				const botIps = document.getElementById('botmon__botips');
2060				if (botIps) {
2061					botIps.appendChild(makeElement('dt', {}, "Top bot Networks"));
2062
2063					const ispList = BotMon.live.data.analytics.getTopBotISPs(5);
2064					//console.log(ispList);
2065					ispList.forEach( (netInfo) => {
2066						const li = makeElement('dd');
2067						li.appendChild(makeElement('span', {'class': 'has_icon ipaddr ip' + netInfo.typ }, netInfo.name));
2068						li.appendChild(makeElement('span', {'class': 'count' }, netInfo.count));
2069						botIps.append(li)
2070					});
2071				}
2072
2073				// update the top bot countries list:
2074				const botCountries = document.getElementById('botmon__botcountries');
2075				if (botCountries) {
2076					botCountries.appendChild(makeElement('dt', {}, `Top bot Countries`));
2077					const countryList = BotMon.live.data.analytics.getCountryList('bot', 5);
2078					countryList.forEach( (cInfo) => {
2079						const cLi = makeElement('dd');
2080						cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name));
2081						cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count));
2082						botCountries.appendChild(cLi);
2083					});
2084				}
2085
2086				// update the webmetrics overview:
2087				const wmoverview = document.getElementById('botmon__today__wm_overview');
2088				if (wmoverview) {
2089
2090					const humanVisits = data.views.total;
2091					const bounceRate = Math.round(100 * (BotMon.live.data.analytics.getBounceCount('users') + BotMon.live.data.analytics.getBounceCount('humans')) / humanVisits);
2092
2093					wmoverview.appendChild(makeElement('dt', {}, "Humans’ metrics"));
2094					for (let i = 0; i <= 4; i++) {
2095						const dd = makeElement('dd');
2096						let title = '';
2097						let value = '';
2098						switch(i) {
2099							case 0:
2100								title = "Registered users (views / visits):";
2101								value = (data.views.users || kNoData) + kSeparator + (data.visits.users || kNoData);
2102								break;
2103							case 1:
2104								title = "Probably humans (views / visits):";
2105								value = (data.views.humans || kNoData) + kSeparator + (data.visits.humans || kNoData);
2106								break;
2107							case 2:
2108								title = "Total human page views:";
2109								value = (data.views.users + data.views.humans) || kNoData;
2110								break;
2111							case 3:
2112								title = "Total human visits:";
2113								value = data.views.total || kNoData;
2114								break;
2115							case 4:
2116								title = "Humans’ bounce rate:";
2117								value = bounceRate + '%';
2118								break;
2119							default:
2120								console.warn(`Unknown list type ${i}.`);
2121						}
2122						dd.appendChild(makeElement('span', {}, title));
2123						dd.appendChild(makeElement('strong', {}, value));
2124						wmoverview.appendChild(dd);
2125					}
2126				}
2127
2128				// update the webmetrics clients list:
2129				const wmclients = document.getElementById('botmon__today__wm_clients');
2130				if (wmclients) {
2131
2132					wmclients.appendChild(makeElement('dt', {}, "Browsers"));
2133
2134					const clientList = BotMon.live.data.analytics.getTopBrowsers(maxItemsPerList);
2135					if (clientList) {
2136						clientList.forEach( (cInfo) => {
2137							const cDd = makeElement('dd');
2138							cDd.appendChild(makeElement('span', {'class': 'has_icon client cl_' + cInfo.id }, ( cInfo.name ? cInfo.name : cInfo.id)));
2139							cDd.appendChild(makeElement('span', {
2140								'class': 'count',
2141								'title': cInfo.count + " page views"
2142							}, cInfo.pct.toFixed(1) + '%'));
2143							wmclients.appendChild(cDd);
2144						});
2145					}
2146				}
2147
2148				// update the webmetrics platforms list:
2149				const wmplatforms = document.getElementById('botmon__today__wm_platforms');
2150				if (wmplatforms) {
2151
2152					wmplatforms.appendChild(makeElement('dt', {}, "Platforms"));
2153
2154					const pfList = BotMon.live.data.analytics.getTopPlatforms(maxItemsPerList);
2155					if (pfList) {
2156						pfList.forEach( (pInfo) => {
2157							const pDd = makeElement('dd');
2158							pDd.appendChild(makeElement('span', {'class': 'has_icon platform pf_' + pInfo.id }, ( pInfo.name ? pInfo.name : pInfo.id)));
2159							pDd.appendChild(makeElement('span', {
2160								'class': 'count',
2161								'title': pInfo.count + " page views"
2162							}, pInfo.pct.toFixed(1) + '%'));
2163							wmplatforms.appendChild(pDd);
2164						});
2165					}
2166				}
2167
2168				// update the top bot countries list:
2169				const usrCountries = document.getElementById('botmon__today__wm_countries');
2170				if (usrCountries) {
2171					usrCountries.appendChild(makeElement('dt', {}, `Top visitor Countries:`));
2172					const usrCtryList = BotMon.live.data.analytics.getCountryList('human', 5);
2173					usrCtryList.forEach( (cInfo) => {
2174						const cLi = makeElement('dd');
2175						cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name));
2176						cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count));
2177						usrCountries.appendChild(cLi);
2178					});
2179				}
2180
2181				// update the top pages;
2182				const wmpages = document.getElementById('botmon__today__wm_pages');
2183				if (wmpages) {
2184
2185					wmpages.appendChild(makeElement('dt', {}, "Top pages"));
2186
2187					const pgList = BotMon.live.data.analytics.getTopPages(maxItemsPerList);
2188					if (pgList) {
2189						pgList.forEach( (pgInfo) => {
2190							const pgDd = makeElement('dd');
2191							pgDd.appendChild(makeElement('a', {
2192								'class': 'page_icon',
2193								'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(pgInfo.id),
2194								'target': 'preview',
2195								'title': "PageID: " + pgInfo.id
2196							}, pgInfo.id));
2197							pgDd.appendChild(makeElement('span', {
2198								'class': 'count',
2199								'title': pgInfo.count + " page views"
2200							}, pgInfo.count));
2201							wmpages.appendChild(pgDd);
2202						});
2203					}
2204				}
2205
2206				// update the top referrers;
2207				const wmreferers = document.getElementById('botmon__today__wm_referers');
2208				if (wmreferers) {
2209
2210					wmreferers.appendChild(makeElement('dt', {}, "Referers"));
2211
2212					const refList = BotMon.live.data.analytics.getTopReferers(maxItemsPerList);
2213					if (refList) {
2214						refList.forEach( (rInfo) => {
2215							const rDd = makeElement('dd');
2216							rDd.appendChild(makeElement('span', {'class': 'has_icon referer ref_' + rInfo.id }, rInfo.name));
2217							rDd.appendChild(makeElement('span', {
2218								'class': 'count',
2219								'title': rInfo.count + " references"
2220							}, rInfo.pct.toFixed(1) + '%'));
2221							wmreferers.appendChild(rDd);
2222						});
2223					}
2224				}
2225			}
2226		},
2227
2228		status: {
2229			setText: function(txt) {
2230				const el = document.getElementById('botmon__today__status');
2231				if (el && BotMon.live.gui.status._errorCount <= 0) {
2232					el.innerText = txt;
2233				}
2234			},
2235
2236			setTitle: function(html) {
2237				const el = document.getElementById('botmon__today__title');
2238				if (el) {
2239					el.innerHTML = html;
2240				}
2241			},
2242
2243			setError: function(txt) {
2244				console.error(txt);
2245				BotMon.live.gui.status._errorCount += 1;
2246				const el = document.getElementById('botmon__today__status');
2247				if (el) {
2248					el.innerText = "Data may be incomplete.";
2249					el.classList.add('error');
2250				}
2251			},
2252			_errorCount: 0,
2253
2254			showBusy: function(txt = null) {
2255				BotMon.live.gui.status._busyCount += 1;
2256				const el = document.getElementById('botmon__today__busy');
2257				if (el) {
2258					el.style.display = 'inline-block';
2259				}
2260				if (txt) BotMon.live.gui.status.setText(txt);
2261			},
2262			_busyCount: 0,
2263
2264			hideBusy: function(txt = null) {
2265				const el = document.getElementById('botmon__today__busy');
2266				BotMon.live.gui.status._busyCount -= 1;
2267				if (BotMon.live.gui.status._busyCount <= 0) {
2268					if (el) el.style.display = 'none';
2269					if (txt) BotMon.live.gui.status.setText(txt);
2270				}
2271			}
2272		},
2273
2274		lists: {
2275			init: function() {
2276
2277				// function shortcut:
2278				const makeElement = BotMon.t._makeElement;
2279
2280				const parent = document.getElementById('botmon__today__visitorlists');
2281				if (parent) {
2282
2283					for (let i=0; i < 4; i++) {
2284
2285						// change the id and title by number:
2286						let listTitle = '';
2287						let listId = '';
2288						let infolink = null;
2289						switch (i) {
2290							case 0:
2291								listTitle = "Registered users";
2292								listId = 'users';
2293								break;
2294							case 1:
2295								listTitle = "Probably humans";
2296								listId = 'humans';
2297								break;
2298							case 2:
2299								listTitle = "Suspected bots";
2300								listId = 'suspectedBots';
2301								infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/suspected_bots';
2302								break;
2303							case 3:
2304								listTitle = "Known bots";
2305								listId = 'knownBots';
2306								infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/known_bots';
2307								break;
2308							default:
2309								console.warn('Unknown list number.');
2310						}
2311
2312						const details = makeElement('details', {
2313							'data-group': listId,
2314							'data-loaded': false
2315						});
2316						const title = details.appendChild(makeElement('summary'));
2317						title.appendChild(makeElement('span', {'class': 'title'}, listTitle));
2318						if (infolink) {
2319							title.appendChild(makeElement('a', {
2320								'class': 'ext_info',
2321								'target': '_blank',
2322								'href': infolink,
2323								'title': "More information"
2324							}, "Info"));
2325						}
2326						details.addEventListener("toggle", this._onDetailsToggle);
2327
2328						parent.appendChild(details);
2329
2330					}
2331				}
2332			},
2333
2334			_onDetailsToggle: function(e) {
2335				//console.info('BotMon.live.gui.lists._onDetailsToggle()');
2336
2337				const target = e.target;
2338
2339				if (target.getAttribute('data-loaded') == 'false') { // only if not loaded yet
2340					target.setAttribute('data-loaded', 'loading');
2341
2342					const fillType = target.getAttribute('data-group');
2343					const fillList = BotMon.live.data.analytics.groups[fillType];
2344					if (fillList && fillList.length > 0) {
2345
2346						const ul = BotMon.t._makeElement('ul');
2347
2348						fillList.forEach( (it) => {
2349							ul.appendChild(BotMon.live.gui.lists._makeVisitorItem(it, fillType));
2350						});
2351
2352						target.appendChild(ul);
2353						target.setAttribute('data-loaded', 'true');
2354					} else {
2355						target.setAttribute('data-loaded', 'false');
2356					}
2357
2358				}
2359			},
2360
2361			_makeVisitorItem: function(data, type) {
2362
2363				// shortcuts for neater code:
2364				const make = BotMon.t._makeElement;
2365				const model = BotMon.live.data.model;
2366
2367				let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' );
2368				if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0';
2369
2370				const platformName = (data._platform ? data._platform.n : 'Unknown');
2371				const clientName = (data._client ? data._client.n: 'Unknown');
2372
2373				const sumClass = ( !data._seenBy || data._seenBy.indexOf(BM_LOGTYPE.SERVER) < 0 ? 'noServer' : 'hasServer');
2374
2375				// combine with other networks?
2376				const combineNets = (BMSettings.hasOwnProperty('combineNets') ? BMSettings['combineNets'] : true)
2377					&& data.hasOwnProperty('_ipRange');
2378
2379				const li = make('li'); // root list item
2380				const details = make('details');
2381				const summary = make('summary', {
2382					'class': sumClass
2383				});
2384				details.appendChild(summary);
2385
2386				const span1 = make('span'); /* left-hand group */
2387
2388				/*if (data._type !== BM_USERTYPE.KNOWN_BOT) { // No platform/client for bots // disabled because no longer relevant
2389					span1.appendChild(make('span', { // Platform
2390						'class': 'icon_only platform pf_' + (data._platform ? data._platform.id : 'unknown'),
2391						'title': "Platform: " + platformName
2392					}, platformName));
2393
2394					span1.appendChild(make('span', { // Client
2395						'class': 'icon_only client client cl_' + (data._client ? data._client.id : 'unknown'),
2396						'title': "Client: " + clientName
2397					}, clientName));
2398				}*/
2399
2400				// identifier:
2401				if (data._type == BM_USERTYPE.KNOWN_BOT) { /* Bot only */
2402
2403					const botName = ( data._bot && data._bot.n ? data._bot.n : "Unknown");
2404					span1.appendChild(make('span', { /* Bot */
2405						'class': 'has_icon bot bot_' + (data._bot ? data._bot.id : 'unknown'),
2406						'title': "Bot: " + botName
2407					}, botName));
2408
2409				} else if (data._type == BM_USERTYPE.KNOWN_USER) { /* User only */
2410
2411					span1.appendChild(make('span', { /* User */
2412						'class': 'has_icon user_known',
2413						'title': "User: " + data.usr
2414					}, data.usr));
2415
2416				} else { /* others */
2417
2418					if (combineNets) {
2419
2420						const ispName = BotMon.live.data.ipRanges.getOwner( data._ipRange.g ) || data._ipRange.g;
2421
2422						span1.appendChild(make('span', { // IP-Address
2423							'class': 'has_icon ipaddr ipnet',
2424							'title': "IP-Range: " + data._ipRange.g
2425						}, ispName));
2426
2427					} else {
2428
2429						span1.appendChild(make('span', { // IP-Address
2430							'class': 'has_icon ipaddr ip' + ipType,
2431							'title': "IP-Address: " + data.ip
2432						}, data.ip));
2433					}
2434				}
2435
2436				span1.appendChild(make('span', { /* page views */
2437					'class': 'has_icon pageseen',
2438					'title': data._pageViews.length + " page load(s)"
2439				}, data._pageViews.length));
2440
2441
2442				// referer icons:
2443				if ((data._type == BM_USERTYPE.PROBABLY_HUMAN || data._type == BM_USERTYPE.LIKELY_BOT) && data.ref) {
2444					const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref);
2445					span1.appendChild(make('span', {
2446						'class': 'icon_only referer ref_' + refInfo.id,
2447						'title': "Referer: " + data.ref
2448					}, refInfo.n));
2449				}
2450
2451				summary.appendChild(span1);
2452				const span2 = make('span'); /* right-hand group */
2453
2454				// country flag:
2455				if (!combineNets) { // not for combined networks
2456					if (data.geo && data.geo !== 'ZZ') {
2457						span2.appendChild(make('span', {
2458							'class': 'icon_only country ctry_' + data.geo.toLowerCase(),
2459							'data-ctry': data.geo,
2460							'title': "Country: " + ( data._country || "Unknown")
2461						}, ( data._country || "Unknown") ));
2462					}
2463				}
2464
2465				span2.appendChild(make('span', { // seen-by icon:
2466					'class': 'icon_only seenby sb_' + data._seenBy.join(''),
2467					'title': "Seen by: " + data._seenBy.join('+')
2468				}, data._seenBy.join(', ')));
2469
2470				// captcha status:
2471				const cCode = ( data._captcha ? data._captcha._str() : '');
2472				if (cCode !== '') {
2473					const cTitle = model._makeCaptchaTitle(data._captcha)
2474					span2.appendChild(make('span', { // captcha status
2475						'class': 'icon_only captcha cap_' + cCode,
2476						'title': "Captcha-status: " + cTitle
2477					}, cTitle));
2478				}
2479
2480				summary.appendChild(span2);
2481
2482				// add details expandable section:
2483				details.appendChild(BotMon.live.gui.lists._makeVisitorDetails(data, type));
2484
2485				li.appendChild(details);
2486				return li;
2487			},
2488
2489			_makeVisitorDetails: function(data, type) {
2490
2491				// shortcuts for neater code:
2492				const make = BotMon.t._makeElement;
2493				const model = BotMon.live.data.model;
2494
2495				let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' );
2496				if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0';
2497				const platformName = (data._platform ? data._platform.n : 'Unknown');
2498				const clientName = (data._client ? data._client.n: 'Unknown');
2499
2500				const dl = make('dl', {'class': 'visitor_details'});
2501
2502				if (data._type == BM_USERTYPE.KNOWN_BOT) {
2503
2504					dl.appendChild(make('dt', {}, "Bot name:")); /* bot info */
2505					dl.appendChild(make('dd', {'class': 'icon_only bot bot_' + (data._bot ? data._bot.id : 'unknown')},
2506						(data._bot ? data._bot.n : 'Unknown')));
2507
2508					if (data._bot && data._bot.url) {
2509						dl.appendChild(make('dt', {}, "Bot info:")); /* bot info */
2510						const botInfoDd = dl.appendChild(make('dd'));
2511						botInfoDd.appendChild(make('a', {
2512							'href': data._bot.url,
2513							'target': '_blank'
2514						}, data._bot.url)); /* bot info link*/
2515
2516					}
2517
2518				} else { /* not for bots */
2519
2520					dl.appendChild(make('dt', {}, "Client:")); /* client */
2521					dl.appendChild(make('dd', {'class': 'has_icon client cl_' + (data._client ? data._client.id : 'unknown')},
2522						clientName + ( data._client.v > 0 ? ' (' + data._client.v + ')' : '' ) ));
2523
2524					dl.appendChild(make('dt', {}, "Platform:")); /* platform */
2525					dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')},
2526						platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) ));
2527
2528					/*dl.appendChild(make('dt', {}, "ID:"));
2529					dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/
2530				}
2531
2532				dl.appendChild(make('dt', {}, "IP-Address:"));
2533				const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType});
2534					ipItem.appendChild(make('span', {'class': 'address'} , data.ip));
2535					ipItem.appendChild(make('a', {
2536						'class': 'icon_only extlink ipinfo',
2537						'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`,
2538						'target': 'ipinfo',
2539						'title': "View this address on IPInfo.io"
2540					} , "DNS Info"));
2541					ipItem.appendChild(make('a', {
2542						'class': 'icon_only extlink abuseipdb',
2543						'href': `https://www.abuseipdb.com/check/${encodeURIComponent(data.ip)}`,
2544						'target': 'abuseipdb',
2545						'title': "Check this address on AbuseIPDB.com"
2546					} , "Check on AbuseIPDB"));
2547				dl.appendChild(ipItem);
2548
2549				if (Math.abs(data._lastSeen - data._firstSeen) < 100) {
2550					dl.appendChild(make('dt', {}, "Seen:"));
2551					dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString()));
2552				} else {
2553					dl.appendChild(make('dt', {}, "First seen:"));
2554					dl.appendChild(make('dd', {'class': 'firstSeen'}, data._firstSeen.toLocaleString()));
2555					dl.appendChild(make('dt', {}, "Last seen:"));
2556					dl.appendChild(make('dd', {'class': 'lastSeen'}, data._lastSeen.toLocaleString()));
2557				}
2558
2559				dl.appendChild(make('dt', {}, "Actions:"));
2560				dl.appendChild(make('dd', {'class': 'views'},
2561					"Page loads: " + data._loadCount.toString() +
2562					( data._captcha['Y'] > 0 || data._captcha['W'] || data._captcha['-'] > 0 ? ", captchas: " + (data._captcha['Y']+data._captcha['W']+data._captcha['-']).toString() : '') +
2563					", views: " + data._viewCount.toString()
2564				));
2565
2566				dl.appendChild(make('dt', {}, "User-Agent:"));
2567				dl.appendChild(make('dd', {'class': 'agent'}, data.agent));
2568
2569				if (data.ref && data.ref !== '') {
2570					dl.appendChild(make('dt', {}, "Referrer:"));
2571
2572					const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref);
2573					const refDd = dl.appendChild(make('dd', {
2574						'class': 'has_icon referer ref_' + refInfo.id
2575					}));
2576					refDd.appendChild(make('a', {
2577						'href': data.ref,
2578						'target': 'refTarget'
2579					}, data.ref));
2580				}
2581
2582				dl.appendChild(make('dt', {}, "Languages:"));
2583				dl.appendChild(make('dd', {'class': 'langs'}, ` [${data.accept}]`));
2584
2585				if (data.geo && data.geo !=='') {
2586					dl.appendChild(make('dt', {}, "Location:"));
2587					dl.appendChild(make('dd', {
2588						'class': 'has_icon country ctry_' + data.geo.toLowerCase(),
2589						'data-ctry': data.geo,
2590						'title': "Country: " + data._country
2591					}, data._country + ' (' + data.geo + ')'));
2592				}
2593
2594				if (data.captcha && data.captcha !=='') {
2595					dl.appendChild(make('dt', {}, "Captcha-status:"));
2596					dl.appendChild(make('dd', {
2597						'class': 'captcha'
2598					}, model._makeCaptchaTitle(data._captcha)));
2599				}
2600
2601				dl.appendChild(make('dt', {}, "Session ID:"));
2602				dl.appendChild(make('dd', {'class': 'has_icon session typ_' + data.typ}, data.id));
2603
2604				dl.appendChild(make('dt', {}, "Seen by:"));
2605				dl.appendChild(make('dd', {'class': 'has_icon seenby sb_' + data._seenBy.join('')}, data._seenBy.join(', ') ));
2606
2607				dl.appendChild(make('dt', {}, "Visited pages:"));
2608				const pagesDd = make('dd', {'class': 'pages'});
2609				const pageList = make('ul');
2610
2611				/* list all page views */
2612				data._pageViews.sort( (a, b) => a._firstSeen - b._firstSeen );
2613				data._pageViews.forEach( (page) => {
2614					pageList.appendChild(BotMon.live.gui.lists._makePageViewItem(page));
2615				});
2616				pagesDd.appendChild(pageList);
2617				dl.appendChild(pagesDd);
2618
2619				/* bot evaluation rating */
2620				if (data._type !== BM_USERTYPE.KNOWN_BOT && data._type !== BM_USERTYPE.KNOWN_USER) {
2621					dl.appendChild(make('dt', undefined, "Bot rating:"));
2622					dl.appendChild(make('dd', {'class': 'bot-rating'}, ( data._botVal ? data._botVal : '–' ) + ' (of ' + BotMon.live.data.rules._threshold + ')'));
2623
2624					/* add bot evaluation details: */
2625					if (data._eval) {
2626						dl.appendChild(make('dt', {}, "Bot evaluation:"));
2627						const evalDd = make('dd', {'class': 'eval'});
2628						const testList = make('ul');
2629						data._eval.forEach( test => {
2630
2631							const tObj = BotMon.live.data.rules.getRuleInfo(test);
2632							let tDesc = tObj ? tObj.desc : test;
2633
2634							// special case for Bot IP range test:
2635							if (tObj.func == 'fromKnownBotIP') {
2636								const rangeInfo = BotMon.live.data.ipRanges.match(data.ip);
2637								if (rangeInfo) {
2638									const owner = BotMon.live.data.ipRanges.getOwner(rangeInfo.g);
2639									tDesc += ' (range: “' + rangeInfo.cidr + '”, ' + owner + ')';
2640								}
2641							}
2642
2643							// create the entry field
2644							const tstLi = make('li');
2645							tstLi.appendChild(make('span', {
2646								'data-testid': test
2647							}, tDesc));
2648							tstLi.appendChild(make('span', {}, ( tObj ? tObj.bot : '—') ));
2649							testList.appendChild(tstLi);
2650						});
2651
2652						// add total row
2653						const tst2Li = make('li', {
2654							'class': 'total'
2655						});
2656						/*tst2Li.appendChild(make('span', {}, "Total:"));
2657						tst2Li.appendChild(make('span', {}, data._botVal));
2658						testList.appendChild(tst2Li);*/
2659
2660						evalDd.appendChild(testList);
2661						dl.appendChild(evalDd);
2662					}
2663				}
2664
2665				// for debugging only. Disable on production:
2666				dl.appendChild(make('dt', {}, "Debug info:"));
2667				const dbgDd = make('dd', {'class': 'debug'});
2668				dbgDd.innerHTML = '<pre>' + JSON.stringify(data, null, 4) + '</pre>';
2669				dl.appendChild(dbgDd);
2670
2671				// return the element to add to the UI:
2672				return dl;
2673			},
2674
2675			// make a page view item:
2676			_makePageViewItem: function(page) {
2677				//console.log("makePageViewItem:",page);
2678
2679				// shortcut for neater code:
2680				const make = BotMon.t._makeElement;
2681
2682				// the actual list item:
2683				const pgLi = make('li');
2684
2685				const row1 = make('div', {'class': 'row'});
2686
2687					row1.appendChild(make('a', { // page id is the left group
2688						'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(page.pg),
2689						'target': 'preview',
2690						'hreflang': page.lang,
2691						'title': "PageID: " + page.pg
2692					}, page.pg)); /* DW Page ID */
2693
2694					const rightGroup = row1.appendChild(make('div')); // right-hand group
2695
2696						// get the time difference:
2697						rightGroup.appendChild(make('span', {
2698							'class': 'first-seen',
2699							'title': "First visited: " + page._firstSeen.toLocaleString() + " UTC"
2700						}, BotMon.t._formatTime(page._firstSeen)));
2701
2702						rightGroup.appendChild(make('span', { /* page loads */
2703							'class': 'has_icon pageviews',
2704							'title': page._viewCount.toString() + " page load(s)"
2705						}, page._viewCount.toString()));
2706
2707				pgLi.appendChild(row1);
2708
2709				/* LINE 2 */
2710
2711				const row2 = make('div', {'class': 'row'});
2712
2713					// page referrer:
2714					if (page._ref) {
2715						row2.appendChild(make('span', {
2716							'class': 'referer',
2717							'title': "Referrer: " + page._ref.href
2718						}, page._ref.hostname));
2719					} else {
2720						row2.appendChild(make('span', {
2721							'class': 'referer'
2722						}, "No referer"));
2723					}
2724
2725					// visit duration:
2726					let visitTimeStr = "Bounce";
2727					const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime();
2728					if (visitDuration > 0) {
2729						visitTimeStr = Math.floor(visitDuration / 1000) + "s";
2730					}
2731					const tDiff = BotMon.t._formatTimeDiff(page._firstSeen, page._lastSeen);
2732					if (tDiff) {
2733						row2.appendChild(make('span', {'class': 'visit-length', 'title': 'Last seen: ' + page._lastSeen.toLocaleString()}, tDiff));
2734					} else {
2735						row2.appendChild(make('span', {
2736							'class': 'bounce',
2737							'title': "Visitor bounced"}, "Bounce"));
2738					}
2739
2740				pgLi.appendChild(row2);
2741
2742				return pgLi;
2743			}
2744		}
2745	}
2746};
2747
2748/* launch only if the BotMon admin panel is open: */
2749if (document.getElementById('botmon__admin')) {
2750	BotMon.init();
2751}