xref: /plugin/botmon/admin.js (revision cef3810678186559c94d70d74859b653db49d4f4)
1"use strict";
2/* DokuWiki BotMon Plugin Script file */
3/* 14.10.2025 - 0.5.0 - pre-release */
4/* Author: Sascha Leib <ad@hominem.info> */
5
6// enumeration of user types:
7const BM_USERTYPE = Object.freeze({
8	'UNKNOWN': 'unknown',
9	'KNOWN_USER': 'user',
10	'PROBABLY_HUMAN': 'human',
11	'LIKELY_BOT': 'likely_bot',
12	'KNOWN_BOT': 'known_bot'
13});
14
15// enumeration of log types:
16const BM_LOGTYPE = Object.freeze({
17	'SERVER': 'srv',
18	'CLIENT': 'log',
19	'TICKER': 'tck'
20});
21
22// enumeration of IP versions:
23const BM_IPVERSION = Object.freeze({
24	'IPv4': 4,
25	'IPv6': 6
26});
27
28/* BotMon root object */
29const BotMon = {
30
31	init: function() {
32		//console.info('BotMon.init()');
33
34		// find the plugin basedir:
35		this._baseDir = document.currentScript.src.substring(0, document.currentScript.src.lastIndexOf('/')+1);
36
37		// read the page language from the DOM:
38		this._lang = document.getRootNode().documentElement.lang || this._lang;
39
40		// get the time offset:
41		this._timeDiff = BotMon.t._getTimeOffset();
42
43		// get yesterday's date:
44		let d = new Date();
45		if (BMSettings.showday == 'yesterday') d.setDate(d.getDate() - 1);
46		this._datestr = d.toISOString().slice(0, 10);
47
48		// init the sub-objects:
49		BotMon.t._callInit(this);
50	},
51
52	_baseDir: null,
53	_lang: 'en',
54	_datestr: '',
55	_timeDiff: '',
56
57	/* internal tools */
58	t: {
59		/* helper function to call inits of sub-objects */
60		_callInit: function(obj) {
61			//console.info('BotMon.t._callInit(obj=',obj,')');
62
63			/* call init / _init on each sub-object: */
64			Object.keys(obj).forEach( (key,i) => {
65				const sub = obj[key];
66				let init = null;
67				if (typeof sub === 'object' && sub.init) {
68					init = sub.init;
69				}
70
71				// bind to object
72				if (typeof init == 'function') {
73					const init2 = init.bind(sub);
74					init2(obj);
75				}
76			});
77		},
78
79		/* helper function to calculate the time difference to UTC: */
80		_getTimeOffset: function() {
81			const now = new Date();
82			let offset = now.getTimezoneOffset(); // in minutes
83			const sign = Math.sign(offset); // +1 or -1
84			offset = Math.abs(offset); // always positive
85
86			let hours = 0;
87			while (offset >= 60) {
88				hours += 1;
89				offset -= 60;
90			}
91			return ( hours > 0 ? sign * hours + ' h' : '') + (offset > 0 ? ` ${offset} min` : '');
92		},
93
94		/* helper function to create a new element with all attributes and text content */
95		_makeElement: function(name, atlist = undefined, text = undefined) {
96			var r = null;
97			try {
98				r = document.createElement(name);
99				if (atlist) {
100					for (let attr in atlist) {
101						r.setAttribute(attr, atlist[attr]);
102					}
103				}
104				if (text) {
105					r.textContent = text.toString();
106				}
107			} catch(e) {
108				console.error(e);
109			}
110			return r;
111		},
112
113		/* helper to convert an ip address string to a normalised format: */
114		_ip2Num: function(ip) {
115			if (!ip) {
116				return 'null';
117			} else if (ip.indexOf(':') > 0) { /* IP6 */
118				return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(':'));
119			} else { /* IP4 */
120				return ip.split('.').map(d => ('000'+d).slice(-3) ).join('.');
121			}
122		},
123
124		/* helper function to format a Date object to show only the time. */
125		/* returns String */
126		_formatTime: function(date) {
127
128			if (date) {
129				return date.getHours() + ':' + ('0'+date.getMinutes()).slice(-2) + ':' + ('0'+date.getSeconds()).slice(-2);
130			} else {
131				return null;
132			}
133
134		},
135
136		/* helper function to show a time difference in seconds or minutes */
137		/* returns String */
138		_formatTimeDiff: function(dateA, dateB) {
139
140			// if the second date is ealier, swap them:
141			if (dateA > dateB) dateB = [dateA, dateA = dateB][0];
142
143			// get the difference in milliseconds:
144			let ms = dateB - dateA;
145
146			if (ms > 50) { /* ignore small time spans */
147				const h = Math.floor((ms / (1000 * 60 * 60)) % 24);
148				const m = Math.floor((ms / (1000 * 60)) % 60);
149				const s = Math.floor((ms / 1000) % 60);
150
151				return ( h>0 ? h + 'h ': '') + ( m>0 ? m + 'm ': '') + ( s>0 ? s + 's': '');
152			}
153
154			return null;
155
156		},
157
158		// calcualte a reduced ration between two numbers
159		// adapted from https://stackoverflow.com/questions/3946373/math-in-js-how-do-i-get-a-ratio-from-a-percentage
160		_getRatio: function(a, b, tolerance) {
161
162			var bg = b;
163			var sm = a;
164			if (a > b) {
165				var bg = a;
166				var sm = b;
167			}
168
169			for (var i = 1; i < 1000000; i++) {
170				var d = sm / i;
171				var res = bg / d;
172				var howClose = Math.abs(res - res.toFixed(0));
173				if (howClose < tolerance) {
174					if (a > b) {
175						return res.toFixed(0) + ':' + i;
176					} else {
177						return i + ':' + res.toFixed(0);
178					}
179				}
180			}
181		}
182	}
183};
184
185/* everything specific to the 'Latest' tab is self-contained in the 'live' object: */
186BotMon.live = {
187	init: function() {
188		//console.info('BotMon.live.init()');
189
190		// set the title:
191		const tDiff = '<abbr title="Coordinated Universal Time">UTC</abbr> ' + (BotMon._timeDiff != '' ? ` (offset: ${BotMon._timeDiff}` : '' ) + ')';
192		BotMon.live.gui.status.setTitle(`Data for <time datetime="${BotMon._datestr}">${BotMon._datestr}</time> ${tDiff}`);
193
194		// init sub-objects:
195		BotMon.t._callInit(this);
196	},
197
198	data: {
199		init: function() {
200			//console.info('BotMon.live.data.init()');
201
202			// call sub-inits:
203			BotMon.t._callInit(this);
204		},
205
206		// this will be called when the known json files are done loading:
207		_dispatch: function(file) {
208			//console.info('BotMon.live.data._dispatch(,',file,')');
209
210			// shortcut to make code more readable:
211			const data = BotMon.live.data;
212
213			// set the flags:
214			switch(file) {
215				case 'rules':
216					data._dispatchRulesLoaded = true;
217					break;
218				case 'ipranges':
219					data._dispatchIPRangesLoaded = true;
220					break;
221				case 'bots':
222					data._dispatchBotsLoaded = true;
223					break;
224				case 'clients':
225					data._dispatchClientsLoaded = true;
226					break;
227				case 'platforms':
228					data._dispatchPlatformsLoaded = true;
229					break;
230				default:
231					// ignore
232			}
233
234			// are all the flags set?
235			if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded && data._dispatchIPRangesLoaded) {
236				// chain the log files loading:
237				BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded);
238			}
239		},
240		// flags to track which data files have been loaded:
241		_dispatchBotsLoaded: false,
242		_dispatchClientsLoaded: false,
243		_dispatchPlatformsLoaded: false,
244		_dispatchIPRangesLoaded: false,
245		_dispatchRulesLoaded: false,
246
247		// event callback, after the server log has been loaded:
248		_onServerLogLoaded: function() {
249			//console.info('BotMon.live.data._onServerLogLoaded()');
250
251			// chain the client log file to load:
252			BotMon.live.data.loadLogFile(BM_LOGTYPE.CLIENT, BotMon.live.data._onClientLogLoaded);
253		},
254
255		// event callback, after the client log has been loaded:
256		_onClientLogLoaded: function() {
257			//console.info('BotMon.live.data._onClientLogLoaded()');
258
259			// chain the ticks file to load:
260			BotMon.live.data.loadLogFile(BM_LOGTYPE.TICKER, BotMon.live.data._onTicksLogLoaded);
261
262		},
263
264		// event callback, after the tiker log has been loaded:
265		_onTicksLogLoaded: function() {
266			//console.info('BotMon.live.data._onTicksLogLoaded()');
267
268			// analyse the data:
269			BotMon.live.data.analytics.analyseAll();
270
271			// sort the data:
272			// #TODO
273
274			// display the data:
275			BotMon.live.gui.overview.make();
276
277			//console.log(BotMon.live.data.model._visitors);
278
279		},
280
281		// the data model:
282		model: {
283			// visitors storage:
284			_visitors: [],
285
286			// find an already existing visitor record:
287			findVisitor: function(visitor, type) {
288				//console.info('BotMon.live.data.model.findVisitor()', type);
289				//console.log(visitor);
290
291				// shortcut to make code more readable:
292				const model = BotMon.live.data.model;
293
294				//const timeout = 60 * 60 * 1000; // session timeout: One hour
295
296				if (visitor._type == BM_USERTYPE.KNOWN_BOT) { // known bots match by their bot ID:
297
298					for (let i=0; i<model._visitors.length; i++) {
299						const v = model._visitors[i];
300
301						// bots match when their ID matches:
302						if (v._bot && v._bot.id == visitor._bot.id) {
303							return v;
304						}
305					}
306
307				} else { // other types match by their DW/PHPIDs:
308
309					// loop over all visitors already registered and check for ID matches:
310					for (let i=0; i<model._visitors.length; i++) {
311						const v = model._visitors[i];
312
313						if ( v.id.trim() !== '' && v.id == visitor.id) { // match the DW/PHP IDs
314							return v;
315						}
316					}
317
318					// if not found, try to match IP address and user agent:
319					for (let i=0; i<model._visitors.length; i++) {
320						const v = model._visitors[i];
321						if (  v.ip == visitor.ip && v.agent == visitor.agent) {
322							return v;
323						}
324					}
325				}
326
327				return null; // nothing found
328			},
329
330			/* if there is already this visit registered, return the page view item */
331			_getPageView: function(visit, view) {
332
333				// shortcut to make code more readable:
334				const model = BotMon.live.data.model;
335
336				for (let i=0; i<visit._pageViews.length; i++) {
337					const pv = visit._pageViews[i];
338					if (pv.pg == view.pg) {
339						return pv;
340					}
341				}
342				return null; // not found
343			},
344
345			// register a new visitor (or update if already exists)
346			registerVisit: function(nv, type) {
347				//console.info('registerVisit', nv, type);
348
349				// shortcut to make code more readable:
350				const model = BotMon.live.data.model;
351
352				// is it a known bot?
353				const bot = BotMon.live.data.bots.match(nv.agent);
354
355				// enrich new visitor with relevant data:
356				if (!nv._bot) nv._bot = bot ?? null; // bot info
357				nv._type = ( bot ? BM_USERTYPE.KNOWN_BOT : ( nv.usr && nv.usr !== '' ? BM_USERTYPE.KNOWN_USER : BM_USERTYPE.UNKNOWN ) ); // user type
358				if (bot && bot.geo) {
359					if (!nv.geo || nv.geo == '' || nv.geo == 'ZZ') nv.geo = bot.geo;
360				} else if (!nv.geo ||nv.geo == '') {
361					nv.geo = 'ZZ';
362				}
363
364				// update first and last seen:
365				if (!nv._firstSeen) nv._firstSeen = nv.ts; // first-seen
366				nv._lastSeen = nv.ts; // last-seen
367
368				// country name:
369				try {
370					nv._country = ( nv.geo == 'local' ? "localhost" : "Unknown" );
371					if (nv.geo && nv.geo !== '' && nv.geo !== 'ZZ' && nv.geo !== 'local') {
372						const countryName = new Intl.DisplayNames(['en', BotMon._lang], {type: 'region'});
373						nv._country = countryName.of(nv.geo.substring(0,2)) ?? nv.geo;
374					}
375				} catch (err) {
376					console.error(err);
377					nv._country = 'Error';
378				}
379
380				// check if it already exists:
381				let visitor = model.findVisitor(nv, type);
382				if (!visitor) {
383					visitor = {...nv, ...{
384						_seenBy: [type],
385						_viewCount: 0, // number of page views
386						_loadCount: 0, // number of page loads (not necessarily views!)
387						_pageViews: [], // array of page views
388						_hasReferrer: false, // has at least one referrer
389						_jsClient: false, // visitor has been seen logged by client js as well
390						_client: BotMon.live.data.clients.match(nv.agent) ?? null, // client info
391						_platform: BotMon.live.data.platforms.match(nv.agent), // platform info
392						_captcha: {'-': 0, 'Y': 0, 'N': 0, 'W':0} // captcha counter
393					}};
394					model._visitors.push(visitor);
395				};
396
397				// update first and last seen:
398				if (visitor._firstSeen > nv.ts) {
399					visitor._firstSeen = nv.ts;
400				}
401				if (visitor._lastSeen < nv.ts) {
402					visitor._lastSeen = nv.ts;
403				}
404
405				// update total loads and views (not the same!):
406				visitor._loadCount += 1;
407				visitor._viewCount += (nv.captcha == 'Y' ? 0 : 1);
408
409				// ...because also a captcha is a "load", but not a "view".
410				// let's count the captcha statuses as well:
411				if (nv.captcha) visitor._captcha[nv.captcha] += 1;
412
413				// is this visit already registered?
414				let prereg = model._getPageView(visitor, nv);
415				if (!prereg) {
416					// add new page view:
417					prereg = model._makePageView(nv, type);
418					visitor._pageViews.push(prereg);
419				}
420
421				// update last seen date
422				prereg._lastSeen = nv.ts;
423
424				// increase view count:
425				prereg._loadCount += (visitor.captcha == 'Y' ? 0 : 1);
426				//prereg._tickCount += 1;
427
428				// update referrer state:
429				visitor._hasReferrer = visitor._hasReferrer ||
430					(prereg.ref !== undefined && prereg.ref !== '');
431
432				// update time stamp for last-seen:
433				if (visitor._lastSeen < nv.ts) {
434					visitor._lastSeen = nv.ts;
435				}
436
437				// if needed:
438				return visitor;
439			},
440
441			// updating visit data from the client-side log:
442			updateVisit: function(dat) {
443				//console.info('updateVisit', dat);
444
445				// shortcut to make code more readable:
446				const model = BotMon.live.data.model;
447
448				const type = BM_LOGTYPE.CLIENT;
449
450				let visitor = BotMon.live.data.model.findVisitor(dat, type);
451				if (!visitor) {
452					visitor = model.registerVisit(dat, type);
453				}
454				if (visitor) {
455
456					if (visitor._lastSeen < dat.ts) {
457						visitor._lastSeen = dat.ts;
458					}
459					if (!visitor._seenBy.includes(type)) {
460						visitor._seenBy.push(type);
461					}
462					visitor._jsClient = true; // seen by client js
463				}
464
465				// find the page view:
466				let prereg = BotMon.live.data.model._getPageView(visitor, dat);
467				if (prereg) {
468					// update the page view:
469					prereg._lastSeen = dat.ts;
470					if (!prereg._seenBy.includes(type)) prereg._seenBy.push(type);
471					prereg._jsClient = true; // seen by client js
472				} else {
473					// add the page view to the visitor:
474					prereg = model._makePageView(dat, type);
475					visitor._pageViews.push(prereg);
476				}
477				prereg._tickCount += 1;
478			},
479
480			// updating visit data from the ticker log:
481			updateTicks: function(dat) {
482				//console.info('updateTicks', dat);
483
484				// shortcut to make code more readable:
485				const model = BotMon.live.data.model;
486
487				const type = BM_LOGTYPE.TICKER;
488
489				// find the visit info:
490				let visitor = model.findVisitor(dat, type);
491				if (!visitor) {
492					console.info(`No visitor with ID “${dat.id}” found, registering as a new one.`);
493					visitor = model.registerVisit(dat, type);
494				}
495				if (visitor) {
496					// update visitor:
497					if (visitor._lastSeen < dat.ts) visitor._lastSeen = dat.ts;
498					if (!visitor._seenBy.includes(type)) visitor._seenBy.push(type);
499
500					// get the page view info:
501					let pv = model._getPageView(visitor, dat);
502					if (!pv) {
503						console.info(`No page view for visit ID “${dat.id}”, page “${dat.pg}”, registering a new one.`);
504						pv = model._makePageView(dat, type);
505						visitor._pageViews.push(pv);
506					}
507
508					// update the page view info:
509					if (!pv._seenBy.includes(type)) pv._seenBy.push(type);
510					if (pv._lastSeen.getTime() < dat.ts.getTime()) pv._lastSeen = dat.ts;
511					pv._tickCount += 1;
512
513				}
514			},
515
516			// helper function to create a new "page view" item:
517			_makePageView: function(data, type) {
518				// console.info('_makePageView', data);
519
520				// try to parse the referrer:
521				let rUrl = null;
522				try {
523					rUrl = ( data.ref && data.ref !== '' ? new URL(data.ref) : null );
524				} catch (e) {
525					console.warn(`Invalid referer: “${data.ref}”.`);
526					console.info(data);
527				}
528
529				return {
530					_by: type,
531					ip: data.ip,
532					pg: data.pg,
533					lang: data.lang || '??',
534					_ref: rUrl,
535					_firstSeen: data.ts,
536					_lastSeen: data.ts,
537					_seenBy: [type],
538					_jsClient: ( type !== BM_LOGTYPE.SERVER),
539					_viewCount: 0,
540					_loadCount: 0,
541					_tickCount: 0
542				};
543			}
544		},
545
546		// functions to analyse the data:
547		analytics: {
548
549			/**
550			 * Initializes the analytics data storage object:
551			 */
552			init: function() {
553				//console.info('BotMon.live.data.analytics.init()');
554			},
555
556			// data storage:
557			data: {
558				totalVisits: 0,
559				totalPageViews: 0,
560				humanPageViews: 0,
561				bots: {
562					known: 0,
563					suspected: 0,
564					human: 0,
565					users: 0
566				}
567			},
568
569			// sort the visits by type:
570			groups: {
571				knownBots: [],
572				suspectedBots: [],
573				humans: [],
574				users: []
575			},
576
577			// all analytics
578			analyseAll: function() {
579				//console.info('BotMon.live.data.analytics.analyseAll()');
580
581				// shortcut to make code more readable:
582				const model = BotMon.live.data.model;
583				const me = BotMon.live.data.analytics;
584
585				BotMon.live.gui.status.showBusy("Analysing data …");
586
587				// loop over all visitors:
588				model._visitors.forEach( (v) => {
589
590					// count visits and page views:
591					this.data.totalVisits += 1;
592					this.data.totalPageViews += v._viewCount;
593
594					// check for typical bot aspects:
595					let botScore = 0;
596
597					if (v._type == BM_USERTYPE.KNOWN_BOT) { // known bots
598
599						this.data.bots.known += v._viewCount;
600						this.groups.knownBots.push(v);
601
602					} else if (v._type == BM_USERTYPE.KNOWN_USER) { // known users */
603
604						this.data.bots.users += v._viewCount;
605						this.groups.users.push(v);
606
607					} else {
608
609						// get evaluation:
610						const e = BotMon.live.data.rules.evaluate(v);
611						v._eval = e.rules;
612						v._botVal = e.val;
613
614						if (e.isBot) { // likely bots
615							v._type = BM_USERTYPE.LIKELY_BOT;
616							this.data.bots.suspected += v._viewCount;
617							this.groups.suspectedBots.push(v);
618						} else { // probably humans
619							v._type = BM_USERTYPE.PROBABLY_HUMAN;
620							this.data.bots.human += v._viewCount;
621							this.groups.humans.push(v);
622						}
623					}
624
625					// perform actions depending on the visitor type:
626					if (v._type == BM_USERTYPE.KNOWN_BOT ) { /* known bots only */
627
628					} else if (v._type == BM_USERTYPE.LIKELY_BOT) { /* probable bots only */
629
630						// add bot views to IP range information:
631						me.addToIpRanges(v);
632
633					} else { /* humans only */
634
635						// add browser and platform statistics:
636						me.addBrowserPlatform(v);
637
638						// add
639						v._pageViews.forEach( pv => {
640							me.addToRefererList(pv._ref);
641							me.addToPagesList(pv.pg);
642						});
643					}
644
645					// add to the country lists:
646					me.addToCountries(v.geo, v._country, v._type);
647
648				});
649
650				BotMon.live.gui.status.hideBusy('Done.');
651			},
652
653			// get a list of known bots:
654			getTopBots: function(max) {
655				//console.info('BotMon.live.data.analytics.getTopBots('+max+')');
656
657				//console.log(BotMon.live.data.analytics.groups.knownBots);
658
659				let botsList = BotMon.live.data.analytics.groups.knownBots.toSorted( (a, b) => {
660					return b._viewCount - a._viewCount;
661				});
662
663				const other = {
664					'id': 'other',
665					'name': "Others",
666					'count': 0
667				};
668
669				const rList = [];
670				const max2 = ( botsList.length > max ? max-1 : botsList.length );
671				let total = 0; // adding up the items
672				for (let i=0; i<botsList.length; i++) {
673					const it = botsList[i];
674					if (it && it._bot) {
675						if (i < max2) {
676							rList.push({
677								id: it._bot.id,
678								name: (it._bot.n ? it._bot.n : it._bot.id),
679								count: it._viewCount
680							});
681						} else {
682							other.count += it._pageViews.length;
683						};
684						total += it._viewCount;
685					}
686				};
687
688				// add the "other" item, if needed:
689				if (botsList.length > max2) {
690					rList.push(other);
691				};
692
693				rList.forEach( it => {
694					it.pct = (it.count * 100 / total);
695				});
696
697				return rList;
698			},
699
700			// most visited pages list:
701			_pagesList: [],
702
703			/**
704			 * Add a page view to the list of most visited pages.
705			 * @param {string} pageId - The page ID to add to the list.
706			 * @example
707			 * BotMon.live.data.analytics.addToPagesList('1234567890');
708			 */
709			addToPagesList: function(pageId) {
710				//console.log('BotMon.live.data.analytics.addToPagesList', pageId);
711
712				const me = BotMon.live.data.analytics;
713
714				// already exists?
715				let pgObj = null;
716				for (let i = 0; i < me._pagesList.length; i++) {
717					if (me._pagesList[i].id == pageId) {
718						pgObj = me._pagesList[i];
719						break;
720					}
721				}
722
723				// if not exists, create it:
724				if (!pgObj) {
725					pgObj = {
726						id: pageId,
727						count: 1
728					};
729					me._pagesList.push(pgObj);
730				} else {
731					pgObj.count += 1;
732				}
733			},
734
735			getTopPages: function(max) {
736				//console.info('BotMon.live.data.analytics.getTopPages('+max+')');
737				const me = BotMon.live.data.analytics;
738				return me._pagesList.toSorted( (a, b) => {
739					return b.count - a.count;
740				}).slice(0,max);
741			},
742
743			// Referer List:
744			_refererList: [],
745
746			addToRefererList: function(ref) {
747				//console.log('BotMon.live.data.analytics.addToRefererList',ref);
748
749				const me = BotMon.live.data.analytics;
750
751				// ignore internal references:
752				if (ref && ref.host == window.location.host) {
753					return;
754				}
755
756				const refInfo = me.getRefererInfo(ref);
757
758				// already exists?
759				let refObj = null;
760				for (let i = 0; i < me._refererList.length; i++) {
761					if (me._refererList[i].id == refInfo.id) {
762						refObj = me._refererList[i];
763						break;
764					}
765				}
766
767				// if not exists, create it:
768				if (!refObj) {
769					refObj = refInfo;
770					refObj.count = 1;
771					me._refererList.push(refObj);
772				} else {
773					refObj.count += 1;
774				}
775			},
776
777			getRefererInfo: function(url) {
778				//console.log('BotMon.live.data.analytics.getRefererInfo',url);
779				try {
780					url = new URL(url);
781				} catch (e) {
782					return {
783						'id': 'null',
784						'n': 'Invalid Referer'
785					};
786				}
787
788				// find the referer ID:
789				let refId = 'null';
790				let refName = 'No Referer';
791				if (url && url.host) {
792					const hArr = url.host.split('.');
793					const tld = hArr[hArr.length-1];
794					refId = ( tld == 'localhost' ? tld : hArr[hArr.length-2]);
795					refName = hArr[hArr.length-2] + '.' + tld;
796				}
797
798				return {
799					'id': refId,
800					'n': refName
801				};
802			},
803
804			/**
805			 * Get a sorted list of the top referers.
806			 * The list is sorted in descending order of count.
807			 * If the array has more items than the given maximum, the rest of the items are added to an "other" item.
808			 * Each item in the list has a "pct" property, which is the percentage of the total count.
809			 * @param {number} max - The maximum number of items to return.
810			 * @return {Array} The sorted list of top referers.
811			 */
812			getTopReferers: function(max) {
813				//console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')'));
814
815				const me = BotMon.live.data.analytics;
816
817				return me._makeTopList(me._refererList, max);
818			},
819
820			/**
821			 * Create a sorted list of top items from a given array.
822			 * The list is sorted in descending order of count.
823			 * If the array has more items than the given maximum, the rest of the items are added to an "other" item.
824			 * Each item in the list has a "pct" property, which is the percentage of the total count.
825			 * @param {Array} arr - The array to sort and truncate.
826			 * @param {number} max - The maximum number of items to return.
827			 * @return {Array} The sorted list of top items.
828			 */
829			_makeTopList: function(arr, max) {
830				//console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')'));
831
832				const me = BotMon.live.data.analytics;
833
834				// sort the list:
835				arr.sort( (a,b) => {
836					return b.count - a.count;
837				});
838
839				const rList = []; // return array
840				const max2 = ( arr.length >= max ? max-1 : arr.length );
841				const other = {
842					'id': 'other',
843					'name': "Others",
844					'typ': 'other',
845					'count': 0
846				};
847				let total = 0; // adding up the items
848				for (let i=0; Math.min(max, arr.length) > i; i++) {
849					const it = arr[i];
850					if (it) {
851						if (i < max2) {
852							const rIt = {
853								id: it.id,
854								name: (it.n ? it.n : it.id),
855								typ: it.typ || it.id,
856								count: it.count
857							};
858							rList.push(rIt);
859						} else {
860							other.count += it.count;
861						}
862						total += it.count;
863					}
864				}
865
866				// add the "other" item, if needed:
867				if (arr.length > max2) {
868					rList.push(other);
869				};
870
871				rList.forEach( it => {
872					it.pct = (it.count * 100 / total);
873				});
874
875				return rList;
876			},
877
878			/* countries of visits */
879			_countries: {
880				'human': [],
881				'bot': []
882			},
883			/**
884			 * Adds a country code to the statistics.
885			 *
886			 * @param {string} iso The ISO 3166-1 alpha-2 country code.
887			 */
888			addToCountries: function(iso, name, type) {
889
890				const me = BotMon.live.data.analytics;
891
892				// find the correct array:
893				let arr = null;
894				switch (type) {
895
896					case BM_USERTYPE.KNOWN_USER:
897					case BM_USERTYPE.PROBABLY_HUMAN:
898						arr = me._countries.human;
899						break;
900					case BM_USERTYPE.LIKELY_BOT:
901					case BM_USERTYPE.KNOWN_BOT:
902						arr = me._countries.bot;
903						break;
904					default:
905						console.warn(`Unknown user type ${type} in function addToCountries.`);
906				}
907
908				if (arr) {
909					let cRec = arr.find( it => it.id == iso);
910					if (!cRec) {
911						cRec = {
912							'id': iso,
913							'n': name,
914							'count': 1
915						};
916						arr.push(cRec);
917					} else {
918						cRec.count += 1;
919					}
920				}
921			},
922
923			/**
924			 * Returns a list of countries with visit counts, sorted by visit count in descending order.
925			 *
926			 * @param {BM_USERTYPE} type array of types type of visitors to return.
927			 * @param {number} max The maximum number of entries to return.
928			 * @return {Array} A list of objects with properties 'iso' (ISO 3166-1 alpha-2 country code) and 'count' (visit count).
929			 */
930			getCountryList: function(type, max) {
931
932				const me = BotMon.live.data.analytics;
933
934				// find the correct array:
935				let arr = null;
936				switch (type) {
937
938					case 'human':
939						arr = me._countries.human;
940						break;
941					case 'bot':
942						arr = me._countries.bot;
943						break;
944					default:
945						console.warn(`Unknown user type ${type} in function getCountryList.`);
946						return;
947				}
948
949				return me._makeTopList(arr, max);
950			},
951
952			/* browser and platform of human visitors */
953			_browsers: [],
954			_platforms: [],
955
956			addBrowserPlatform: function(visitor) {
957				//console.info('addBrowserPlatform', visitor);
958
959				const me = BotMon.live.data.analytics;
960
961				// add to browsers list:
962				let browserRec = ( visitor._client ? visitor._client : {'id': 'unknown'});
963				if (visitor._client) {
964					let bRec = me._browsers.find( it => it.id == browserRec.id);
965					if (!bRec) {
966						bRec = {
967							id: browserRec.id,
968							n: browserRec.n,
969							count: 1
970						};
971						me._browsers.push(bRec);
972					} else {
973						bRec.count += 1;
974					}
975				}
976
977				// add to platforms list:
978				let platformRec = ( visitor._platform ? visitor._platform : {'id': 'unknown'});
979				if (visitor._platform) {
980					let pRec = me._platforms.find( it => it.id == platformRec.id);
981					if (!pRec) {
982						pRec = {
983							id: platformRec.id,
984							n: platformRec.n,
985							count: 1
986						};
987						me._platforms.push(pRec);
988					} else {
989						pRec.count += 1;
990					}
991				}
992
993			},
994
995			getTopBrowsers: function(max) {
996
997				const me = BotMon.live.data.analytics;
998
999				return me._makeTopList(me._browsers, max);
1000			},
1001
1002			getTopPlatforms: function(max) {
1003
1004				const me = BotMon.live.data.analytics;
1005
1006				return me._makeTopList(me._platforms, max);
1007			},
1008
1009			/* bounces are counted, not calculates: */
1010			getBounceCount: function(type) {
1011
1012				const me = BotMon.live.data.analytics;
1013				var bounces = 0;
1014				const list = me.groups[type];
1015
1016				list.forEach(it => {
1017					bounces += (it._viewCount <= 1 ? 1 : 0);
1018				});
1019
1020				return bounces;
1021			},
1022
1023			_ipRanges: [],
1024
1025			/* adds a visit to the ip ranges arrays */
1026			addToIpRanges: function(v) {
1027				//console.info('addToIpRanges', v.ip);
1028
1029				const me = BotMon.live.data.analytics;
1030				const ipRanges = BotMon.live.data.ipRanges;
1031
1032				// Number of IP address segments to look at:
1033				const kIP4Segments = 1;
1034				const kIP6Segments = 3;
1035
1036				let ipGroup = ''; // group name
1037				let ipSeg = []; // IP segment array
1038				let rawIP = ''; // raw ip prefix
1039				let ipName = ''; // IP group display name
1040
1041				const ipType = v.ip.indexOf(':') > 0 ? BM_IPVERSION.IPv6 : BM_IPVERSION.IPv4;
1042				const ipAddr = BotMon.t._ip2Num(v.ip);
1043
1044				// is there already a known IP range assigned?
1045				if (v._ipRange) {
1046
1047					ipGroup = v._ipRange.g; // group name
1048					ipName = ipRanges.getOwner( v._ipRange.g ) || "Unknown";
1049
1050				} else { // no known IP range, let's collect necessary information:
1051
1052
1053					// collect basic IP address info:
1054					if (ipType == BM_IPVERSION.IPv6) {
1055						ipSeg = ipAddr.split(':');
1056						const prefix = v.ip.split(':').slice(0, kIP6Segments).join(':');
1057						rawIP = ipSeg.slice(0, kIP6Segments).join(':');
1058						ipGroup = 'ip6-' + rawIP.replaceAll(':', '-');
1059						ipName = prefix + '::'; // + '/' + (16 * kIP6Segments);
1060					} else {
1061						ipSeg = ipAddr.split('.');
1062						const prefix = v.ip.split('.').slice(0, kIP4Segments).join('.');
1063						rawIP = ipSeg.slice(0, kIP4Segments).join('.') ;
1064						ipGroup = 'ip4-' + rawIP.replaceAll('.', '-');
1065						ipName = prefix + '.x.x.x'.substring(0, 1+(4-kIP4Segments)*2); // + '/' + (8 * kIP4Segments);
1066					}
1067				}
1068
1069				// check if record already exists:
1070				let ipRec = me._ipRanges.find( it => it.g == ipGroup);
1071				if (!ipRec) {
1072
1073					// ip info record initialised:
1074					ipRec = {
1075						g: ipGroup,
1076						n: ipName,
1077						count: 0
1078					}
1079
1080					// existing record?
1081					if (v._ipRange) {
1082
1083						ipRec.from = v._ipRange.from;
1084						ipRec.to = v._ipRange.to;
1085						ipRec.typ = 'net';
1086
1087					} else { // no known IP range, let's collect necessary information:
1088
1089						// complete the ip info record:
1090						if (ipType == BM_IPVERSION.IPv6) {
1091							ipRec.from = rawIP + ':0000:0000:0000:0000:0000:0000:0000'.substring(0, (8-kIP6Segments)*5);
1092							ipRec.to = rawIP + ':FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF'.substring(0, (8-kIP6Segments)*5);
1093							ipRec.typ = '6';
1094						} else {
1095							ipRec.from = rawIP + '.000.000.000.000'.substring(0, (4-kIP4Segments)*4);
1096							ipRec.to = rawIP + '.255.255.255.254'.substring(0, (4-kIP4Segments)*4);
1097							ipRec.typ = '4';
1098						}
1099					}
1100
1101					me._ipRanges.push(ipRec);
1102				}
1103
1104				// add to counter:
1105				ipRec.count += v._viewCount;
1106
1107			},
1108
1109			getTopBotISPs: function(max) {
1110
1111				const me = BotMon.live.data.analytics;
1112
1113				return me._makeTopList(me._ipRanges, max);
1114			},
1115		},
1116
1117		// information on "known bots":
1118		bots: {
1119			// loads the list of known bots from a JSON file:
1120			init: async function() {
1121				//console.info('BotMon.live.data.bots.init()');
1122
1123				// Load the list of known bots:
1124				BotMon.live.gui.status.showBusy("Loading known bots …");
1125				const url = BotMon._baseDir + 'config/known-bots.json';
1126				try {
1127					const response = await fetch(url);
1128					if (!response.ok) {
1129						throw new Error(`${response.status} ${response.statusText}`);
1130					}
1131
1132					this._list = await response.json();
1133					this._ready = true;
1134
1135				} catch (error) {
1136					BotMon.live.gui.status.setError("Error while loading the known bots file:", error.message);
1137				} finally {
1138					BotMon.live.gui.status.hideBusy("Status: Done.");
1139					BotMon.live.data._dispatch('bots')
1140				}
1141			},
1142
1143			// returns bot info if the clientId matches a known bot, null otherwise:
1144			match: function(agent) {
1145				//console.info('BotMon.live.data.bots.match(',agent,')');
1146
1147				const BotList = BotMon.live.data.bots._list;
1148
1149				// default is: not found!
1150				let botInfo = null;
1151
1152				if (!agent) return null;
1153
1154				// check for known bots:
1155				BotList.find(bot => {
1156					let r = false;
1157					for (let j=0; j<bot.rx.length; j++) {
1158						const rxr = agent.match(new RegExp(bot.rx[j]));
1159						if (rxr) {
1160							botInfo = {
1161								n : bot.n,
1162								id: bot.id,
1163								geo: (bot.geo ? bot.geo : null),
1164								url: bot.url,
1165								v: (rxr.length > 1 ? rxr[1] : -1)
1166							};
1167							r = true;
1168							break;
1169						};
1170					};
1171					return r;
1172				});
1173
1174				// check for unknown bots:
1175				if (!botInfo) {
1176					const botmatch = agent.match(/([\s\d\w\-]*bot|[\s\d\w\-]*crawler|[\s\d\w\-]*spider)[\/\s\w\-;\),\\.$]/i);
1177					if(botmatch) {
1178						botInfo = {'id': ( botmatch[1] || "other_" ), 'n': "Other" + ( botmatch[1] ? " (" + botmatch[1] + ")" : "" ) , "bot": botmatch[1] };
1179					}
1180				}
1181
1182				//console.log("botInfo:", botInfo);
1183				return botInfo;
1184			},
1185
1186
1187			// indicates if the list is loaded and ready to use:
1188			_ready: false,
1189
1190			// the actual bot list is stored here:
1191			_list: []
1192		},
1193
1194		// information on known clients (browsers):
1195		clients: {
1196			// loads the list of known clients from a JSON file:
1197			init: async function() {
1198				//console.info('BotMon.live.data.clients.init()');
1199
1200				// Load the list of known bots:
1201				BotMon.live.gui.status.showBusy("Loading known clients");
1202				const url = BotMon._baseDir + 'config/known-clients.json';
1203				try {
1204					const response = await fetch(url);
1205					if (!response.ok) {
1206						throw new Error(`${response.status} ${response.statusText}`);
1207					}
1208
1209					BotMon.live.data.clients._list = await response.json();
1210					BotMon.live.data.clients._ready = true;
1211
1212				} catch (error) {
1213					BotMon.live.gui.status.setError("Error while loading the known clients file: " + error.message);
1214				} finally {
1215					BotMon.live.gui.status.hideBusy("Status: Done.");
1216					BotMon.live.data._dispatch('clients')
1217				}
1218			},
1219
1220			// returns bot info if the user-agent matches a known bot, null otherwise:
1221			match: function(agent) {
1222				//console.info('BotMon.live.data.clients.match(',agent,')');
1223
1224				let match = {"n": "Unknown", "v": -1, "id": 'null'};
1225
1226				if (agent) {
1227					BotMon.live.data.clients._list.find(client => {
1228						let r = false;
1229						for (let j=0; j<client.rx.length; j++) {
1230							const rxr = agent.match(new RegExp(client.rx[j]));
1231							if (rxr) {
1232								match.n = client.n;
1233								match.v = (rxr.length > 1 ? rxr[1] : -1);
1234								match.id = client.id || null;
1235								r = true;
1236								break;
1237							}
1238						}
1239						return r;
1240					});
1241				}
1242
1243				//console.log(match)
1244				return match;
1245			},
1246
1247			// return the browser name for a browser ID:
1248			getName: function(id) {
1249				const it = BotMon.live.data.clients._list.find(client => client.id == id);
1250				return ( it && it.n ? it.n : "Unknown"); //it.n;
1251			},
1252
1253			// indicates if the list is loaded and ready to use:
1254			_ready: false,
1255
1256			// the actual bot list is stored here:
1257			_list: []
1258
1259		},
1260
1261		// information on known platforms (operating systems):
1262		platforms: {
1263			// loads the list of known platforms from a JSON file:
1264			init: async function() {
1265				//console.info('BotMon.live.data.platforms.init()');
1266
1267				// Load the list of known bots:
1268				BotMon.live.gui.status.showBusy("Loading known platforms");
1269				const url = BotMon._baseDir + 'config/known-platforms.json';
1270				try {
1271					const response = await fetch(url);
1272					if (!response.ok) {
1273						throw new Error(`${response.status} ${response.statusText}`);
1274					}
1275
1276					BotMon.live.data.platforms._list = await response.json();
1277					BotMon.live.data.platforms._ready = true;
1278
1279				} catch (error) {
1280					BotMon.live.gui.status.setError("Error while loading the known platforms file: " + error.message);
1281				} finally {
1282					BotMon.live.gui.status.hideBusy("Status: Done.");
1283					BotMon.live.data._dispatch('platforms')
1284				}
1285			},
1286
1287			// returns bot info if the browser id matches a known platform:
1288			match: function(cid) {
1289				//console.info('BotMon.live.data.platforms.match(',cid,')');
1290
1291				let match = {"n": "Unknown", "id": 'null'};
1292
1293				if (cid) {
1294					BotMon.live.data.platforms._list.find(platform => {
1295						let r = false;
1296						for (let j=0; j<platform.rx.length; j++) {
1297							const rxr = cid.match(new RegExp(platform.rx[j]));
1298							if (rxr) {
1299								match.n = platform.n;
1300								match.v = (rxr.length > 1 ? rxr[1] : -1);
1301								match.id = platform.id || null;
1302								r = true;
1303								break;
1304							}
1305						}
1306						return r;
1307					});
1308				}
1309
1310				return match;
1311			},
1312
1313			// return the platform name for a given ID:
1314			getName: function(id) {
1315				const it = BotMon.live.data.platforms._list.find( pf => pf.id == id);
1316				return ( it ? it.n : 'Unknown' );
1317			},
1318
1319
1320			// indicates if the list is loaded and ready to use:
1321			_ready: false,
1322
1323			// the actual bot list is stored here:
1324			_list: []
1325
1326		},
1327
1328		// storage and functions for the known bot IP-Ranges:
1329		ipRanges: {
1330
1331			init: function() {
1332				//console.log('BotMon.live.data.ipRanges.init()');
1333				// #TODO: Load from separate IP-Ranges file
1334				// load the rules file:
1335				const me = BotMon.live.data;
1336
1337				try {
1338					BotMon.live.data._loadSettingsFile(['user-ipranges', 'known-ipranges'],
1339						(json) => {
1340
1341						// groups can be just saved in the data structure:
1342						if (json.groups && json.groups.constructor.name == 'Array') {
1343							me.ipRanges._groups = json.groups;
1344						}
1345
1346						// groups can be just saved in the data structure:
1347						if (json.ranges && json.ranges.constructor.name == 'Array') {
1348							json.ranges.forEach(range => {
1349								me.ipRanges.add(range);
1350							})
1351						}
1352
1353						// finished loading
1354						BotMon.live.gui.status.hideBusy("Status: Done.");
1355						BotMon.live.data._dispatch('ipranges')
1356					});
1357				} catch (error) {
1358					BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
1359				}
1360			},
1361
1362			// the actual bot list is stored here:
1363			_list: [],
1364			_groups: [],
1365
1366			add: function(data) {
1367				//console.log('BotMon.live.data.ipRanges.add(',data,')');
1368
1369				const me = BotMon.live.data.ipRanges;
1370
1371				// convert IP address to easier comparable form:
1372				const ip2Num = BotMon.t._ip2Num;
1373
1374				let item = {
1375					'cidr': data.from.replaceAll(/::+/g, '::') + '/' + ( data.m ? data.m : '??' ),
1376					'from': ip2Num(data.from),
1377					'to': ip2Num(data.to),
1378					'm': data.m,
1379					'g': data.g
1380				};
1381				me._list.push(item);
1382
1383			},
1384
1385			getOwner: function(gid) {
1386
1387				const me = BotMon.live.data.ipRanges;
1388
1389				for (let i=0; i < me._groups.length; i++) {
1390					const it = me._groups[i];
1391					if (it.id == gid) {
1392						return it.name;
1393					}
1394				}
1395				return null;
1396			},
1397
1398			match: function(ip) {
1399				//console.log('BotMon.live.data.ipRanges.match(',ip,')');
1400
1401				const me = BotMon.live.data.ipRanges;
1402
1403				// convert IP address to easier comparable form:
1404				const ipNum = BotMon.t._ip2Num(ip);
1405
1406				for (let i=0; i < me._list.length; i++) {
1407					const ipRange = me._list[i];
1408
1409					if (ipNum >= ipRange.from && ipNum <= ipRange.to) {
1410						return ipRange;
1411					}
1412
1413				};
1414				return null;
1415			}
1416		},
1417
1418		// storage for the rules and related functions
1419		rules: {
1420
1421			/**
1422			 * Initializes the rules data.
1423			 *
1424			 * Loads the default config file and the user config file (if present).
1425			 * The default config file is used if the user config file does not have a certain setting.
1426			 * The user config file can override settings from the default config file.
1427			 *
1428			 * The rules are loaded from the `rules` property of the config files.
1429			 * The IP ranges are loaded from the `ipRanges` property of the config files.
1430			 *
1431			 * If an error occurs while loading the config file, it is displayed in the status bar.
1432			 * After the config file is loaded, the status bar is hidden.
1433			 */
1434			init: async function() {
1435				//console.info('BotMon.live.data.rules.init()');
1436
1437				// Load the list of known bots:
1438				BotMon.live.gui.status.showBusy("Loading list of rules …");
1439
1440				// load the rules file:
1441				const me = BotMon.live.data;
1442
1443				try {
1444					BotMon.live.data._loadSettingsFile(['user-config', 'default-config'],
1445						(json) => {
1446
1447							// override the threshold?
1448							if (json.threshold) me._threshold = json.threshold;
1449
1450							// set the rules list:
1451							if (json.rules && json.rules.constructor.name == 'Array') {
1452								me.rules._rulesList = json.rules;
1453							}
1454
1455							BotMon.live.gui.status.hideBusy("Status: Done.");
1456							BotMon.live.data._dispatch('rules')
1457						}
1458					);
1459				} catch (error) {
1460					BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
1461				}
1462			},
1463
1464			_rulesList: [], // list of rules to find out if a visitor is a bot
1465			_threshold: 100, // above this, it is considered a bot.
1466
1467			// returns a descriptive text for a rule id
1468			getRuleInfo: function(ruleId) {
1469				// console.info('getRuleInfo', ruleId);
1470
1471				// shortcut for neater code:
1472				const me = BotMon.live.data.rules;
1473
1474				for (let i=0; i<me._rulesList.length; i++) {
1475					const rule = me._rulesList[i];
1476					if (rule.id == ruleId) {
1477						return rule;
1478					}
1479				}
1480				return null;
1481
1482			},
1483
1484			// evaluate a visitor for lkikelihood of being a bot
1485			evaluate: function(visitor) {
1486
1487				// shortcut for neater code:
1488				const me = BotMon.live.data.rules;
1489
1490				let r =  {	// evaluation result
1491					'val': 0,
1492					'rules': [],
1493					'isBot': false
1494				};
1495
1496				for (let i=0; i<me._rulesList.length; i++) {
1497					const rule = me._rulesList[i];
1498					const params = ( rule.params ? rule.params : [] );
1499
1500					if (rule.func) { // rule is calling a function
1501						if (me.func[rule.func]) {
1502							if(me.func[rule.func](visitor, ...params)) {
1503								r.val += rule.bot;
1504								r.rules.push(rule.id)
1505							}
1506						} else {
1507							//console.warn("Unknown rule function: “${rule.func}”. Ignoring rule.")
1508						}
1509					}
1510				}
1511
1512				// is a bot?
1513				r.isBot = (r.val >= me._threshold);
1514
1515				return r;
1516			},
1517
1518			// list of functions that can be called by the rules list to evaluate a visitor:
1519			func: {
1520
1521				// check if client is on the list passed as parameter:
1522				matchesClient: function(visitor, ...clients) {
1523
1524					const clientId = ( visitor._client ? visitor._client.id : '');
1525					return clients.includes(clientId);
1526				},
1527
1528				// check if OS/Platform is one of the obsolete ones:
1529				matchesPlatform: function(visitor, ...platforms) {
1530
1531					const pId = ( visitor._platform ? visitor._platform.id : '');
1532
1533					if (visitor._platform.id == null) console.log(visitor._platform);
1534
1535					return platforms.includes(pId);
1536				},
1537
1538				// are there at lest num pages loaded?
1539				smallPageCount: function(visitor, num) {
1540					return (visitor._viewCount <= Number(num));
1541				},
1542
1543				// There was no entry in a specific log file for this visitor:
1544				// note that this will also trigger the "noJavaScript" rule:
1545				noRecord: function(visitor, type) {
1546					if (!visitor._seenBy.includes('srv')) return false; // only if 'srv' is also specified!
1547					return !visitor._seenBy.includes(type);
1548				},
1549
1550				// there are no referrers in any of the page visits:
1551				noReferrer: function(visitor) {
1552
1553					let r = false; // return value
1554					for (let i = 0; i < visitor._pageViews.length; i++) {
1555						if (!visitor._pageViews[i]._ref) {
1556							r = true;
1557							break;
1558						}
1559					}
1560					return r;
1561				},
1562
1563				// test for specific client identifiers:
1564				matchesUserAgent: function(visitor, ...list) {
1565
1566					for (let i=0; i<list.length; i++) {
1567						if (visitor.agent == list[i]) {
1568							return true
1569						}
1570					};
1571					return false;
1572				},
1573
1574				// unusual combinations of Platform and Client:
1575				combinationTest: function(visitor, ...combinations) {
1576
1577					for (let i=0; i<combinations.length; i++) {
1578
1579						if (visitor._platform.id == combinations[i][0]
1580							&& visitor._client.id == combinations[i][1]) {
1581								return true
1582						}
1583					};
1584
1585					return false;
1586				},
1587
1588				// is the IP address from a known bot network?
1589				fromKnownBotIP: function(visitor) {
1590					//console.info('fromKnownBotIP()', visitor.ip);
1591
1592					const ipInfo = BotMon.live.data.ipRanges.match(visitor.ip);
1593
1594					if (ipInfo) {
1595						visitor._ipInKnownBotRange = true;
1596						visitor._ipRange = ipInfo;
1597					}
1598
1599					return (ipInfo !== null);
1600				},
1601
1602				// is the page language mentioned in the client's accepted languages?
1603				// the parameter holds an array of exceptions, i.e. page languages that should be ignored.
1604				matchLang: function(visitor, ...exceptions) {
1605
1606					if (visitor.lang && visitor.accept && exceptions.indexOf(visitor.lang) < 0) {
1607						return (visitor.accept.split(',').indexOf(visitor.lang) < 0);
1608					}
1609					return false;
1610				},
1611
1612				// the "Accept language" header contains certain entries:
1613				clientAccepts: function(visitor, ...languages) {
1614					//console.info('clientAccepts', visitor.accept, languages);
1615
1616					if (visitor.accept && languages) {;
1617						return ( visitor.accept.split(',').filter(lang => languages.includes(lang)).length > 0 );
1618					}
1619					return false;
1620				},
1621
1622				// Is there an accept-language field defined at all?
1623				noAcceptLang: function(visitor) {
1624
1625					if (!visitor.accept || visitor.accept.length <= 0) { // no accept-languages header
1626						return true;
1627					}
1628					// TODO: parametrize this!
1629					return false;
1630				},
1631				// At least x page views were recorded, but they come within less than y seconds
1632				loadSpeed: function(visitor, minItems, maxTime) {
1633
1634					if (visitor._viewCount >= minItems) {
1635						//console.log('loadSpeed', visitor._pageViews.length, minItems, maxTime);
1636
1637						const pvArr = visitor._pageViews.map(pv => pv._lastSeen).sort();
1638
1639						let totalTime = 0;
1640						for (let i=1; i < pvArr.length; i++) {
1641							totalTime += (pvArr[i] - pvArr[i-1]);
1642						}
1643
1644						//console.log('     ', totalTime , Math.round(totalTime / (pvArr.length * 1000)), (( totalTime / pvArr.length ) <= maxTime * 1000), visitor.ip);
1645
1646						return (( totalTime / pvArr.length ) <= maxTime * 1000);
1647					}
1648				},
1649
1650				// Country code matches one of those in the list:
1651				matchesCountry: function(visitor, ...countries) {
1652
1653					// ingore if geoloc is not set or unknown:
1654					if (visitor.geo) {
1655						return (countries.indexOf(visitor.geo) >= 0);
1656					}
1657					return false;
1658				},
1659
1660				// Country does not match one of the given codes.
1661				notFromCountry: function(visitor, ...countries) {
1662
1663					// ingore if geoloc is not set or unknown:
1664					if (visitor.geo && visitor.geo !== 'ZZ') {
1665						return (countries.indexOf(visitor.geo) < 0);
1666					}
1667					return false;
1668				}
1669			}
1670		},
1671
1672		/**
1673		 * Loads a settings file from the specified list of filenames.
1674		 * If the file is successfully loaded, it will call the callback function
1675		 * with the loaded JSON data.
1676		 * If no file can be loaded, it will display an error message.
1677		 *
1678		 * @param {string[]} fns - list of filenames to load
1679		 * @param {function} callback - function to call with the loaded JSON data
1680		 */
1681		_loadSettingsFile: async function(fns, callback) {
1682			//console.info('BotMon.live.data._loadSettingsFile()', fns);
1683
1684			const kJsonExt = '.json';
1685			let loaded = false; // if successfully loaded file
1686
1687			for (let i=0; i<fns.length; i++) {
1688				const filename = fns[i] +kJsonExt;
1689				try {
1690					const response = await fetch(DOKU_BASE + 'lib/plugins/botmon/config/' + filename);
1691					if (!response.ok) {
1692						continue;
1693					} else {
1694						loaded = true;
1695					}
1696					const json = await response.json();
1697					if (callback && typeof callback === 'function') {
1698						callback(json);
1699					}
1700					break;
1701				} catch (e) {
1702					BotMon.live.gui.status.setError("Error while loading the config file: " + filename);
1703				}
1704			}
1705
1706			if (!loaded) {
1707				BotMon.live.gui.status.setError("Could not load a config file.");
1708			}
1709		},
1710
1711		/**
1712		 * Loads a log file (server, page load, or ticker) and parses it.
1713		 * @param {String} type - the type of the log file to load (srv, log, or tck)
1714		 * @param {Function} [onLoaded] - an optional callback function to call after loading is finished.
1715		 */
1716		loadLogFile: async function(type, onLoaded = undefined) {
1717			//console.info('BotMon.live.data.loadLogFile(',type,')');
1718
1719			let typeName = '';
1720			let columns = [];
1721
1722			switch (type) {
1723				case "srv":
1724					typeName = "Server";
1725					columns = ['ts','ip','pg','id','typ','usr','agent','ref','lang','accept','geo','captcha'];
1726					break;
1727				case "log":
1728					typeName = "Page load";
1729					columns = ['ts','ip','pg','id','usr','lt','ref','agent'];
1730					break;
1731				case "tck":
1732					typeName = "Ticker";
1733					columns = ['ts','ip','pg','id','agent'];
1734					break;
1735				default:
1736					console.warn(`Unknown log type ${type}.`);
1737					return;
1738			}
1739
1740			// Show the busy indicator and set the visible status:
1741			BotMon.live.gui.status.showBusy(`Loading ${typeName} log file …`);
1742
1743			// compose the URL from which to load:
1744			const url = BotMon._baseDir + `logs/${BotMon._datestr}.${type}.txt`;
1745			//console.log("Loading:",url);
1746
1747			// fetch the data:
1748			try {
1749				const response = await fetch(url);
1750				if (!response.ok) {
1751
1752					throw new Error(`${response.status} ${response.statusText}`);
1753
1754				} else {
1755
1756					// parse the data:
1757					const logtxt = await response.text();
1758					if (logtxt.length <= 0) {
1759						throw new Error(`Empty log file ${url}.`);
1760					}
1761
1762					logtxt.split('\n').forEach((line) => {
1763						if (line.trim() === '') return; // skip empty lines
1764						const cols = line.split('\t');
1765
1766						// assign the columns to an object:
1767						const data = {};
1768						cols.forEach( (colVal,i) => {
1769							const colName = columns[i] || `col${i}`;
1770							const colValue = (colName == 'ts' ? new Date(colVal) : colVal.trim());
1771							data[colName] = colValue;
1772						});
1773
1774						// register the visit in the model:
1775						switch(type) {
1776							case BM_LOGTYPE.SERVER:
1777								BotMon.live.data.model.registerVisit(data, type);
1778								break;
1779							case BM_LOGTYPE.CLIENT:
1780								data.typ = 'js';
1781								BotMon.live.data.model.updateVisit(data);
1782								break;
1783							case BM_LOGTYPE.TICKER:
1784								data.typ = 'js';
1785								BotMon.live.data.model.updateTicks(data);
1786								break;
1787							default:
1788								console.warn(`Unknown log type ${type}.`);
1789								return;
1790						}
1791					});
1792				}
1793
1794			} catch (error) {
1795				BotMon.live.gui.status.setError(`Error while loading the ${typeName} log file: ${error.message} – data may be incomplete.`);
1796			} finally {
1797				BotMon.live.gui.status.hideBusy("Status: Done.");
1798				if (onLoaded) {
1799					onLoaded(); // callback after loading is finished.
1800				}
1801			}
1802		}
1803	},
1804
1805	gui: {
1806		init: function() {
1807			//console.log('BotMon.live.gui.init()');
1808
1809			// init sub-objects:
1810			BotMon.t._callInit(this);
1811		},
1812
1813		tabs: {
1814			init: function() {
1815				//console.log('BotMon.live.gui.tabs.init()');
1816
1817				/* find and add all existing tabs */
1818				document.querySelectorAll('#botmon__admin *[role=tablist]')
1819					.forEach((tablist) => {
1820						tablist.querySelectorAll('*[role=tab]')
1821					.forEach( t => t.addEventListener('click', this._onTabClick) )
1822					});
1823			},
1824
1825			/* callback for tab click */
1826			_onTabClick: function(e) {
1827				//console.log('BotMon.live.gui.tabs._onTabClick()');
1828
1829				/* reusable constants: */
1830				const kAriaSelected = 'aria-selected';
1831				const kAriaControls = 'aria-controls';
1832				const kTrue = 'true';
1833				const kFalse = 'false';
1834				const kHidden = 'hidden';
1835
1836				/* cancel default action */
1837				e.preventDefault();
1838
1839				/* if the active tab is clicked, do nothing: */
1840				let selState = this.getAttribute(kAriaSelected);
1841				if ( selState && selState == kTrue ) {
1842					return;
1843				}
1844
1845				/* find the active tab element: */
1846				var aItem = null;
1847				let tablist = this.parentNode;
1848				while (tablist.getAttribute('role') !== 'tablist') {
1849					tablist = tablist.parentNode;
1850				}
1851
1852				if (tablist.getAttribute('role') == 'tablist') {
1853					let lis = tablist.querySelectorAll('*[role=tab]');
1854					lis.forEach( (it) => {
1855						let selected = it.getAttribute(kAriaSelected);
1856						if ( selected && selected == kTrue ) {
1857							aItem = it;
1858						}
1859					});
1860				}
1861
1862				/* swap the active states: */
1863				this.setAttribute(kAriaSelected, kTrue);
1864				if (aItem) {
1865					aItem.setAttribute(kAriaSelected, kFalse);
1866					let aId = aItem.getAttribute(kAriaControls);
1867					let aObj = document.getElementById(aId);
1868					if (aObj) aObj.hidden = true;
1869				}
1870
1871				/* show the new panel: */
1872				let nId = this.getAttribute(kAriaControls);
1873				let nObj = document.getElementById(nId);
1874				if (nObj) nObj.hidden = false;
1875			}
1876		},
1877
1878		/* The Overview / web metrics section of the live tab */
1879		overview: {
1880			/**
1881			 * Populates the overview part of the today tab with the analytics data.
1882			 *
1883			 * @method make
1884			 * @memberof BotMon.live.gui.overview
1885			 */
1886			make: function() {
1887
1888				const data = BotMon.live.data.analytics.data;
1889
1890				const maxItemsPerList = 5; // how many list items to show?
1891
1892				const kNoData = '–'; // shown when data is missing
1893
1894				// shortcut for neater code:
1895				const makeElement = BotMon.t._makeElement;
1896
1897				const botsVsHumans = document.getElementById('botmon__today__botsvshumans');
1898				if (botsVsHumans) {
1899					botsVsHumans.appendChild(makeElement('dt', {}, "Page views"));
1900
1901					for (let i = 0; i <= 5; i++) {
1902						const dd = makeElement('dd');
1903						let title = '';
1904						let value = '';
1905						switch(i) {
1906							case 0:
1907								title = "Known bots:";
1908								value = data.bots.known || kNoData;
1909								break;
1910							case 1:
1911								title = "Suspected bots:";
1912								value = data.bots.suspected || kNoData;
1913								break;
1914							case 2:
1915								title = "Probably humans:";
1916								value = data.bots.human || kNoData;
1917								break;
1918							case 3:
1919								title = "Registered users:";
1920								value = data.bots.users || kNoData;
1921								break;
1922							case 4:
1923								title = "Total:";
1924								value = data.totalPageViews || kNoData;
1925								break;
1926							case 5:
1927								title = "Bots-humans ratio:";
1928								value = BotMon.t._getRatio(data.bots.suspected + data.bots.known, data.bots.users + data.bots.human, 100);
1929								break;
1930							default:
1931								console.warn(`Unknown list type ${i}.`);
1932						}
1933						dd.appendChild(makeElement('span', {}, title));
1934						dd.appendChild(makeElement('strong', {}, value));
1935						botsVsHumans.appendChild(dd);
1936					}
1937				}
1938
1939				// update known bots list:
1940				const botElement = document.getElementById('botmon__botslist'); /* Known bots */
1941				if (botElement) {
1942					botElement.appendChild(makeElement('dt', {}, `Top known bots`));
1943
1944					let botList = BotMon.live.data.analytics.getTopBots(maxItemsPerList);
1945					botList.forEach( (botInfo) => {
1946						const bli = makeElement('dd');
1947						bli.appendChild(makeElement('span', {'class': 'has_icon bot bot_' + botInfo.id }, botInfo.name));
1948						bli.appendChild(makeElement('span', {'class': 'count' }, botInfo.count));
1949						botElement.append(bli)
1950					});
1951				}
1952
1953				// update the suspected bot IP ranges list:
1954				const botIps = document.getElementById('botmon__botips');
1955				if (botIps) {
1956					botIps.appendChild(makeElement('dt', {}, "Top bot Networks"));
1957
1958					const ispList = BotMon.live.data.analytics.getTopBotISPs(5);
1959					//console.log(ispList);
1960					ispList.forEach( (netInfo) => {
1961						const li = makeElement('dd');
1962						li.appendChild(makeElement('span', {'class': 'has_icon ipaddr ip' + netInfo.typ }, netInfo.name));
1963						li.appendChild(makeElement('span', {'class': 'count' }, netInfo.count));
1964						botIps.append(li)
1965					});
1966				}
1967
1968				// update the top bot countries list:
1969				const botCountries = document.getElementById('botmon__botcountries');
1970				if (botCountries) {
1971					botCountries.appendChild(makeElement('dt', {}, `Top bot Countries`));
1972					const countryList = BotMon.live.data.analytics.getCountryList('bot', 5);
1973					countryList.forEach( (cInfo) => {
1974						const cLi = makeElement('dd');
1975						cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name));
1976						cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count));
1977						botCountries.appendChild(cLi);
1978					});
1979				}
1980
1981				// update the webmetrics overview:
1982				const wmoverview = document.getElementById('botmon__today__wm_overview');
1983				if (wmoverview) {
1984
1985					const humanVisits = BotMon.live.data.analytics.groups.users.length + BotMon.live.data.analytics.groups.humans.length;
1986					const bounceRate = Math.round(100 * (BotMon.live.data.analytics.getBounceCount('users') + BotMon.live.data.analytics.getBounceCount('humans')) / humanVisits);
1987
1988					wmoverview.appendChild(makeElement('dt', {}, "Humans’ metrics"));
1989					for (let i = 0; i <= 4; i++) {
1990						const dd = makeElement('dd');
1991						let title = '';
1992						let value = '';
1993						switch(i) {
1994							case 0:
1995								title = "Registered users’ page views:";
1996								value = data.bots.users || kNoData;
1997								break;
1998							case 1:
1999								title = "“Probably humans” page views:";
2000								value = data.bots.human || kNoData;
2001								break;
2002							case 2:
2003								title = "Total human page views:";
2004								value = (data.bots.users + data.bots.human) || kNoData;
2005								break;
2006							case 3:
2007								title = "Total human visits:";
2008								value = humanVisits || kNoData;
2009								break;
2010							case 4:
2011								title = "Humans’ bounce rate:";
2012								value = bounceRate + '%';
2013								break;
2014							default:
2015								console.warn(`Unknown list type ${i}.`);
2016						}
2017						dd.appendChild(makeElement('span', {}, title));
2018						dd.appendChild(makeElement('strong', {}, value));
2019						wmoverview.appendChild(dd);
2020					}
2021				}
2022
2023				// update the webmetrics clients list:
2024				const wmclients = document.getElementById('botmon__today__wm_clients');
2025				if (wmclients) {
2026
2027					wmclients.appendChild(makeElement('dt', {}, "Browsers"));
2028
2029					const clientList = BotMon.live.data.analytics.getTopBrowsers(maxItemsPerList);
2030					if (clientList) {
2031						clientList.forEach( (cInfo) => {
2032							const cDd = makeElement('dd');
2033							cDd.appendChild(makeElement('span', {'class': 'has_icon client cl_' + cInfo.id }, ( cInfo.name ? cInfo.name : cInfo.id)));
2034							cDd.appendChild(makeElement('span', {
2035								'class': 'count',
2036								'title': cInfo.count + " page views"
2037							}, cInfo.pct.toFixed(1) + '%'));
2038							wmclients.appendChild(cDd);
2039						});
2040					}
2041				}
2042
2043				// update the webmetrics platforms list:
2044				const wmplatforms = document.getElementById('botmon__today__wm_platforms');
2045				if (wmplatforms) {
2046
2047					wmplatforms.appendChild(makeElement('dt', {}, "Platforms"));
2048
2049					const pfList = BotMon.live.data.analytics.getTopPlatforms(maxItemsPerList);
2050					if (pfList) {
2051						pfList.forEach( (pInfo) => {
2052							const pDd = makeElement('dd');
2053							pDd.appendChild(makeElement('span', {'class': 'has_icon platform pf_' + pInfo.id }, ( pInfo.name ? pInfo.name : pInfo.id)));
2054							pDd.appendChild(makeElement('span', {
2055								'class': 'count',
2056								'title': pInfo.count + " page views"
2057							}, pInfo.pct.toFixed(1) + '%'));
2058							wmplatforms.appendChild(pDd);
2059						});
2060					}
2061				}
2062
2063				// update the top bot countries list:
2064				const usrCountries = document.getElementById('botmon__today__wm_countries');
2065				if (usrCountries) {
2066					usrCountries.appendChild(makeElement('dt', {}, `Top visitor Countries:`));
2067					const usrCtryList = BotMon.live.data.analytics.getCountryList('human', 5);
2068					usrCtryList.forEach( (cInfo) => {
2069						const cLi = makeElement('dd');
2070						cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name));
2071						cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count));
2072						usrCountries.appendChild(cLi);
2073					});
2074				}
2075
2076				// update the top pages;
2077				const wmpages = document.getElementById('botmon__today__wm_pages');
2078				if (wmpages) {
2079
2080					wmpages.appendChild(makeElement('dt', {}, "Top pages"));
2081
2082					const pgList = BotMon.live.data.analytics.getTopPages(maxItemsPerList);
2083					if (pgList) {
2084						pgList.forEach( (pgInfo) => {
2085							const pgDd = makeElement('dd');
2086							pgDd.appendChild(makeElement('a', {
2087								'class': 'page_icon',
2088								'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(pgInfo.id),
2089								'target': 'preview',
2090								'title': "PageID: " + pgInfo.id
2091							}, pgInfo.id));
2092							pgDd.appendChild(makeElement('span', {
2093								'class': 'count',
2094								'title': pgInfo.count + " page views"
2095							}, pgInfo.count));
2096							wmpages.appendChild(pgDd);
2097						});
2098					}
2099				}
2100
2101				// update the top referrers;
2102				const wmreferers = document.getElementById('botmon__today__wm_referers');
2103				if (wmreferers) {
2104
2105					wmreferers.appendChild(makeElement('dt', {}, "Referers"));
2106
2107					const refList = BotMon.live.data.analytics.getTopReferers(maxItemsPerList);
2108					if (refList) {
2109						refList.forEach( (rInfo) => {
2110							const rDd = makeElement('dd');
2111							rDd.appendChild(makeElement('span', {'class': 'has_icon referer ref_' + rInfo.id }, rInfo.name));
2112							rDd.appendChild(makeElement('span', {
2113								'class': 'count',
2114								'title': rInfo.count + " references"
2115							}, rInfo.pct.toFixed(1) + '%'));
2116							wmreferers.appendChild(rDd);
2117						});
2118					}
2119				}
2120			}
2121		},
2122
2123		status: {
2124			setText: function(txt) {
2125				const el = document.getElementById('botmon__today__status');
2126				if (el && BotMon.live.gui.status._errorCount <= 0) {
2127					el.innerText = txt;
2128				}
2129			},
2130
2131			setTitle: function(html) {
2132				const el = document.getElementById('botmon__today__title');
2133				if (el) {
2134					el.innerHTML = html;
2135				}
2136			},
2137
2138			setError: function(txt) {
2139				console.error(txt);
2140				BotMon.live.gui.status._errorCount += 1;
2141				const el = document.getElementById('botmon__today__status');
2142				if (el) {
2143					el.innerText = "Data may be incomplete.";
2144					el.classList.add('error');
2145				}
2146			},
2147			_errorCount: 0,
2148
2149			showBusy: function(txt = null) {
2150				BotMon.live.gui.status._busyCount += 1;
2151				const el = document.getElementById('botmon__today__busy');
2152				if (el) {
2153					el.style.display = 'inline-block';
2154				}
2155				if (txt) BotMon.live.gui.status.setText(txt);
2156			},
2157			_busyCount: 0,
2158
2159			hideBusy: function(txt = null) {
2160				const el = document.getElementById('botmon__today__busy');
2161				BotMon.live.gui.status._busyCount -= 1;
2162				if (BotMon.live.gui.status._busyCount <= 0) {
2163					if (el) el.style.display = 'none';
2164					if (txt) BotMon.live.gui.status.setText(txt);
2165				}
2166			}
2167		},
2168
2169		lists: {
2170			init: function() {
2171
2172				// function shortcut:
2173				const makeElement = BotMon.t._makeElement;
2174
2175				const parent = document.getElementById('botmon__today__visitorlists');
2176				if (parent) {
2177
2178					for (let i=0; i < 4; i++) {
2179
2180						// change the id and title by number:
2181						let listTitle = '';
2182						let listId = '';
2183						let infolink = null;
2184						switch (i) {
2185							case 0:
2186								listTitle = "Registered users";
2187								listId = 'users';
2188								break;
2189							case 1:
2190								listTitle = "Probably humans";
2191								listId = 'humans';
2192								break;
2193							case 2:
2194								listTitle = "Suspected bots";
2195								listId = 'suspectedBots';
2196								infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/suspected_bots';
2197								break;
2198							case 3:
2199								listTitle = "Known bots";
2200								listId = 'knownBots';
2201								infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/known_bots';
2202								break;
2203							default:
2204								console.warn('Unknown list number.');
2205						}
2206
2207						const details = makeElement('details', {
2208							'data-group': listId,
2209							'data-loaded': false
2210						});
2211						const title = details.appendChild(makeElement('summary'));
2212						title.appendChild(makeElement('span', {'class': 'title'}, listTitle));
2213						if (infolink) {
2214							title.appendChild(makeElement('a', {
2215								'class': 'ext_info',
2216								'target': '_blank',
2217								'href': infolink,
2218								'title': "More information"
2219							}, "Info"));
2220						}
2221						details.addEventListener("toggle", this._onDetailsToggle);
2222
2223						parent.appendChild(details);
2224
2225					}
2226				}
2227			},
2228
2229			_onDetailsToggle: function(e) {
2230				//console.info('BotMon.live.gui.lists._onDetailsToggle()');
2231
2232				const target = e.target;
2233
2234				if (target.getAttribute('data-loaded') == 'false') { // only if not loaded yet
2235					target.setAttribute('data-loaded', 'loading');
2236
2237					const fillType = target.getAttribute('data-group');
2238					const fillList = BotMon.live.data.analytics.groups[fillType];
2239					if (fillList && fillList.length > 0) {
2240
2241						const ul = BotMon.t._makeElement('ul');
2242
2243						fillList.forEach( (it) => {
2244							ul.appendChild(BotMon.live.gui.lists._makeVisitorItem(it, fillType));
2245						});
2246
2247						target.appendChild(ul);
2248						target.setAttribute('data-loaded', 'true');
2249					} else {
2250						target.setAttribute('data-loaded', 'false');
2251					}
2252
2253				}
2254			},
2255
2256			_makeVisitorItem: function(data, type) {
2257
2258				// shortcut for neater code:
2259				const make = BotMon.t._makeElement;
2260
2261				let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' );
2262				if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0';
2263
2264				const platformName = (data._platform ? data._platform.n : 'Unknown');
2265				const clientName = (data._client ? data._client.n: 'Unknown');
2266
2267				const sumClass = ( !data._seenBy || data._seenBy.indexOf(BM_LOGTYPE.SERVER) < 0 ? 'noServer' : 'hasServer');
2268
2269				const li = make('li'); // root list item
2270				const details = make('details');
2271				const summary = make('summary', {
2272					'class': sumClass
2273				});
2274				details.appendChild(summary);
2275
2276				const span1 = make('span'); /* left-hand group */
2277
2278				if (data._type !== BM_USERTYPE.KNOWN_BOT) { /* No platform/client for bots */
2279					span1.appendChild(make('span', { /* Platform */
2280						'class': 'icon_only platform pf_' + (data._platform ? data._platform.id : 'unknown'),
2281						'title': "Platform: " + platformName
2282					}, platformName));
2283
2284					span1.appendChild(make('span', { /* Client */
2285						'class': 'icon_only client client cl_' + (data._client ? data._client.id : 'unknown'),
2286						'title': "Client: " + clientName
2287					}, clientName));
2288				}
2289
2290				// identifier:
2291				if (data._type == BM_USERTYPE.KNOWN_BOT) { /* Bot only */
2292
2293					const botName = ( data._bot && data._bot.n ? data._bot.n : "Unknown");
2294					span1.appendChild(make('span', { /* Bot */
2295						'class': 'has_icon bot bot_' + (data._bot ? data._bot.id : 'unknown'),
2296						'title': "Bot: " + botName
2297					}, botName));
2298
2299				} else if (data._type == BM_USERTYPE.KNOWN_USER) { /* User only */
2300
2301					span1.appendChild(make('span', { /* User */
2302						'class': 'has_icon user_known',
2303						'title': "User: " + data.usr
2304					}, data.usr));
2305
2306				} else { /* others */
2307
2308
2309					span1.appendChild(make('span', { // IP-Address
2310						'class': 'has_icon ipaddr ip' + ipType,
2311						'title': "IP-Address: " + data.ip
2312					}, data.ip));
2313
2314					/*span1.appendChild(make('span', { // Internal ID
2315						'class': 'has_icon session typ_' + data.typ,
2316						'title': "ID: " + data.id
2317					}, data.id));*/
2318				}
2319
2320				span1.appendChild(make('span', { /* page views */
2321					'class': 'has_icon pageseen',
2322					'title': data._pageViews.length + " page load(s)"
2323				}, data._pageViews.length));
2324
2325
2326				// referer icons:
2327				if ((data._type == BM_USERTYPE.PROBABLY_HUMAN || data._type == BM_USERTYPE.LIKELY_BOT) && data.ref) {
2328					const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref);
2329					span1.appendChild(make('span', {
2330						'class': 'icon_only referer ref_' + refInfo.id,
2331						'title': "Referer: " + data.ref
2332					}, refInfo.n));
2333				}
2334
2335				summary.appendChild(span1);
2336				const span2 = make('span'); /* right-hand group */
2337
2338				// country flag:
2339					if (data.geo && data.geo !== 'ZZ') {
2340						span2.appendChild(make('span', {
2341							'class': 'icon_only country ctry_' + data.geo.toLowerCase(),
2342							'data-ctry': data.geo,
2343							'title': "Country: " + ( data._country || "Unknown")
2344						}, ( data._country || "Unknown") ));
2345					}
2346
2347					span2.appendChild(make('span', { // seen-by icon:
2348						'class': 'icon_only seenby sb_' + data._seenBy.join(''),
2349						'title': "Seen by: " + data._seenBy.join('+')
2350					}, data._seenBy.join(', ')));
2351
2352					const captchaCode = '' + ( data._captcha['Y'] > 0 ? 'Y' : '' ) + ( data._captcha['N'] > 0 ? 'N' : '' ) + ( data._captcha['W'] > 0 ? 'W' : '' );
2353					if (captchaCode !== '') {
2354						span2.appendChild(make('span', { // captcha status
2355							'class': 'icon_only captcha cap_' + captchaCode,
2356						}, captchaCode));
2357					}
2358
2359				summary.appendChild(span2);
2360
2361				// add details expandable section:
2362				details.appendChild(BotMon.live.gui.lists._makeVisitorDetails(data, type));
2363
2364				li.appendChild(details);
2365				return li;
2366			},
2367
2368			_makeVisitorDetails: function(data, type) {
2369
2370				// shortcut for neater code:
2371				const make = BotMon.t._makeElement;
2372
2373				let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' );
2374				if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0';
2375				const platformName = (data._platform ? data._platform.n : 'Unknown');
2376				const clientName = (data._client ? data._client.n: 'Unknown');
2377
2378				const dl = make('dl', {'class': 'visitor_details'});
2379
2380				if (data._type == BM_USERTYPE.KNOWN_BOT) {
2381
2382					dl.appendChild(make('dt', {}, "Bot name:")); /* bot info */
2383					dl.appendChild(make('dd', {'class': 'icon_only bot bot_' + (data._bot ? data._bot.id : 'unknown')},
2384						(data._bot ? data._bot.n : 'Unknown')));
2385
2386					if (data._bot && data._bot.url) {
2387						dl.appendChild(make('dt', {}, "Bot info:")); /* bot info */
2388						const botInfoDd = dl.appendChild(make('dd'));
2389						botInfoDd.appendChild(make('a', {
2390							'href': data._bot.url,
2391							'target': '_blank'
2392						}, data._bot.url)); /* bot info link*/
2393
2394					}
2395
2396				} else { /* not for bots */
2397
2398					dl.appendChild(make('dt', {}, "Client:")); /* client */
2399					dl.appendChild(make('dd', {'class': 'has_icon client cl_' + (data._client ? data._client.id : 'unknown')},
2400						clientName + ( data._client.v > 0 ? ' (' + data._client.v + ')' : '' ) ));
2401
2402					dl.appendChild(make('dt', {}, "Platform:")); /* platform */
2403					dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')},
2404						platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) ));
2405
2406					/*dl.appendChild(make('dt', {}, "ID:"));
2407					dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/
2408				}
2409
2410				dl.appendChild(make('dt', {}, "IP-Address:"));
2411				const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType});
2412					ipItem.appendChild(make('span', {'class': 'address'} , data.ip));
2413					ipItem.appendChild(make('a', {
2414						'class': 'icon_only extlink ipinfo',
2415						'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`,
2416						'target': 'ipinfo',
2417						'title': "View this address on IPInfo.io"
2418					} , "DNS Info"));
2419					ipItem.appendChild(make('a', {
2420						'class': 'icon_only extlink abuseipdb',
2421						'href': `https://www.abuseipdb.com/check/${encodeURIComponent(data.ip)}`,
2422						'target': 'abuseipdb',
2423						'title': "Check this address on AbuseIPDB.com"
2424					} , "Check on AbuseIPDB"));
2425				dl.appendChild(ipItem);
2426
2427				if (Math.abs(data._lastSeen - data._firstSeen) < 100) {
2428					dl.appendChild(make('dt', {}, "Seen:"));
2429					dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString()));
2430				} else {
2431					dl.appendChild(make('dt', {}, "First seen:"));
2432					dl.appendChild(make('dd', {'class': 'firstSeen'}, data._firstSeen.toLocaleString()));
2433					dl.appendChild(make('dt', {}, "Last seen:"));
2434					dl.appendChild(make('dd', {'class': 'lastSeen'}, data._lastSeen.toLocaleString()));
2435				}
2436
2437				dl.appendChild(make('dt', {}, "Actions:"));
2438				dl.appendChild(make('dd', {'class': 'views'},
2439					"Page loads: " + data._loadCount.toString() +
2440					( data._captcha['Y'] > 0 || data._captcha['W'] || data._captcha['-'] > 0 ? ", captchas: " + (data._captcha['Y']+data._captcha['W']+data._captcha['-']).toString() : '') +
2441					", views: " + data._viewCount.toString()
2442				));
2443
2444				dl.appendChild(make('dt', {}, "User-Agent:"));
2445				dl.appendChild(make('dd', {'class': 'agent'}, data.agent));
2446
2447				if (data.ref && data.ref !== '') {
2448					dl.appendChild(make('dt', {}, "Referrer:"));
2449
2450					const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref);
2451					const refDd = dl.appendChild(make('dd', {
2452						'class': 'has_icon referer ref_' + refInfo.id
2453					}));
2454					refDd.appendChild(make('a', {
2455						'href': data.ref,
2456						'target': 'refTarget'
2457					}, data.ref));
2458				}
2459
2460				dl.appendChild(make('dt', {}, "Languages:"));
2461				dl.appendChild(make('dd', {'class': 'langs'}, ` [${data.accept}]`));
2462
2463				if (data.geo && data.geo !=='') {
2464					dl.appendChild(make('dt', {}, "Location:"));
2465					dl.appendChild(make('dd', {
2466						'class': 'has_icon country ctry_' + data.geo.toLowerCase(),
2467						'data-ctry': data.geo,
2468						'title': "Country: " + data._country
2469					}, data._country + ' (' + data.geo + ')'));
2470				}
2471				if (data.captcha && data.captcha !=='') {
2472					dl.appendChild(make('dt', {}, "Captcha-status:"));
2473					dl.appendChild(make('dd', {
2474						'class': 'captcha'
2475					}, data.captcha));
2476				}
2477
2478				dl.appendChild(make('dt', {}, "Session ID:"));
2479				dl.appendChild(make('dd', {'class': 'has_icon session typ_' + data.typ}, data.id));
2480
2481				dl.appendChild(make('dt', {}, "Seen by:"));
2482				dl.appendChild(make('dd', {'class': 'has_icon seenby sb_' + data._seenBy.join('')}, data._seenBy.join(', ') ));
2483
2484				dl.appendChild(make('dt', {}, "Visited pages:"));
2485				const pagesDd = make('dd', {'class': 'pages'});
2486				const pageList = make('ul');
2487
2488				/* list all page views */
2489				data._pageViews.sort( (a, b) => a._firstSeen - b._firstSeen );
2490				data._pageViews.forEach( (page) => {
2491					pageList.appendChild(BotMon.live.gui.lists._makePageViewItem(page));
2492				});
2493				pagesDd.appendChild(pageList);
2494				dl.appendChild(pagesDd);
2495
2496				/* bot evaluation rating */
2497				if (data._type !== BM_USERTYPE.KNOWN_BOT && data._type !== BM_USERTYPE.KNOWN_USER) {
2498					dl.appendChild(make('dt', undefined, "Bot rating:"));
2499					dl.appendChild(make('dd', {'class': 'bot-rating'}, ( data._botVal ? data._botVal : '–' ) + ' (of ' + BotMon.live.data.rules._threshold + ')'));
2500
2501					/* add bot evaluation details: */
2502					if (data._eval) {
2503						dl.appendChild(make('dt', {}, "Bot evaluation:"));
2504						const evalDd = make('dd', {'class': 'eval'});
2505						const testList = make('ul');
2506						data._eval.forEach( test => {
2507
2508							const tObj = BotMon.live.data.rules.getRuleInfo(test);
2509							let tDesc = tObj ? tObj.desc : test;
2510
2511							// special case for Bot IP range test:
2512							if (tObj.func == 'fromKnownBotIP') {
2513								const rangeInfo = BotMon.live.data.ipRanges.match(data.ip);
2514								if (rangeInfo) {
2515									const owner = BotMon.live.data.ipRanges.getOwner(rangeInfo.g);
2516									tDesc += ' (range: “' + rangeInfo.cidr + '”, ' + owner + ')';
2517								}
2518							}
2519
2520							// create the entry field
2521							const tstLi = make('li');
2522							tstLi.appendChild(make('span', {
2523								'data-testid': test
2524							}, tDesc));
2525							tstLi.appendChild(make('span', {}, ( tObj ? tObj.bot : '—') ));
2526							testList.appendChild(tstLi);
2527						});
2528
2529						// add total row
2530						const tst2Li = make('li', {
2531							'class': 'total'
2532						});
2533						/*tst2Li.appendChild(make('span', {}, "Total:"));
2534						tst2Li.appendChild(make('span', {}, data._botVal));
2535						testList.appendChild(tst2Li);*/
2536
2537						evalDd.appendChild(testList);
2538						dl.appendChild(evalDd);
2539					}
2540				}
2541				// return the element to add to the UI:
2542				return dl;
2543			},
2544
2545			// make a page view item:
2546			_makePageViewItem: function(page) {
2547				//console.log("makePageViewItem:",page);
2548
2549				// shortcut for neater code:
2550				const make = BotMon.t._makeElement;
2551
2552				// the actual list item:
2553				const pgLi = make('li');
2554
2555				const row1 = make('div', {'class': 'row'});
2556
2557					row1.appendChild(make('a', { // page id is the left group
2558						'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(page.pg),
2559						'target': 'preview',
2560						'hreflang': page.lang,
2561						'title': "PageID: " + page.pg
2562					}, page.pg)); /* DW Page ID */
2563
2564					const rightGroup = row1.appendChild(make('div')); // right-hand group
2565
2566						// get the time difference:
2567						rightGroup.appendChild(make('span', {
2568							'class': 'first-seen',
2569							'title': "First visited: " + page._firstSeen.toLocaleString() + " UTC"
2570						}, BotMon.t._formatTime(page._firstSeen)));
2571
2572						rightGroup.appendChild(make('span', { /* page loads */
2573							'class': 'has_icon pageviews',
2574							'title': page._viewCount.toString() + " page load(s)"
2575						}, page._viewCount.toString()));
2576
2577				pgLi.appendChild(row1);
2578
2579				/* LINE 2 */
2580
2581				const row2 = make('div', {'class': 'row'});
2582
2583					// page referrer:
2584					if (page._ref) {
2585						row2.appendChild(make('span', {
2586							'class': 'referer',
2587							'title': "Referrer: " + page._ref.href
2588						}, page._ref.hostname));
2589					} else {
2590						row2.appendChild(make('span', {
2591							'class': 'referer'
2592						}, "No referer"));
2593					}
2594
2595					// visit duration:
2596					let visitTimeStr = "Bounce";
2597					const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime();
2598					if (visitDuration > 0) {
2599						visitTimeStr = Math.floor(visitDuration / 1000) + "s";
2600					}
2601					const tDiff = BotMon.t._formatTimeDiff(page._firstSeen, page._lastSeen);
2602					if (tDiff) {
2603						row2.appendChild(make('span', {'class': 'visit-length', 'title': 'Last seen: ' + page._lastSeen.toLocaleString()}, tDiff));
2604					} else {
2605						row2.appendChild(make('span', {
2606							'class': 'bounce',
2607							'title': "Visitor bounced"}, "Bounce"));
2608					}
2609
2610				pgLi.appendChild(row2);
2611
2612				return pgLi;
2613			}
2614		}
2615	}
2616};
2617
2618/* launch only if the BotMon admin panel is open: */
2619if (document.getElementById('botmon__admin')) {
2620	BotMon.init();
2621}