Fixed a couple of bugs in the userStatusLines, and added support for ignoring [monitorid] when doing the prefix matching.

Author: ben-spiller

CHANGELOG.rst

@@ -1,6 +1,6 @@
 3.6 (in development)
 ---
-...
+- Fixed a couple of bugs in the userStatusLines, and added support for ignoring ``[monitorid]`` when doing the prefix matching. 
 
 3.5
 ---

README.rst

@@ -77,9 +77,22 @@ To do this create a .json configuration file containing a "userStatusLines" dici
 
 	{
 		"userStatusLines":{
+		
+		// This is for a typical application-defined status line. Note that any text inside [...] brackets (
+		// typically the monitor instance id) is ignored. 
+		"com.mycompany.MyMonitor [1] MyApplication Status:": {
+			// This prefix is added to the start of each alias to avoid clashes with other status KPIs
+			"keyPrefix":"myApp.",
+			
+			// Specifying user-friendly aliases for each key is optional. Always include any units (e.g. MB) in the key or alias
+			"key:alias":{
+				"kpi1":"",
+				"kpi2":"kpi2AliasWithUnits",
+				"kpi3":""
+			}},
+		
 		// This detects INFO level lines beginning with "JMS Status:"
 		"JMS Status:": {
-			// This prefix is added to the start of each alias to avoid clashes with other status KPIs
 			"keyPrefix":"jms.",
 			"key:alias":{
 				"s":"s=senders",
@@ -107,15 +120,14 @@ To do this create a .json configuration file containing a "userStatusLines" dici
 		}
 	}
 
-
 Any user-defined status lines should be of the same form as the Correlator status lines, logged at INFO level, 
 for example::
 
 	on all wait(5.0) {
-		log "MyApplication status:"
+		log "MyApplication Status:"
 			+" kpi1="+kpi1.toString()
 			+" kpi2="+kpi2.toString()
-			+" string_kpi=\""+kpi3+"\"" at INFO;
+			+" kpi3=\""+kpi3+"\"" at INFO;
 	}
 
 Technical detail: the frequency and timing of other status lines may not match when the main "Correlator status:" lines 

apamax/log_analyzer.py

@@ -611,8 +611,10 @@ def handleLine(self, file, line, previousLine, **extra):
 			return
 		level = line.level
 
-		for userStatusPrefix, userStatus in self.args.userStatusLines.items():
+		for (userStatusPrefix, userStatusPrefixAfterBracket), userStatus in self.args.userStatusLines.items():
 			if m.startswith(userStatusPrefix):
+				if userStatusPrefixAfterBracket is not None: # special handling of [n] values
+					if userStatusPrefixAfterBracket not in m: continue
 				self.handleRawStatusLine(file=file, line=line, userStatus=userStatus)
 				break
 
@@ -691,6 +693,7 @@ def handleRawStatusLine(self, file, line, userStatus=None, **extra):
 				i+=1
 			if endchar != '"':
 				try:
+					if val.endswith('%') and val[:-1].replace('.','').isdigit(): val = val[:-1] # for user-defined % values which would otherwise not be graphable
 					if '.' in val:
 						val = float(val)
 					else:
@@ -2210,7 +2213,8 @@ def main(self, args):
 
 		globbedpaths = [toLongPathSafe(p) for p in globbedpaths]	
 		globbedpaths.sort() # best we can do until when start reading them - hopefully puts the latest one at the end
-		
+
+		userCharts = {}
 		if args.config:
 			with open(args.config, 'rb') as f:
 				jsonbytes = f.read()
@@ -2223,17 +2227,25 @@ def main(self, args):
 						columns = {k or COLUMN_DISPLAY_NAMES[k] for k in COLUMN_DISPLAY_NAMES}
 						for userStatusPrefix, userStatus in v.items():
 							if not userStatusPrefix.endswith(':'): raise UserError('userStatus prefixes should end with a ":"')
+							
 							for k, alias in userStatus['key:alias'].items():
 								alias = userStatus['keyPrefix']+(alias or k)
 								if alias in columns: raise UserError(f"User status line '{userStatusPrefix}' contains display name '{alias}' which is already in use; consider using keyPrefix to ensure this status line doesn't conflict with display names from others")
 								columns.add(alias)
+						
+						# need a hack to cope with [n] placeholders for monitor instance id
+						args.userStatusLines = {
+							(k[:k.index('[')+1] if ('[' in k and ']' in k) else k, # simple and hence efficient prefix match
+							k[k.index(']'):] if ('[' in k and ']' in k) else None,
+							):v for k, v in args.userStatusLines.items()
+						}
+						
 					elif k == 'userCharts':
 						userCharts = v # allow overriding existing charts if desired
 					else:
 						raise UserError('Unknown key in config file: '%key)
 		else:
 			args.userStatusLines = {}
-			userCharts = {}
 
 		if not globbedpaths: raise UserError('No log files specified')
 

tests/correctness/Cor_018/Input/analyzer_config.json

@@ -1,4 +1,13 @@
 {"userStatusLines":{
+
+	"com.mycompany.MyMonitor [1] MyApplication Status:": {
+		"keyPrefix":"myApp.",
+		"key:alias":{
+			"kpi1":"",
+			"kpi2":"kpi2AliasWithUnits",
+			"kpi3":""
+		}},
+
 	"JMS Status:": {
 		"keyPrefix":"jms.",
 		"key:alias":{

tests/correctness/Cor_018/Input/correlator.log

@@ -174,6 +174,7 @@
 2020-06-12 16:06:08.600 CRIT  [14100] - test.Test [2] Some sample EPL output
 2020-06-12 16:06:09.599 CRIT  [33128] - test.Test [2] Some sample EPL output
 2020-06-12 16:06:10.600 CRIT  [33128] - test.Test [2] Some sample EPL output
+2020-06-12 16:06:16.500 INFO  [140103309588224] - com.mycompany.MyMonitor [1] MyApplication Status: kpi1=100 kpi2=50% kpi3="a string" jeall=1390 jemap=1708 jeret=360 jeefficiency=81% sys=3136 use=1971 max=3122 tot=0 cur=3024 efficiency=65%
 2020-06-12 16:06:11.533 INFO  [47556] - Correlator Status: sm=2 nctx=1 ls=5 rq=0 iq=0 oq=0 icq=0 lcn="<none>" lcq=0 lct=0.0 rx=11 tx=6 rt=0 nc=1 vm=615468 pm=267224 runq=0 si=14.4 so=0.0 srn="<none>" srq=0 jvm=76
 2020-06-12 16:06:11.533 INFO  [47556:Status] - JMS Status: s=1 tx=6 sRate=1 sOutst=0 r=2 rx=4 rRate=0 rWindow=197 rRedel=0 rMaxDeliverySecs=0.0 rDupsDet=1 rDupIds=3 connErr=0 jvmMB=77
 2020-06-12 16:06:11.534 INFO  [47556:Status] -     JMSConnection myConnection: s=1 r=2 connErr=0 sessionsCreated=3
@@ -186,6 +187,7 @@
 2020-06-12 16:06:13.600 CRIT  [14100] - test.Test [2] Some sample EPL output
 2020-06-12 16:06:14.600 CRIT  [14100] - test.Test [2] Some sample EPL output
 2020-06-12 16:06:15.599 CRIT  [14100] - test.Test [2] Some sample EPL output
+2020-06-12 16:06:16.500 INFO  [140103309588224] - com.mycompany.MyMonitor [5] MyApplication Status: kpi1=1585 kpi2=80.2% kpi3="abc def" jeall=1390 jemap=1708 jeret=360 jeefficiency=81% sys=3136 use=1971 max=3122 tot=0 cur=3024 efficiency=65%
 2020-06-12 16:06:16.533 INFO  [47556] - Correlator Status: sm=2 nctx=1 ls=5 rq=0 iq=0 oq=0 icq=0 lcn="<none>" lcq=0 lct=0.0 rx=11 tx=6 rt=0 nc=1 vm=615620 pm=267444 runq=0 si=0.0 so=0.0 srn="<none>" srq=0 jvm=107
 2020-06-12 16:06:16.534 INFO  [47556:Status] - JMS Status: s=1 tx=6 sRate=0 sOutst=0 r=2 rx=4 rRate=0 rWindow=197 rRedel=0 rMaxDeliverySecs=0.0 rDupsDet=1 rDupIds=3 connErr=0 jvmMB=108
 2020-06-12 16:06:16.534 INFO  [47556:Status] -     JMSConnection myConnection: s=1 r=2 connErr=0 sessionsCreated=3

tests/correctness/Cor_018/Reference/user_status_final.txt

@@ -1,3 +1,6 @@
+myApp.kpi1=1585
+myApp.kpi2AliasWithUnits=80.2
+myApp.kpi3='abc def'
 jms.s=senders=1
 jms.r=receivers=2
 jms.rx /sec=0
@@ -19,7 +22,7 @@ jms.r.q1.msgErrors=1
 jms.r.q1.jvmMB=None
 jms.r.q1.datetime='2020-06-12 16:06:11.534'
 jms.r.q1.epoch secs=1591977971.534
-jms.r.q1.line num=181
+jms.r.q1.line num=182
 p.numSnapshots=42
 p.lastSnapshotTime=624772
 p.snapshotWaitTimeEwmaMillis=0.03

tests/correctness/Cor_018/run.py

@@ -7,17 +7,6 @@
 from pysys.constants import *
 from correlatorloganalyzer.analyzer_basetest import AnalyzerBaseTest
 
-"""
-
-custom status line: just add teh values to the most recent real status line
-else we have to put them into separate files and it all gets tricky
-
-add --help
-
-
-deal with error in first status line
-"""
-
 class PySysTest(AnalyzerBaseTest):
 
 	def execute(self):
@@ -30,7 +19,7 @@ def execute(self):
 	def validate(self):
 		self.checkForAnalyzerErrors()
 		header = self.getExprFromFile('loganalyzer_output/status.correlator.csv', '# (.*)').strip().split(',')
-		header = header[header.index('jms.s=senders'):header.index('# metadata: ')]
+		header = header[header.index('myApp.kpi1'):header.index('# metadata: ')]
 		with open(self.output+'/loganalyzer_output/summary_status.correlator.json', 'rb') as f:
 			summary = json.load(f)['status']
 			final = [x for x in summary if x['statistic']=='100% (end)'][0]