Skip to content

Commit 41748fc

Browse files
GraceRuanGraceRuan
committed
feat: add wildfire monitors
1 parent ec4105f commit 41748fc

File tree

6 files changed

+380
-0
lines changed

6 files changed

+380
-0
lines changed

‎workflow-cli/configuration-opensearch/alerting/server-cpu.config.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"type": "server"
3+
}

‎workflow-cli/configuration-opensearch/alerting/server-cpu.monitor.json‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
{
2+
"$$OMIT": "<%= server.name != 'beneath' ? 'true' : 'false' %>",
3+
"name": "nrids_cpu_<%= server.name %>",
4+
"type": "monitor",
5+
"monitor_type": "query_level_monitor",
6+
"enabled": true,
7+
"schedule": {
8+
"period": {
9+
"interval": 15,
10+
"unit": "MINUTES"
11+
}
12+
},
13+
"inputs": [
14+
{
15+
"search": {
16+
"indices": [
17+
"nrm-metrics-*"
18+
],
19+
"query": {
20+
"size": 0,
21+
"aggregations": {
22+
"metric": {
23+
"avg": {
24+
"field": "host.cpu.usage"
25+
}
26+
}
27+
},
28+
"query": {
29+
"bool": {
30+
"filter": [
31+
{
32+
"range": {
33+
"@timestamp": {
34+
"from": "{{period_end}}||-30m",
35+
"to": "{{period_end}}",
36+
"include_lower": true,
37+
"include_upper": true,
38+
"format": "epoch_millis",
39+
"boost": 1.0
40+
}
41+
}
42+
},
43+
{
44+
"term": {
45+
"host.hostname": {
46+
"value": "<%= server.name %>",
47+
"boost": 1.0
48+
}
49+
}
50+
}
51+
],
52+
"adjust_pure_negative": true,
53+
"boost": 1.0
54+
}
55+
}
56+
}
57+
}
58+
}
59+
],
60+
"triggers": [
61+
{
62+
"query_level_trigger": {
63+
"id": "<%= idgen('trigger', server.name) %>",
64+
"name": "AbnormalCPU from server <%= server.name %>",
65+
"severity": "4",
66+
"condition": {
67+
"script": {
68+
"source": "return ctx.results[0].aggregations.metric.value == null ? false : ctx.results[0].aggregations.metric.value > 95",
69+
"lang": "painless"
70+
}
71+
},
72+
"actions": [
73+
{
74+
"$$OMIT": "<%= !serverTag('wildfire') || !serverTag('nonproduction') ? 'true' : 'false' %>",
75+
"id": "<%= idgen('action-nonprod-wf', server.name) %>",
76+
"name": "notify",
77+
"destination_id": "wf-nonprod-msteams",
78+
"message_template": {
79+
"source": "{\"text\": \"Abnormal high CPU usage alerting received from <%= server.name %> between {{ctx.periodStart}} and {{ctx.periodEnd}}\"}",
80+
"lang": "mustache"
81+
},
82+
"throttle_enabled": true,
83+
"subject_template": {
84+
"source": "<%= server.name %> CPU Usage Alert",
85+
"lang": "mustache"
86+
},
87+
"throttle": {
88+
"value": 1440,
89+
"unit": "MINUTES"
90+
}
91+
},
92+
{
93+
"$$OMIT": "<%= !serverTag('wildfire') || !serverTag('production') ? 'true' : 'false' %>",
94+
"id": "<%= idgen('action-prod-wf', server.name) %>",
95+
"name": "notify",
96+
"destination_id": "wf-prod-msteams",
97+
"message_template": {
98+
"source": "{\"text\": \"Abnormal high CPU usage alerting received from <%= server.name %> between {{ctx.periodStart}} and {{ctx.periodEnd}}\"}",
99+
"lang": "mustache"
100+
},
101+
"throttle_enabled": true,
102+
"subject_template": {
103+
"source": "<%= server.name %> CPU Usage Alert",
104+
"lang": "mustache"
105+
},
106+
"throttle": {
107+
"value": 30,
108+
"unit": "MINUTES"
109+
}
110+
}
111+
]
112+
}
113+
}
114+
],
115+
"data_sources": {
116+
"alerts_history_index": ".opendistro-alerting-alert-history-write",
117+
"alerts_history_index_pattern": "<.opendistro-alerting-alert-history-{now/d}-1>",
118+
"alerts_index": ".opendistro-alerting-alerts",
119+
"findings_enabled": false,
120+
"findings_index": ".opensearch-alerting-finding-history-write",
121+
"findings_index_pattern": "<.opensearch-alerting-finding-history-{now/d}-1>",
122+
"query_index": ".opensearch-alerting-queries",
123+
"query_index_mappings_by_type": {}
124+
}
125+
}

‎workflow-cli/configuration-opensearch/alerting/server-disk.config.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"type": "server"
3+
}

‎workflow-cli/configuration-opensearch/alerting/server-disk.monitor.json‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
{
2+
"$$OMIT": "<%= server.name != 'beneath' ? 'true' : 'false' %>",
3+
"name": "nrids_disk_<%= server.name %>",
4+
"type": "monitor",
5+
"monitor_type": "query_level_monitor",
6+
"enabled": true,
7+
"schedule": {
8+
"period": {
9+
"interval": 15,
10+
"unit": "MINUTES"
11+
}
12+
},
13+
"inputs": [
14+
{
15+
"search": {
16+
"indices": [
17+
"nrm-metrics-*"
18+
],
19+
"query": {
20+
"size": 0,
21+
"aggregations": {
22+
"metric": {
23+
"avg": {
24+
"field": "host.disk.used_percentage"
25+
}
26+
}
27+
},
28+
"query": {
29+
"bool": {
30+
"filter": [
31+
{
32+
"range": {
33+
"@timestamp": {
34+
"from": "{{period_end}}||-30m",
35+
"to": "{{period_end}}",
36+
"include_lower": true,
37+
"include_upper": true,
38+
"format": "epoch_millis",
39+
"boost": 1.0
40+
}
41+
}
42+
},
43+
{
44+
"term": {
45+
"host.hostname": {
46+
"value": "<%= server.name %>",
47+
"boost": 1.0
48+
}
49+
}
50+
}
51+
]
52+
}
53+
}
54+
}
55+
}
56+
}
57+
],
58+
"triggers": [
59+
{
60+
"query_level_trigger": {
61+
"id": "<%= idgen('trigger', server.name) %>",
62+
"name": "AbnormalDisk from server <%= server.name %>",
63+
"severity": "4",
64+
"condition": {
65+
"script": {
66+
"source": "return ctx.results[0].aggregations.metric.value == null ? false : ctx.results[0].aggregations.metric.value > 0.999",
67+
"lang": "painless"
68+
}
69+
},
70+
"actions": [
71+
{
72+
"$$OMIT": "<%= !serverTag('wildfire') || !serverTag('nonproduction') ? 'true' : 'false' %>",
73+
"id": "<%= idgen('action-nonprod-wf', server.name) %>",
74+
"name": "notify",
75+
"destination_id": "wf-nonprod-msteams",
76+
"message_template": {
77+
"source": "{\"text\": \"AbnormalDisk alerting received from <%= server.name %> between {{ctx.periodStart}} and {{ctx.periodEnd}}\"}",
78+
"lang": "mustache"
79+
},
80+
"throttle_enabled": true,
81+
"subject_template": {
82+
"source": "<%= server.name %> Low Disk Space",
83+
"lang": "mustache"
84+
},
85+
"throttle": {
86+
"value": 1440,
87+
"unit": "MINUTES"
88+
}
89+
},
90+
{
91+
"$$OMIT": "<%= !serverTag('wildfire') || !serverTag('production') ? 'true' : 'false' %>",
92+
"id": "<%= idgen('action-prod-wf', server.name) %>",
93+
"name": "notify",
94+
"destination_id": "wf-prod-msteams",
95+
"message_template": {
96+
"source": "{\"text\": \"AbnormalDisk alerting received from <%= server.name %> between {{ctx.periodStart}} and {{ctx.periodEnd}}\"}",
97+
"lang": "mustache"
98+
},
99+
"throttle_enabled": true,
100+
"subject_template": {
101+
"source": "<%= server.name %> Low Disk Space",
102+
"lang": "mustache"
103+
},
104+
"throttle": {
105+
"value": 30,
106+
"unit": "MINUTES"
107+
}
108+
}
109+
]
110+
}
111+
}
112+
],
113+
"data_sources": {
114+
"alerts_history_index": ".opendistro-alerting-alert-history-write",
115+
"alerts_history_index_pattern": "<.opendistro-alerting-alert-history-{now/d}-1>",
116+
"alerts_index": ".opendistro-alerting-alerts",
117+
"findings_enabled": false,
118+
"findings_index": ".opensearch-alerting-finding-history-write",
119+
"findings_index_pattern": "<.opensearch-alerting-finding-history-{now/d}-1>",
120+
"query_index": ".opensearch-alerting-queries",
121+
"query_index_mappings_by_type": {}
122+
}
123+
}

‎workflow-cli/configuration-opensearch/alerting/server-memory.config.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"type": "server"
3+
}

‎workflow-cli/configuration-opensearch/alerting/server-memory.monitor.json‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
{
2+
"$$OMIT": "<%= server.name != 'beneath' ? 'true' : 'false' %>",
3+
"name": "nrids_memory_<%= server.name %>",
4+
"type": "monitor",
5+
"monitor_type": "query_level_monitor",
6+
"enabled": true,
7+
"schedule": {
8+
"period": {
9+
"interval": 15,
10+
"unit": "MINUTES"
11+
}
12+
},
13+
"inputs": [
14+
{
15+
"search": {
16+
"indices": [
17+
"nrm-metrics-*"
18+
],
19+
"query": {
20+
"size": 0,
21+
"aggregations": {
22+
"metric": {
23+
"avg": {
24+
"field": "host.memory.free"
25+
}
26+
}
27+
},
28+
"query": {
29+
"bool": {
30+
"filter": [
31+
{
32+
"range": {
33+
"@timestamp": {
34+
"from": "{{period_end}}||-30m",
35+
"to": "{{period_end}}",
36+
"include_lower": true,
37+
"include_upper": true,
38+
"format": "epoch_millis",
39+
"boost": 1.0
40+
}
41+
}
42+
},
43+
{
44+
"term": {
45+
"host.hostname": {
46+
"value": "<%= server.name %>",
47+
"boost": 1.0
48+
}
49+
}
50+
}
51+
]
52+
}
53+
}
54+
}
55+
}
56+
}
57+
],
58+
"triggers": [
59+
{
60+
"query_level_trigger": {
61+
"id": "<%= idgen('trigger', server.name) %>",
62+
"name": "AbnormalMemory from server <%= server.name %>",
63+
"severity": "4",
64+
"condition": {
65+
"script": {
66+
"source": "return ctx.results[0].aggregations.metric.value == null ? false : ctx.results[0].aggregations.metric.value < 50000",
67+
"lang": "painless"
68+
}
69+
},
70+
"actions": [
71+
{
72+
"$$OMIT": "<%= !serverTag('wildfire') || !serverTag('nonproduction') ? 'true' : 'false' %>",
73+
"id": "<%= idgen('action-nonprod-wf', server.name) %>",
74+
"name": "notify",
75+
"destination_id": "wf-nonprod-msteams",
76+
"message_template": {
77+
"source": "{\"text\": \"AbnormalMemory alerting received from <%= server.name %> between {{ctx.periodStart}} and {{ctx.periodEnd}}\"}",
78+
"lang": "mustache"
79+
},
80+
"throttle_enabled": true,
81+
"subject_template": {
82+
"source": "<%= server.name %> Low Memory",
83+
"lang": "mustache"
84+
},
85+
"throttle": {
86+
"value": 1440,
87+
"unit": "MINUTES"
88+
}
89+
},
90+
{
91+
"$$OMIT": "<%= !serverTag('wildfire') || !serverTag('production') ? 'true' : 'false' %>",
92+
"id": "<%= idgen('action-prod-wf', server.name) %>",
93+
"name": "notify",
94+
"destination_id": "wf-prod-msteams",
95+
"message_template": {
96+
"source": "{\"text\": \"AbnormalMemory alerting received from <%= server.name %> between {{ctx.periodStart}} and {{ctx.periodEnd}}\"}",
97+
"lang": "mustache"
98+
},
99+
"throttle_enabled": true,
100+
"subject_template": {
101+
"source": "<%= server.name %> Low Memory",
102+
"lang": "mustache"
103+
},
104+
"throttle": {
105+
"value": 30,
106+
"unit": "MINUTES"
107+
}
108+
}
109+
]
110+
}
111+
}
112+
],
113+
"data_sources": {
114+
"alerts_history_index": ".opendistro-alerting-alert-history-write",
115+
"alerts_history_index_pattern": "<.opendistro-alerting-alert-history-{now/d}-1>",
116+
"alerts_index": ".opendistro-alerting-alerts",
117+
"findings_enabled": false,
118+
"findings_index": ".opensearch-alerting-finding-history-write",
119+
"findings_index_pattern": "<.opensearch-alerting-finding-history-{now/d}-1>",
120+
"query_index": ".opensearch-alerting-queries",
121+
"query_index_mappings_by_type": {}
122+
}
123+
}

0 commit comments

Comments
 (0)