Skip to content
This repository was archived by the owner on Dec 14, 2022. It is now read-only.

Commit c5153fe

Browse files
author
Chris Wiechmann
committed
Optimized indexing of documents into ES
Using specific field mappings to redruce disk-usage and index load on ES.
1 parent a243797 commit c5153fe

12 files changed

+409
-286
lines changed

‎elk-traffic-monitor-api/test/_base.js

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,19 +71,25 @@ async function sendToElasticsearch(elasticConfig, index, dataset) {
7171
const client = new Client({
7272
node: elasticConfig.node
7373
});
74+
debugger;
7475
const mappingConfig = JSON.parse(fs.readFileSync('../logstash/config/traffic_details_index_template.json')).mappings;
75-
await client.indices.create({
76+
const createdIndexResponse = await client.indices.create({
7677
index: index,
7778
body: {
7879
mappings: mappingConfig
7980
}
8081
}, { ignore: [400] });
82+
if (createdIndexResponse.statusCode!=200) {
83+
throw Error(`Error creating index: ${index} with template: ${createdIndexResponse.body.error.reason}`);
84+
}
85+
8186

8287
const body = dataset.flatMap(doc => [{ index: { _index: index } }, doc]);
8388
const { body: bulkResponse } = await client.bulk({ refresh: true, body });
8489

8590
if (bulkResponse.errors) {
86-
console.log(JSON.stringify(bulkResponse.errors));
91+
console.log(JSON.stringify(bulkResponse.items[0].index.error));
92+
throw Error(`Error inserting test document into index: ${index}`);
8793
}
8894
console.log(`Inserted test data into index: ${index}`);
8995
}

‎elk-traffic-monitor-api/test/documents/basic/circuitpath_test_documents.js

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ module.exports = [
1212
"groupId": "group-2",
1313
"serviceName": "QuickStart Server"
1414
},
15-
"@version": "1",
1615
"@timestamp": getDate('8m'),
1716
"circuitPath": [
1817
{
@@ -218,7 +217,6 @@ module.exports = [
218217
"groupId": "group-2"
219218
},
220219
"@timestamp": "2020-07-15T18:19:20.744Z",
221-
"@version": "1",
222220
"logtype": "openlog",
223221
"transactionSummary": {
224222
"path": "/healthcheck",
@@ -279,7 +277,6 @@ module.exports = [
279277
"finalStatus": "Error"
280278
}
281279
},
282-
"@version": "1",
283280
"correlationId": "bb30715e5300e189d1da43fc",
284281
"circuitPath": [
285282
{
@@ -361,7 +358,6 @@ module.exports = [
361358
}
362359
},
363360
"correlationId": "edb1705e7d0168a34d74bfba",
364-
"@version": "1",
365361
"circuitPath": [],
366362
"tags": [
367363
"openlog"
@@ -405,7 +401,6 @@ module.exports = [
405401
}
406402
},
407403
"correlationId": "1ab3705e920284217e6aae73",
408-
"@version": "1",
409404
"circuitPath": [
410405
{
411406
"execTime": 1,

‎elk-traffic-monitor-api/test/documents/basic/getinfo_test_documents.js

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ const getDate = require('../../util');
22

33
module.exports = [
44
{
5-
"@version": "1",
65
"@timestamp": "2020-07-03T15:56:11.597Z",
76
"transactionElements": {
87
"leg1": {

‎elk-traffic-monitor-api/test/documents/basic/search_count_documents.js

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ module.exports = [
5858
"serviceId": "instance-1"
5959
},
6060
"@timestamp": getDate('15m'),
61-
"@version": "1",
6261
"logtype": "openlog",
6362
"transactionSummary": {
6463
"path": "/favicon.ico",
@@ -141,7 +140,6 @@ module.exports = [
141140
"serviceId": "instance-1"
142141
},
143142
"@timestamp": getDate('120h'),
144-
"@version": "1",
145143
"logtype": "openlog",
146144
"transactionSummary": {
147145
"path": "/petstore/v2/pet/findByStatus",
@@ -234,7 +232,6 @@ module.exports = [
234232
"groupId": "group-2"
235233
},
236234
"@timestamp": getDate('65m'),
237-
"@version": "1",
238235
"logtype": "openlog",
239236
"transactionSummary": {
240237
"path": "/healthcheck",
@@ -328,7 +325,6 @@ module.exports = [
328325
"serviceId": "instance-1"
329326
},
330327
"@timestamp": getDate('3000h'),
331-
"@version": "1",
332328
"logtype": "openlog",
333329
"transactionSummary": {
334330
"path": "/healthcheck",
@@ -446,7 +442,6 @@ module.exports = [
446442
"serviceId": "instance-1"
447443
},
448444
"@timestamp": getDate('8m'),
449-
"@version": "1",
450445
"logtype": "openlog",
451446
"circuitPath": [
452447
{
@@ -623,7 +618,6 @@ module.exports = [
623618
"serviceId": "instance-1"
624619
},
625620
"@timestamp": getDate('8m'),
626-
"@version": "1",
627621
"logtype": "openlog",
628622
"circuitPath": [
629623
{
@@ -800,7 +794,6 @@ module.exports = [
800794
"serviceId": "instance-1"
801795
},
802796
"@timestamp": getDate('8m'),
803-
"@version": "1",
804797
"logtype": "openlog",
805798
"circuitPath": [
806799
{
@@ -977,7 +970,6 @@ module.exports = [
977970
"serviceId": "instance-1"
978971
},
979972
"@timestamp": getDate('8m'),
980-
"@version": "1",
981973
"logtype": "openlog",
982974
"circuitPath": [
983975
{
@@ -1154,7 +1146,6 @@ module.exports = [
11541146
"serviceId": "instance-1"
11551147
},
11561148
"@timestamp": getDate('8m'),
1157-
"@version": "1",
11581149
"logtype": "openlog",
11591150
"circuitPath": [
11601151
{
@@ -1331,7 +1322,6 @@ module.exports = [
13311322
"serviceId": "instance-1"
13321323
},
13331324
"@timestamp": getDate('8m'),
1334-
"@version": "1",
13351325
"logtype": "openlog",
13361326
"circuitPath": [
13371327
{
@@ -1508,7 +1498,6 @@ module.exports = [
15081498
"serviceId": "instance-1"
15091499
},
15101500
"@timestamp": getDate('8m'),
1511-
"@version": "1",
15121501
"logtype": "openlog",
15131502
"circuitPath": [
15141503
{
@@ -1685,7 +1674,6 @@ module.exports = [
16851674
"serviceId": "instance-1"
16861675
},
16871676
"@timestamp": getDate('8m'),
1688-
"@version": "1",
16891677
"logtype": "openlog",
16901678
"circuitPath": [
16911679
{
@@ -1862,7 +1850,6 @@ module.exports = [
18621850
"serviceId": "instance-1"
18631851
},
18641852
"@timestamp": getDate('8m'),
1865-
"@version": "1",
18661853
"logtype": "openlog",
18671854
"circuitPath": [
18681855
{

‎elk-traffic-monitor-api/test/documents/basic/search_test_documents.js

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ module.exports = [
5858
"serviceId": "instance-1"
5959
},
6060
"@timestamp": getDate('15m'),
61-
"@version": "1",
6261
"logtype": "openlog",
6362
"transactionSummary": {
6463
"path": "/favicon.ico",
@@ -141,7 +140,6 @@ module.exports = [
141140
"serviceId": "instance-2"
142141
},
143142
"@timestamp": getDate('120h'),
144-
"@version": "1",
145143
"logtype": "openlog",
146144
"transactionSummary": {
147145
"path": "/petstore/v2/pet/findByStatus",
@@ -234,7 +232,6 @@ module.exports = [
234232
"groupId": "group-2"
235233
},
236234
"@timestamp": getDate('65m'),
237-
"@version": "1",
238235
"logtype": "openlog",
239236
"transactionSummary": {
240237
"path": "/healthcheck",
@@ -328,7 +325,6 @@ module.exports = [
328325
"serviceId": "instance-1"
329326
},
330327
"@timestamp": getDate('3000h'),
331-
"@version": "1",
332328
"logtype": "openlog",
333329
"transactionSummary": {
334330
"path": "/healthcheck",
@@ -446,7 +442,6 @@ module.exports = [
446442
"serviceId" : "instance-1"
447443
},
448444
"@timestamp" : getDate('8m'),
449-
"@version" : "1",
450445
"logtype" : "openlog",
451446
"circuitPath" : [
452447
{

0 commit comments

Comments
 (0)