feat: add sns and sqs to aws template (#375)

mbystedt · web-flow · commit d5290d919b42 · 2024-08-28T11:50:49.000-07:00
* feat: add sns and sqs to aws template

* feat: add email subs

* feat: individual sub entities

* feat: individual sub entities

* fix: tweak subs

* feat: add etc to internal
diff --git a/.github/workflows/deploy-apm-development.yml b/.github/workflows/deploy-apm-development.yml
@@ -32,6 +32,10 @@ jobs:
         run: ./bin/dev lambda-asset-download
         working-directory: ./workflow-cli
 
+      - name: Generate aws template
+        run: ./bin/dev aws-render
+        working-directory: ./workflow-cli
+
       - name: Setup AWS SAM
         uses: aws-actions/setup-sam@v2
         with:
diff --git a/.github/workflows/deploy-apm-production.yml b/.github/workflows/deploy-apm-production.yml
@@ -34,6 +34,10 @@ jobs:
         run: ./bin/dev lambda-asset-download
         working-directory: ./workflow-cli
 
+      - name: Generate aws template
+        run: ./bin/dev aws-render
+        working-directory: ./workflow-cli
+
       - name: Setup AWS SAM
         uses: aws-actions/setup-sam@v2
         with:
diff --git a/.gitignore b/.gitignore
@@ -121,3 +121,4 @@ terraform/.terraform.lock.hcl
 workflow-cli/indicesusage*.csv
 
 .aws-sam
+/template.yaml
diff --git a/template.yaml.tpl b/template.yaml.tpl
@@ -262,6 +262,23 @@ Resources:
           Enabled: true
           LogGroupName: /aws/kinesisfirehose/apm-dlq-stream
           LogStreamName: DestinationDelivery
+<% notifications.filter((n) => n.configType == 'sns').forEach((notification) => { %>
+  <%= notification.entity %>:
+    Type: AWS::SNS::Topic
+    Properties:
+      TopicName: "<%= notification.sns.topicArn %>"
+      DisplayName: "<%= notification.name %>"<% if (notification.sns.subscriptions) { %><% notification.sns.subscriptions.forEach((sub) => { %>
+  <%= sub.entity %>Sub:
+    Type: AWS::SNS::Subscription
+    Properties:
+      Endpoint: <% if (sub.protocol == 'sqs') { %>!GetAtt <%= sub.entity %>.Arn<% } else if (sub.protocol == 'email' || sub.protocol == 'sns') { %>"<%= sub.endpoint %>"<% } %>
+      Protocol: "<%= sub.protocol %>"
+      TopicArn: !GetAtt <%= notification.entity %>.TopicArn <% if (sub.protocol == 'sqs') { %>
+  <%= sub.entity %>:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: "<%= sub.endpoint %>"<% }}) } -%>
+<% }); -%>
 
 Outputs:
   FunctionName:
diff --git a/workflow-cli/configuration-opensearch/ecs_nrm_1.0/etc.json b/workflow-cli/configuration-opensearch/ecs_nrm_1.0/etc.json
@@ -0,0 +1,15 @@
+{
+  "_meta": {
+    "documentation": "https://apps.nrs.gov.bc.ca/int/confluence/x/PqJvBQ",
+    "version": "1.0"
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "etc": {
+          "type": "flat_object"
+        }
+      }
+    }
+  }
+}
diff --git a/workflow-cli/configuration-opensearch/index_template/access_internal.json b/workflow-cli/configuration-opensearch/index_template/access_internal.json
@@ -30,6 +30,7 @@
     "ecs_url_8.11",
     "ecs_user_agent_8.11",
     "ecs_nrm_base_1.0",
+    "ecs_nrm_etc_1.0",
     "ecs_nrm_kinesis_1.0",
     "ecs_nrm_user_agent_1.0",
     "ecs_nrm_url_context_1.0"
diff --git a/workflow-cli/configuration-opensearch/notification/appdlvr-normal.sns.json b/workflow-cli/configuration-opensearch/notification/appdlvr-normal.sns.json
@@ -1,5 +1,6 @@
 {
   "id": "appdlvr-normal-sns",
+  "entity": "SnsAppDlvrNormal",
   "name": "Appdlvr SNS",
   "description": "SNS destination (APM Deployment Managed)",
   "configType": "sns",
diff --git a/workflow-cli/configuration-opensearch/notification/appdlvr-priority.sns.json b/workflow-cli/configuration-opensearch/notification/appdlvr-priority.sns.json
@@ -1,11 +1,17 @@
 {
   "id": "appdlvr-priority-sns",
+  "entity": "SnsAppDlvrPriority",
   "name": "Appdlvr Priority SNS",
   "description": "SNS destination (APM Deployment Managed)",
   "configType": "sns",
   "isEnabled": true,
   "sns": {
     "topicArn": "nress-prod-appdlvr-priority",
-    "roleArn": "opensearch_sns_nress-prod"
+    "roleArn": "opensearch_sns_nress-prod",
+    "subscriptions": [{
+      "entity": "SnsAppDlvrPriorityEmailApp",
+      "endpoint": "nrids.applicationdelivery@gov.bc.ca",
+      "protocol": "email"
+    }]
   }
 }
diff --git a/workflow-cli/configuration-opensearch/notification/automation-sqs.sns.json b/workflow-cli/configuration-opensearch/notification/automation-sqs.sns.json
@@ -1,11 +1,17 @@
 {
   "id": "automation-sqs-sns",
+  "entity": "SnsAutomationSqs",
   "name": "Automation SQS SNS",
   "description": "SNS destination (APM Deployment Managed)",
   "configType": "sns",
   "isEnabled": true,
   "sns": {
     "topicArn": "nress-prod-message-queue",
-    "roleArn": "opensearch_sns_nress-prod"
+    "roleArn": "opensearch_sns_nress-prod",
+    "subscriptions": [{
+      "entity": "SqsAutomation",
+      "endpoint": "nress-prod-message-queue",
+      "protocol": "sqs"
+    }]
   }
 }
diff --git a/workflow-cli/configuration-opensearch/notification/midtier-normal.sns.json b/workflow-cli/configuration-opensearch/notification/midtier-normal.sns.json
@@ -1,5 +1,6 @@
 {
   "id": "midtier-normal-sns",
+  "entity": "SnsMidtierNormal",
   "name": "Midtier SNS",
   "description": "SNS destination (APM Deployment Managed)",
   "configType": "sns",
diff --git a/workflow-cli/configuration-opensearch/notification/midtier-priority.sns.json b/workflow-cli/configuration-opensearch/notification/midtier-priority.sns.json
@@ -1,5 +1,6 @@
 {
   "id": "midtier-priority-sns",
+  "entity": "SnsPriorityNormal",
   "name": "Midtier Priority SNS",
   "description": "SNS destination (APM Deployment Managed)",
   "configType": "sns",
diff --git a/workflow-cli/configuration-opensearch/notification/oneteam-priority.sns.json b/workflow-cli/configuration-opensearch/notification/oneteam-priority.sns.json
@@ -1,5 +1,6 @@
 {
   "id": "oneteam-priority-sns",
+  "entity": "SnsOneTeamPriority",
   "name": "OneTeam Priority SNS",
   "description": "SNS destination (APM Deployment Managed)",
   "configType": "sns",
diff --git a/workflow-cli/configuration-opensearch/notification/wf-normal.sns.json b/workflow-cli/configuration-opensearch/notification/wf-normal.sns.json
@@ -1,11 +1,17 @@
 {
   "id": "wildfire-normal-sns",
+  "entity": "SnsWildfireNormal",
   "name": "Wildfire SNS",
   "description": "SNS destination (APM Deployment Managed)",
   "configType": "sns",
   "isEnabled": true,
   "sns": {
     "topicArn": "nress-prod-wildfire-normal",
-    "roleArn": "opensearch_sns_nress-prod"
+    "roleArn": "opensearch_sns_nress-prod",
+    "subscriptions": [{
+      "entity": "SnsWildfireNormalEmailWf",
+      "endpoint": "CSNRINFRAWMB@Victoria1.gov.bc.ca",
+      "protocol": "email"
+    }]
   }
 }
diff --git a/workflow-cli/configuration-opensearch/notification/wso2-alert.sns.json b/workflow-cli/configuration-opensearch/notification/wso2-alert.sns.json
@@ -0,0 +1,17 @@
+{
+  "id": "wso2-alert-sns",
+  "entity": "SnsWso2Alert",
+  "name": "WSO2 Alert",
+  "description": "SNS destination (APM Deployment Managed)",
+  "configType": "sns",
+  "isEnabled": true,
+  "sns": {
+    "topicArn": "nress-prod-wso2-alert",
+    "roleArn": "opensearch_sns_nress-prod",
+    "subscriptions": [{
+      "entity": "SnsWso2AlertEmailWf",
+      "endpoint": "CSNRINFRAWMB@Victoria1.gov.bc.ca",
+      "protocol": "email"
+    }]
+  }
+}
diff --git a/workflow-cli/src/commands/automation-message.ts b/workflow-cli/src/commands/automation-message.ts
@@ -38,7 +38,6 @@ export default class AutomationMessage extends Command {
 
   public async run(): Promise<void> {
     const { flags } = await this.parse(AutomationMessage);
-    // eslint-disable-next-line max-len
     const queueUrl = `https://sqs.${flags.region}.amazonaws.com/${flags.accountNumber}/${flags.domainName}-message-queue`;
 
     await AwsService.assumeIdentity(flags);
diff --git a/workflow-cli/src/commands/aws-render.ts b/workflow-cli/src/commands/aws-render.ts
@@ -0,0 +1,24 @@
+import 'reflect-metadata';
+import { Args, Command } from '@oclif/core';
+import { vsContainer } from '../inversify.config';
+import { TYPES } from '../inversify.types';
+import AwsRenderService from '../services/aws-render.service';
+
+export default class AwsRender extends Command {
+  static description = 'Renders AWS Cloudformation doc';
+
+  static examples = ['<%= config.bin %> <%= command.id %>'];
+
+  static flags = {};
+
+  static args = {
+    file: Args.string({ name: 'file' }),
+  };
+
+  public async run(): Promise<void> {
+    await this.parse(AwsRender);
+    this.log(`AWS Template render - start`);
+    vsContainer.get<AwsRenderService>(TYPES.AwsRenderService).render();
+    this.log(`AWS Template render - end`);
+  }
+}
diff --git a/workflow-cli/src/controller/aws-render.controller.ts b/workflow-cli/src/controller/aws-render.controller.ts
diff --git a/workflow-cli/src/inversify.config.ts b/workflow-cli/src/inversify.config.ts
@@ -9,6 +9,8 @@ import OpenSearchNotificationsService from './services/opensearch-notifications.
 import OpenSearchPolicyService from './services/opensearch-policy.service';
 import OpenSearchTemplateService from './services/opensearch-template.service';
 import VaultApi from './vault/vault.api';
+import AwsRenderService from './services/aws-render.service';
+import NotificationService from './services/notification.service';
 
 const vsContainer = new Container();
 // Controllers
@@ -18,6 +20,10 @@ vsContainer
 // Services
 vsContainer.bind<BrokerApi>(TYPES.BrokerApi).to(BrokerApi);
 vsContainer.bind<VaultApi>(TYPES.VaultApi).to(VaultApi);
+vsContainer.bind<AwsRenderService>(TYPES.AwsRenderService).to(AwsRenderService);
+vsContainer
+  .bind<NotificationService>(TYPES.NotificationService)
+  .to(NotificationService);
 vsContainer
   .bind<LambdaAssetDownloadService>(TYPES.LambdaAssetDownloadService)
   .to(LambdaAssetDownloadService);
diff --git a/workflow-cli/src/inversify.types.ts b/workflow-cli/src/inversify.types.ts
@@ -5,7 +5,9 @@ const TYPES = {
   BrokerApiUrl: Symbol.for('BrokerApiUrl'),
   BrokerToken: Symbol.for('BrokerToken'),
   OpenSearchController: Symbol.for('OpenSearchController'),
+  AwsRenderService: Symbol.for('AwsRenderService'),
   LambdaAssetDownloadService: Symbol.for('LambdaAssetDownloadService'),
+  NotificationService: Symbol.for('NotificationService'),
   OpenSearchDomainService: Symbol.for('OpenSearchDomainService'),
   OpenSearchTemplateService: Symbol.for('OpenSearchTemplateService'),
   OpenSearchPolicyService: Symbol.for('OpenSearchPolicyService'),
diff --git a/workflow-cli/src/services/aws-render.service.ts b/workflow-cli/src/services/aws-render.service.ts
@@ -0,0 +1,33 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import ejs from 'ejs';
+import { inject, injectable } from 'inversify';
+import NotificationService from './notification.service';
+import { TYPES } from '../inversify.types';
+
+const TEMPLATE_DIR = path.resolve(__dirname, '../../..');
+
+@injectable()
+export default class AwsRenderService {
+  constructor(
+    @inject(TYPES.NotificationService)
+    private notificationService: NotificationService,
+  ) {}
+
+  render() {
+    const templateStr = fs.readFileSync(
+      path.resolve(TEMPLATE_DIR, 'template.yaml.tpl'),
+      { encoding: 'utf8' },
+    );
+
+    const val = ejs.render(templateStr, {
+      notifications: this.notificationService.renderConfigs({}),
+    });
+
+    console.log(val);
+
+    fs.writeFileSync(path.resolve(TEMPLATE_DIR, 'template.yaml'), val, {
+      encoding: 'utf8',
+    });
+  }
+}
diff --git a/workflow-cli/src/services/notification.service.ts b/workflow-cli/src/services/notification.service.ts
@@ -0,0 +1,59 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import ejs from 'ejs';
+import { injectable } from 'inversify';
+
+export interface NotificationSettings {
+  hostname: string;
+  region: string;
+  accountNumber: string;
+}
+
+export interface NotificationConfig {
+  id: string;
+  name: string;
+  description: string;
+  configType: string;
+  isEnabled: boolean;
+  sns: {
+    topicArn: string;
+    roleArn: string;
+  };
+  microsoft_teams: {
+    url: string;
+  };
+}
+
+const NOTIFICATION_CONFIG_DIR = path.resolve(
+  __dirname,
+  '../../configuration-opensearch/notification',
+);
+
+@injectable()
+export default class NotificationService {
+  public renderConfigs(secrets: any): NotificationConfig[] {
+    const configs: NotificationConfig[] = [];
+    for (const notificationFile of fs.readdirSync(NOTIFICATION_CONFIG_DIR)) {
+      const configStr = fs.readFileSync(
+        path.resolve(NOTIFICATION_CONFIG_DIR, notificationFile),
+        { encoding: 'utf8' },
+      );
+
+      const config = JSON.parse(
+        this.renderConfig(notificationFile, configStr, secrets),
+      );
+      configs.push(config);
+    }
+    return configs;
+  }
+
+  private renderConfig(file: string, configStr: string, secrets: any) {
+    if (file.endsWith('microsoft_teams.json')) {
+      const key = `notification_teams_${file.slice(0, -21).replaceAll(/[^a-zA-Z_0-9]/gi, '_')}`;
+      return ejs.render(configStr, {
+        url: secrets[key] ?? '',
+      });
+    }
+    return configStr;
+  }
+}
diff --git a/workflow-cli/src/services/opensearch-notifications.service.ts b/workflow-cli/src/services/opensearch-notifications.service.ts

Original file line number	Diff line number	Diff line change
`@@ -121,3 +121,4 @@ terraform/.terraform.lock.hcl`
`121`	`121`	`workflow-cli/indicesusage*.csv`
`122`	`122`
`123`	`123`	`.aws-sam`
	`124`	`+/template.yaml`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"id": "appdlvr-normal-sns",`
	`3`	`+ "entity": "SnsAppDlvrNormal",`
`3`	`4`	`"name": "Appdlvr SNS",`
`4`	`5`	`"description": "SNS destination (APM Deployment Managed)",`
`5`	`6`	`"configType": "sns",`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,17 @@`
`1`	`1`	`{`
`2`	`2`	`"id": "appdlvr-priority-sns",`
	`3`	`+ "entity": "SnsAppDlvrPriority",`
`3`	`4`	`"name": "Appdlvr Priority SNS",`
`4`	`5`	`"description": "SNS destination (APM Deployment Managed)",`
`5`	`6`	`"configType": "sns",`
`6`	`7`	`"isEnabled": true,`
`7`	`8`	`"sns": {`
`8`	`9`	`"topicArn": "nress-prod-appdlvr-priority",`
`9`		`- "roleArn": "opensearch_sns_nress-prod"`
	`10`	`+ "roleArn": "opensearch_sns_nress-prod",`
	`11`	`+ "subscriptions": [{`
	`12`	`+ "entity": "SnsAppDlvrPriorityEmailApp",`
	`13`	`+ "endpoint": "nrids.applicationdelivery@gov.bc.ca",`
	`14`	`+ "protocol": "email"`
	`15`	`+ }]`
`10`	`16`	`}`
`11`	`17`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,11 +1,17 @@`
`1`	`1`	`{`
`2`	`2`	`"id": "automation-sqs-sns",`
	`3`	`+ "entity": "SnsAutomationSqs",`
`3`	`4`	`"name": "Automation SQS SNS",`
`4`	`5`	`"description": "SNS destination (APM Deployment Managed)",`
`5`	`6`	`"configType": "sns",`
`6`	`7`	`"isEnabled": true,`
`7`	`8`	`"sns": {`
`8`	`9`	`"topicArn": "nress-prod-message-queue",`
`9`		`- "roleArn": "opensearch_sns_nress-prod"`
	`10`	`+ "roleArn": "opensearch_sns_nress-prod",`
	`11`	`+ "subscriptions": [{`
	`12`	`+ "entity": "SqsAutomation",`
	`13`	`+ "endpoint": "nress-prod-message-queue",`
	`14`	`+ "protocol": "sqs"`
	`15`	`+ }]`
`10`	`16`	`}`
`11`	`17`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"id": "midtier-normal-sns",`
	`3`	`+ "entity": "SnsMidtierNormal",`
`3`	`4`	`"name": "Midtier SNS",`
`4`	`5`	`"description": "SNS destination (APM Deployment Managed)",`
`5`	`6`	`"configType": "sns",`