lambda template has been added (#72)

Author: NishanthMuruganantham

aws/mens_t20i_dataset_stack.py

@@ -50,6 +50,9 @@ def __init__(
 
         ########################################  SECRET MANAGER Configurations ##########################################
         __db_secrets = get_secret_from_secrets_manager(self._secret_manager_client, "db_secret")
+        __kaggle_secrets = get_secret_from_secrets_manager(self._secret_manager_client, "kaggle_credentials")
+
+        ########################################  LAMBDA LAYER Configurations ##########################################
 
         # Lambda layer containing the necessary code and packages
         package_layer = _lambda.LayerVersion(
@@ -236,7 +239,7 @@ def __init__(
                 package_layer,
                 pandas_layer,
             ],
-            memory_size=300,
+            memory_size=1024,
             timeout=Duration.minutes(10),
         )
         # Permissions for lambda functions to the S3 bucket
@@ -252,3 +255,35 @@ def __init__(
                 resources=["*"],
             )
         )
+
+        # Lambda function to upload the dataset to KAGGLE and create a new version of dataset
+        upload_dataset_to_kaggle_lambda = _lambda.Function(
+            self,
+            "upload_dataset_to_kaggle_lambda",
+            code=_lambda.Code.from_asset("output/upload_dataset_to_kaggle_lambda.zip"),
+            handler="upload_dataset_to_kaggle_lambda.handler",
+            runtime=_lambda.Runtime.PYTHON_3_11,
+            environment={
+                "DOWNLOAD_BUCKET_NAME": cricsheet_data_downloading_bucket.bucket_name,
+                **__kaggle_secrets,
+            },
+            function_name="upload-dataset-to-kaggle-lambda",
+            layers=[
+                package_layer,
+            ],
+            memory_size=300,
+            timeout=Duration.minutes(10),
+        )
+        # Permissions for lambda functions to the S3 bucket
+        cricsheet_data_downloading_bucket.grant_read(upload_dataset_to_kaggle_lambda)
+        # Policy for CloudWatch logging
+        upload_dataset_to_kaggle_lambda.add_to_role_policy(
+            iam.PolicyStatement(
+                actions=[
+                    "logs:CreateLogGroup",
+                    "logs:CreateLogStream",
+                    "logs:PutLogEvents",
+                ],
+                resources=["*"],
+            )
+        )

requirements.txt

@@ -1,2 +1,3 @@
+kaggle == 1.7.4.5
 pymongo == 4.10.1
 requests == 2.32.3

src/build/constants.py

@@ -12,6 +12,7 @@
     'src/mens_t20i_data_collector/_lambdas/download_from_cricsheet/download_from_cricsheet_lambda_function.py',
     'src/mens_t20i_data_collector/_lambdas/extract_deliverywise_cricsheet_data/extract_deliverywise_cricsheet_data_lambda_function.py',
     'src/mens_t20i_data_collector/_lambdas/extract_matchwise_cricsheet_data/extract_matchwise_cricsheet_data_lambda_function.py',
+    'src/mens_t20i_data_collector/_lambdas/upload_dataset_to_kaggle/upload_dataset_to_kaggle_lambda.py',
 ]
 PACKAGE_NAME = "mens_t20i_data_collector"
 PYTHON_VERSION = "3.11"

src/mens_t20i_data_collector/_lambdas/upload_dataset_to_kaggle/__init__.py

@@ -0,0 +1,14 @@
+import logging
+from mens_t20i_data_collector._lambdas.utils import exception_handler
+
+# Set up Logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+@exception_handler      # noqa: Vulture
+def handler():
+    """
+    Lambda function handler to upload dataset to Kaggle.
+    """
+    logger.info("Uploading dataset to Kaggle")
+    return "Dataset uploaded to Kaggle successfully"