Skip to content

Commit 8af2d90

Browse files
committed
Fix aws deployment
1 parent 9b45571 commit 8af2d90

13 files changed

+331
-54
lines changed

investing_algorithm_framework/__init__.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
TradingDataType, TradingTimeFrame, OrderType, OperationalException, \
88
OrderStatus, OrderSide, TimeUnit, TimeInterval, Order, Portfolio, \
99
Position, TimeFrame, BACKTESTING_INDEX_DATETIME, MarketCredential, \
10-
PortfolioConfiguration, RESOURCE_DIRECTORY, \
10+
PortfolioConfiguration, RESOURCE_DIRECTORY, AWS_LAMBDA_LOGGING_CONFIG, \
1111
Trade, OHLCVMarketDataSource, OrderBookMarketDataSource, SYMBOLS, \
1212
TickerMarketDataSource, MarketService, \
1313
RESERVED_BALANCES, APP_MODE, AppMode, DATETIME_FORMAT, \
@@ -92,5 +92,6 @@
9292
"add_metrics",
9393
"add_html_report",
9494
"AWSS3StorageStateHandler",
95-
"AWS_S3_STATE_BUCKET_NAME"
95+
"AWS_S3_STATE_BUCKET_NAME",
96+
"AWS_LAMBDA_LOGGING_CONFIG"
9697
]

investing_algorithm_framework/cli/cli.py

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -177,12 +177,17 @@ def deploy_azure_function(
177177
default=None,
178178
help='List of directories to ignore when deploying.'
179179
)
180+
@click.option(
181+
'--memory_size',
182+
default=3000,
183+
type=int,
184+
help='The memory size for the Lambda function in MB. Default is 3000 MB.'
185+
)
180186
def deploy_aws_lambda(
181187
lambda_function_name,
182188
region,
183-
lambda_handler,
184189
project_dir=None,
185-
ignore_dirs=None
190+
memory_size=3000
186191
):
187192
"""
188193
Command-line tool for deploying a trading bot to AWS lambda
@@ -192,22 +197,20 @@ def deploy_aws_lambda(
192197
to deploy.
193198
region (str): The AWS region where the Lambda function will
194199
be deployed.
195-
lambda_handler (str): The Lambda handler function in the format
196-
"module_name.function_name".
197200
project_dir (str): The path to the project directory containing the
198201
Lambda function code. If not provided, it defaults to
199202
the current directory.
200-
ignore_dirs (list): List of directories to ignore when deploying.
203+
memory_size (int): The memory size for the Lambda function in MB.
204+
Default is 3000 MB.
201205
202206
Returns:
203207
None
204208
"""
205209
deploy_to_aws_lambda_command(
206210
lambda_function_name=lambda_function_name,
207211
region=region,
208-
lambda_handler=lambda_handler,
209212
project_dir=project_dir,
210-
ignore_dirs=ignore_dirs
213+
memory_size=memory_size
211214
)
212215

213216

investing_algorithm_framework/cli/deploy_to_aws_lambda.py

Lines changed: 112 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import json
22
import os
33
import re
4+
import subprocess
45
import time
5-
import uuid
66
import zipfile
77

88
import boto3
@@ -126,9 +126,9 @@ def create_iam_role(role_name):
126126
def deploy_lambda(
127127
function_name,
128128
region,
129-
zip_file_path,
130-
handler_name,
129+
image_uri,
131130
role_arn,
131+
memory_size,
132132
runtime="python3.10",
133133
env_vars=None,
134134
):
@@ -138,32 +138,29 @@ def deploy_lambda(
138138
Args:
139139
function_name: str, the name of the Lambda function to
140140
create or update.
141-
zip_file_path: str, the path to the zip file containing
142-
the Lambda function code.
143141
region: str, the AWS region where the Lambda
144142
function will be deployed.
145-
handler_name: str, the name of the handler function
146-
in the code (e.g., "main.lambda_handler").
143+
image_uri: str, the URI of the Docker image in ECR.
147144
role_arn: str, the ARN of the IAM role that Lambda will assume.
148145
runtime: str, the runtime environment for the
149146
Lambda function (default is "python3.10").
150147
env_vars: dict, optional environment variables
151148
to set for the Lambda function.
149+
memory_size: int, the amount of memory allocated
150+
to the Lambda function.
151+
152152
Returns:
153153
None
154154
"""
155155
lambda_client = boto3.client('lambda', region_name=region)
156156

157-
with open(zip_file_path, 'rb') as f:
158-
zipped_code = f.read()
159-
160157
try:
161158
lambda_client.get_function(FunctionName=function_name)
162159
click.echo(f"Function {function_name} already exists. Updating...")
163160

164161
lambda_client.update_function_code(
165162
FunctionName=function_name,
166-
ZipFile=zipped_code
163+
ImageUri=image_uri
167164
)
168165
wait_for_lambda_update(lambda_client, function_name, timeout=120)
169166
lambda_client.update_function_configuration(
@@ -174,15 +171,18 @@ def deploy_lambda(
174171
click.echo(f"Creating new function: {function_name}")
175172

176173
try:
174+
click.echo(
175+
"Creating new container-based "
176+
f"Lambda function: {function_name}"
177+
)
177178
lambda_client.create_function(
178179
FunctionName=function_name,
179-
Runtime=runtime,
180180
Role=role_arn,
181-
Handler=handler_name,
182-
Code={'ZipFile': zipped_code},
181+
PackageType="Image",
182+
Code={"ImageUri": image_uri},
183183
Timeout=900,
184-
MemorySize=256,
185-
Environment={'Variables': env_vars or {}}
184+
MemorySize=memory_size,
185+
Environment={"Variables": env_vars or {}}
186186
)
187187
except Exception as e:
188188
raise click.ClickException(
@@ -213,6 +213,82 @@ def s3_bucket_exists(bucket_name, region):
213213
raise
214214

215215

216+
def create_ecr_repository(repository_name, region):
217+
"""
218+
Function to create an ECR repository for storing Docker images.
219+
It checks if the repository already exists and creates it if not.
220+
221+
Args:
222+
repository_name: str, the name of the ECR repository to create.
223+
region: str, the AWS region where the repository will be created.
224+
225+
Returns:
226+
None
227+
"""
228+
229+
ecr = boto3.client('ecr', region_name=region)
230+
try:
231+
response = ecr.create_repository(repositoryName=repository_name)
232+
click.echo(
233+
"Created ECR repository: "
234+
f"{response['repository']['repositoryUri']}"
235+
)
236+
except ecr.exceptions.RepositoryAlreadyExistsException:
237+
click.echo(f"ECR repository {repository_name} already exists.")
238+
239+
240+
def build_and_push_docker_image(
241+
repository_name, region, dockerfile_path='Dockerfile', tag='latest'
242+
):
243+
"""
244+
Function to build a Docker image and push it to an ECR repository.
245+
246+
Args:
247+
repository_name: str, the name of the ECR repository.
248+
region: str, the AWS region where the repository is located.
249+
dockerfile_path: str, path to the Dockerfile (default is 'Dockerfile').
250+
tag: str, the tag for the Docker image (default is 'latest').
251+
252+
Returns:
253+
None
254+
"""
255+
256+
# Retrieve the ECR repository URI
257+
ecr = boto3.client('ecr', region_name=region)
258+
try:
259+
response = ecr.describe_repositories(repositoryNames=[repository_name])
260+
repository_uri = response['repositories'][0]['repositoryUri']
261+
except ecr.exceptions.RepositoryNotFoundException:
262+
raise click.ClickException(
263+
f"ECR repository {repository_name} does "
264+
f"not exist in region {region}."
265+
)
266+
267+
# Authenticate Docker to the ECR registry
268+
auth = ecr.get_authorization_token()
269+
proxy = auth['authorizationData'][0]['proxyEndpoint']
270+
271+
click.echo(f"Authenticating Docker to ECR repository {repository_name}...")
272+
subprocess.run(
273+
f"aws ecr get-login-password --region {region} | "
274+
f"docker login --username AWS --password-stdin {proxy}",
275+
shell=True, check=True
276+
)
277+
278+
click.echo(f"Building Docker image {repository_name}:{tag}...")
279+
# Build and push Docker image with the docker file path
280+
image_full_uri = f"{repository_uri}:{tag}"
281+
subprocess.run([
282+
"docker", "build",
283+
"--platform=linux/amd64",
284+
"-t", image_full_uri,
285+
"-f", dockerfile_path,
286+
"."
287+
], check=True)
288+
subprocess.run(f"docker push {image_full_uri}", shell=True, check=True)
289+
return image_full_uri
290+
291+
216292
def create_s3_bucket(bucket_name, region):
217293
"""
218294
Function to create an S3 bucket for storing Lambda function code.
@@ -273,7 +349,8 @@ def check_lambda_permissions(required_actions=None):
273349
"lambda:GetFunction",
274350
"lambda:UpdateFunctionCode",
275351
"lambda:UpdateFunctionConfiguration",
276-
"lambda:CreateFunction"
352+
"lambda:CreateFunction",
353+
"ecr:CreateRepository"
277354
]
278355

279356
sts = boto3.client("sts")
@@ -356,9 +433,8 @@ def wait_for_lambda_update(lambda_client, function_name, timeout=60):
356433
def command(
357434
lambda_function_name,
358435
region,
359-
lambda_handler,
360436
project_dir=None,
361-
ignore_dirs=None
437+
memory_size=3000
362438
):
363439
"""
364440
Command-line tool for deploying a trading bot to AWS Lambda.
@@ -368,9 +444,8 @@ def command(
368444
region: str, the AWS region where the Lambda function will be deployed.
369445
project_dir: str, the directory containing the Lambda function code.
370446
If None, it defaults to the current directory.
371-
lambda_handler: str, the name of the handler function in the code
372-
(default is "aws_function.lambda_handler").
373-
ignore_dirs: list, directories to ignore when zipping the code.
447+
memory_size: int, the amount of memory allocated
448+
to the Lambda function
374449
375450
Returns:
376451
None
@@ -381,12 +456,11 @@ def command(
381456

382457
check_lambda_permissions()
383458

384-
click.echo(f"Deploying to AWS Lambda "
385-
f"function: {lambda_function_name} in region: {region}")
459+
click.echo(
460+
"Deploying to AWS Lambda "
461+
f"function: {lambda_function_name} in region: {region}"
462+
)
386463
click.echo(f"Project directory: {project_dir}")
387-
zip_file_path = f"/tmp/deploy-{uuid.uuid4().hex}.zip"
388-
zip_code(project_dir, zip_file_path)
389-
click.echo(f"Zipped code to {zip_file_path}")
390464

391465
# Create s3 bucket for state handler
392466
bucket_name = f"{lambda_function_name}-state-handler-{region}"
@@ -405,13 +479,21 @@ def command(
405479
click.echo("Adding S3 bucket name to environment variables")
406480
env_vars[AWS_S3_STATE_BUCKET_NAME] = bucket_name
407481

482+
click.echo("Building and pushing Docker image to ECR")
483+
create_ecr_repository(lambda_function_name, region)
484+
image_uri = build_and_push_docker_image(
485+
lambda_function_name,
486+
region,
487+
dockerfile_path=os.path.join(project_dir, "Dockerfile"),
488+
tag="latest"
489+
)
408490
click.echo("Creating IAM role for Lambda execution")
409491
role_arn = create_iam_role("lambda-execution-role")
410492
deploy_lambda(
411493
lambda_function_name,
412-
zip_file_path=zip_file_path,
413-
handler_name=lambda_handler,
494+
image_uri=image_uri,
414495
role_arn=role_arn,
415496
env_vars=env_vars,
416-
region=region
497+
region=region,
498+
memory_size=memory_size
417499
)

investing_algorithm_framework/cli/initialize_app.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,16 @@ def create_aws_lambda_app(path=None, replace=False):
372372
"templates",
373373
"app_aws_lambda_function.py.template"
374374
)
375+
aws_dockerfile_template_path = os.path.join(
376+
os.path.dirname(current_script_path),
377+
"templates",
378+
"aws_lambda_dockerfile.template"
379+
)
380+
aws_dockerignore_template_path = os.path.join(
381+
os.path.dirname(current_script_path),
382+
"templates",
383+
"aws_lambda_dockerignore.template"
384+
)
375385
run_backtest_template_path = os.path.join(
376386
os.path.dirname(current_script_path),
377387
"templates",
@@ -446,6 +456,16 @@ def create_aws_lambda_app(path=None, replace=False):
446456
os.path.join(path, "aws_function.py"),
447457
replace=replace
448458
)
459+
create_file_from_template(
460+
aws_dockerfile_template_path,
461+
os.path.join(path, "Dockerfile"),
462+
replace=replace
463+
)
464+
create_file_from_template(
465+
aws_dockerignore_template_path,
466+
os.path.join(path, ".dockerignore"),
467+
replace=replace
468+
)
449469

450470

451471
def create_azure_function_app(path=None, replace=False):

investing_algorithm_framework/cli/templates/app_aws_lambda_function.py.template

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,20 @@
1+
import logging.config
12
import os
2-
from investing_algorithm_framework import create_app, \
3-
AWSS3StorageStateHandler, AWS_S3_STATE_BUCKET_NAME
3+
from logging import getLogger
44

5-
from .strategies.strategy import MyTradingStrategy
6-
app = create_app()
5+
from dotenv import load_dotenv
6+
from finterion_investing_algorithm_framework import FinterionOrderExecutor, \
7+
FinterionPingAction, FinterionPortfolioProvider
8+
from investing_algorithm_framework import create_app, RESOURCE_DIRECTORY, \
9+
AWSS3StorageStateHandler, AWS_S3_STATE_BUCKET_NAME, AWS_LAMBDA_LOGGING_CONFIG
10+
11+
from strategies.strategy import MyTradingStrategy
12+
13+
14+
# Make sure to set the resource directory to /tmp because this dir is writable
15+
app = create_app(config={RESOURCE_DIRECTORY: os.path.join("/tmp", "resources")})
16+
logging.config.dictConfig(AWS_LAMBDA_LOGGING_CONFIG)
17+
logger = logging.getLogger(__name__)
718

819
# Get the s3 state bucket name from environment variables for database
920
# state storage, this is set during the deployment
@@ -12,7 +23,7 @@ app.add_state_handler(
1223
AWSS3StorageStateHandler(bucket_name=os.getenv(AWS_S3_STATE_BUCKET_NAME))
1324
)
1425
app.add_strategy(MyTradingStrategy)
15-
app.add_market(market="BITVAVO", trading_symbol="EUR", initial_balance=1000)
26+
app.add_market(market="BITVAVO", trading_symbol="EUR")
1627

1728

1829
def lambda_handler(event, context):
@@ -27,10 +38,11 @@ def lambda_handler(event, context):
2738
dict: The result of the trading strategy execution.
2839
"""
2940
try:
30-
app.run(number_of_iterations=1)
41+
app.run(payload={"ACTION": "RUN_STRATEGY"})
3142
return {
3243
"statusCode": 200,
3344
"body": "Trading strategy executed successfully."
3445
}
3546
except Exception as e:
47+
logger.exception(e)
3648
return {"statusCode": 500, "body": str(e)}

0 commit comments

Comments
 (0)