Skip to content

Commit 6d6db31

Browse files
NicolasDuchenneNicolasDuchenne
authored
Merge pull request #7 from NicolasDuchenne/feature/pre_commit_github_action
create pre commit workflow
2 parents 3b802e2 + f53f492 commit 6d6db31

File tree

5 files changed

+165
-22
lines changed

5 files changed

+165
-22
lines changed

‎.github/workflows/pre-commit.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: pre-commit
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches: [main]
7+
8+
jobs:
9+
pre-commit:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout code
13+
uses: actions/checkout@v3
14+
15+
- name: Install a specific version of uv
16+
uses: astral-sh/setup-uv@v5
17+
with:
18+
version: ">=0.4.0"
19+
- name: Install dependencies
20+
run: uv sync
21+
- name: Run pre-commit
22+
run: uv run pre-commit run --all-files

‎pipelines/config/config.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
from dotenv import load_dotenv
21
import os
2+
3+
from dotenv import load_dotenv
4+
35
current_dir = os.path.dirname(os.path.abspath(__file__))
46
# Construct the path to the .env file
5-
dotenv_path = os.path.join(current_dir, '.env')
7+
dotenv_path = os.path.join(current_dir, ".env")
8+
69

7-
# Load the .env file
8-
load_dotenv(dotenv_path)
10+
def load_env_variables():
11+
load_dotenv(dotenv_path)

‎pipelines/notebooks/test_storage_utils.ipynb‎

Lines changed: 128 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"cells": [
33
{
44
"cell_type": "code",
5-
"execution_count": 1,
5+
"execution_count": 13,
66
"id": "30627a53-3b54-4e00-98a8-9283034fa572",
77
"metadata": {},
88
"outputs": [],
@@ -14,18 +14,74 @@
1414
},
1515
{
1616
"cell_type": "code",
17-
"execution_count": 2,
18-
"id": "38862ce2-4723-41e3-ab29-bcf76e1248e7",
17+
"execution_count": 14,
18+
"id": "f10cbeca",
1919
"metadata": {},
2020
"outputs": [],
21+
"source": [
22+
"import pipelines.config.config as test"
23+
]
24+
},
25+
{
26+
"cell_type": "code",
27+
"execution_count": 15,
28+
"id": "1fb5871d",
29+
"metadata": {},
30+
"outputs": [
31+
{
32+
"data": {
33+
"text/plain": [
34+
"['__builtins__',\n",
35+
" '__cached__',\n",
36+
" '__doc__',\n",
37+
" '__file__',\n",
38+
" '__loader__',\n",
39+
" '__name__',\n",
40+
" '__package__',\n",
41+
" '__spec__',\n",
42+
" 'current_dir',\n",
43+
" 'dotenv_path',\n",
44+
" 'load_dotenv',\n",
45+
" 'os']"
46+
]
47+
},
48+
"execution_count": 15,
49+
"metadata": {},
50+
"output_type": "execute_result"
51+
}
52+
],
53+
"source": [
54+
"dir(test)"
55+
]
56+
},
57+
{
58+
"cell_type": "code",
59+
"execution_count": 16,
60+
"id": "38862ce2-4723-41e3-ab29-bcf76e1248e7",
61+
"metadata": {},
62+
"outputs": [
63+
{
64+
"ename": "ImportError",
65+
"evalue": "cannot import name 'load_env_variables' from 'pipelines.config.config' (C:\\Users\\nicol\\Documents\\DataForGood\\13_rendre_visible_pollution_eau_potable\\fork\\13_pollution_eau\\pipelines\\config\\config.py)",
66+
"output_type": "error",
67+
"traceback": [
68+
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
69+
"\u001b[1;31mImportError\u001b[0m Traceback (most recent call last)",
70+
"Cell \u001b[1;32mIn[16], line 2\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;66;03m# nécéssaire pour load le .env\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mpipelines\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m load_env_variables\n\u001b[0;32m 3\u001b[0m load_env_variables()\n",
71+
"\u001b[1;31mImportError\u001b[0m: cannot import name 'load_env_variables' from 'pipelines.config.config' (C:\\Users\\nicol\\Documents\\DataForGood\\13_rendre_visible_pollution_eau_potable\\fork\\13_pollution_eau\\pipelines\\config\\config.py)"
72+
]
73+
}
74+
],
2175
"source": [
2276
"# nécéssaire pour load le .env\n",
23-
"import pipelines.config.config"
77+
"from pipelines.config.config import load_env_variables\n",
78+
"\n",
79+
"load_env_variables()"
2480
]
2581
},
2682
{
2783
"cell_type": "code",
28-
"execution_count": 3,
84+
"execution_count": 17,
2985
"id": "d133d716-172f-4273-ad61-f4bc88cf9413",
3086
"metadata": {},
3187
"outputs": [],
@@ -44,7 +100,7 @@
44100
},
45101
{
46102
"cell_type": "code",
47-
"execution_count": 3,
103+
"execution_count": 18,
48104
"id": "f92361d8-8b48-40a4-a6c6-e91e9b5f0fae",
49105
"metadata": {},
50106
"outputs": [],
@@ -54,7 +110,7 @@
54110
},
55111
{
56112
"cell_type": "code",
57-
"execution_count": null,
113+
"execution_count": 19,
58114
"id": "6796b3fc-9332-429c-8217-6661dd7dd08c",
59115
"metadata": {},
60116
"outputs": [],
@@ -64,10 +120,71 @@
64120
},
65121
{
66122
"cell_type": "code",
67-
"execution_count": null,
123+
"execution_count": 20,
68124
"id": "eb314161-952b-4a68-875e-de52ce042578",
69125
"metadata": {},
70-
"outputs": [],
126+
"outputs": [
127+
{
128+
"data": {
129+
"text/html": [
130+
"<div>\n",
131+
"<style scoped>\n",
132+
" .dataframe tbody tr th:only-of-type {\n",
133+
" vertical-align: middle;\n",
134+
" }\n",
135+
"\n",
136+
" .dataframe tbody tr th {\n",
137+
" vertical-align: top;\n",
138+
" }\n",
139+
"\n",
140+
" .dataframe thead th {\n",
141+
" text-align: right;\n",
142+
" }\n",
143+
"</style>\n",
144+
"<table border=\"1\" class=\"dataframe\">\n",
145+
" <thead>\n",
146+
" <tr style=\"text-align: right;\">\n",
147+
" <th></th>\n",
148+
" <th>Name</th>\n",
149+
" <th>Age</th>\n",
150+
" <th>City</th>\n",
151+
" </tr>\n",
152+
" </thead>\n",
153+
" <tbody>\n",
154+
" <tr>\n",
155+
" <th>0</th>\n",
156+
" <td>Alice</td>\n",
157+
" <td>25</td>\n",
158+
" <td>New York</td>\n",
159+
" </tr>\n",
160+
" <tr>\n",
161+
" <th>1</th>\n",
162+
" <td>Bob</td>\n",
163+
" <td>30</td>\n",
164+
" <td>Los Angeles</td>\n",
165+
" </tr>\n",
166+
" <tr>\n",
167+
" <th>2</th>\n",
168+
" <td>Charlie</td>\n",
169+
" <td>35</td>\n",
170+
" <td>Chicago</td>\n",
171+
" </tr>\n",
172+
" </tbody>\n",
173+
"</table>\n",
174+
"</div>"
175+
],
176+
"text/plain": [
177+
" Name Age City\n",
178+
"0 Alice 25 New York\n",
179+
"1 Bob 30 Los Angeles\n",
180+
"2 Charlie 35 Chicago"
181+
]
182+
},
183+
"execution_count": 20,
184+
"metadata": {},
185+
"output_type": "execute_result"
186+
}
187+
],
71188
"source": [
72189
"storage_client.download_object(\"test/test.csv\", \"download_test.csv\")\n",
73190
"pd.read_csv(\"download_test.csv\")"
@@ -153,7 +270,7 @@
153270
],
154271
"metadata": {
155272
"kernelspec": {
156-
"display_name": ".venv",
273+
"display_name": "Python 3 (ipykernel)",
157274
"language": "python",
158275
"name": "python3"
159276
},
@@ -167,7 +284,7 @@
167284
"name": "python",
168285
"nbconvert_exporter": "python",
169286
"pygments_lexer": "ipython3",
170-
"version": "3.12.5"
287+
"version": "3.12.7"
171288
}
172289
},
173290
"nbformat": 4,

‎pipelines/tasks/_common.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,3 @@
1414
def clear_cache():
1515
"""Clear the cache folder."""
1616
shutil.rmtree(CACHE_FOLDER)
17-

‎pipelines/utils/storage_client.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ def __init__(self):
2020
@staticmethod
2121
def build_client(signature_version: str = "s3v4"):
2222
return boto3.session.Session().client(
23-
service_name='s3',
23+
service_name="s3",
2424
config=Config(signature_version=signature_version),
2525
region_name=ObjectStorageClient.region_name,
2626
use_ssl=True,
2727
endpoint_url=ObjectStorageClient.endpoint_url,
28-
aws_access_key_id=os.getenv('SCW_ACCESS_KEY'),
29-
aws_secret_access_key=os.getenv('SCW_SECRET_KEY'),
28+
aws_access_key_id=os.getenv("SCW_ACCESS_KEY"),
29+
aws_secret_access_key=os.getenv("SCW_SECRET_KEY"),
3030
)
3131

3232
# def list_buckets(self):
@@ -35,8 +35,8 @@ def build_client(signature_version: str = "s3v4"):
3535

3636
def list_objects(self):
3737
response = self.client_v4.list_objects(Bucket=self.bucket_name)
38-
if 'Contents' in response:
39-
return response['Contents']
38+
if "Contents" in response:
39+
return response["Contents"]
4040
else:
4141
return []
4242

@@ -53,7 +53,9 @@ def upload_dataframe(self, df, file_key):
5353
df.to_csv(csv_buffer, index=False)
5454

5555
# Upload the buffer to S3
56-
self.client_v2.put_object(Bucket=self.bucket_name, Key=file_key, Body=csv_buffer.getvalue())
56+
self.client_v2.put_object(
57+
Bucket=self.bucket_name, Key=file_key, Body=csv_buffer.getvalue()
58+
)
5759

5860
def read_object_as_dataframe(self, file_key):
5961
response = self.client_v4.get_object(Bucket=self.bucket_name, Key=file_key)

0 commit comments

Comments
 (0)