Change environment from monthly to weekly (#49)

* Change environment from monthly to weekly

* Update consolidate upload path

* Move PyTorchLightning to weekly

* Make the upload of the consolidation be at the top level

Author: pvk-developer

.github/workflows/monthly_collection.yaml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   monthly_github_collection:
-    environment: monthly
+    environment: weekly
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v6
@@ -36,7 +36,7 @@ jobs:
     - name: Consolidate GitHub Data
       run: |
         uv run gitmetrics consolidate \
-          --config-file monthly_extraction_config.yaml \
+          --config-file weekly_extraction_config.yaml monthly_extraction_config.yaml \
           --output-folder ${{ secrets.OUTPUT_FOLDER }}
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}

.github/workflows/weekly_collection.yaml

@@ -32,7 +32,7 @@ jobs:
     - name: Consolidate GitHub Data
       run: |
         uv run gitmetrics consolidate \
-          --config-file weekly_extraction_config.yaml \
+          --config-file weekly_extraction_config.yaml monthly_extraction_config.yaml \
           --output-folder ${{ secrets.OUTPUT_FOLDER }}
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}

gitmetrics/__main__.py

@@ -140,11 +140,13 @@ def _summarize(args, parser):
 
 
 def _consolidate(args, parser):
-    config = _load_config(args.config_file)
-    projects = config['projects']
+    all_projects = []
+    for config_file in args.config_file:
+        config = _load_config(config_file)
+        all_projects.extend(config.get('projects', []))
 
     consolidate_metrics(
-        projects=projects,
+        projects=all_projects,
         output_folder=args.output_folder,
         dry_run=args.dry_run,
         verbose=args.verbose,
@@ -223,8 +225,9 @@ def _get_parser():
         '-c',
         '--config-file',
         type=str,
-        default=DEFAULT_PROJECT_DEFINITIONS,
-        help='Path to the configuration file.',
+        nargs='+',
+        default=[DEFAULT_PROJECT_DEFINITIONS],
+        help='Path(s) to configuration file(s).',
     )
     consolidate.add_argument(
         '-d',

gitmetrics/consolidate.py

@@ -12,6 +12,7 @@
     METRICS_SHEET_NAME,
     VALUE_COLUMN_NAME,
 )
+from gitmetrics.drive import _get_drive_client, is_drive_path, split_drive_path
 from gitmetrics.output import create_spreadsheet, load_spreadsheet
 
 OUTPUT_FILENAME = 'gitmetrics_consolidated_summary_to_date'
@@ -55,13 +56,31 @@ def consolidate_metrics(projects, output_folder, dry_run=False, verbose=True):
         row_info.update(row_values)
         if verbose:
             LOGGER.info(f' {project} values: {row_info}')
+
         rows.append(row_info)
 
     consolidated_df = pd.DataFrame(rows)
     sheets = {SHEET_NAME: consolidated_df}
     if verbose:
         LOGGER.info(f'Sheet Name: {SHEET_NAME}')
         LOGGER.info(consolidated_df.to_string())
+
     if not dry_run:
         output_path = os.path.join(output_folder, OUTPUT_FILENAME)
+
+    if is_drive_path(output_folder):
+        drive = _get_drive_client()
+        gdrive_folder = output_folder.rstrip('/') + '/'
+        folder_id, _ = split_drive_path(gdrive_folder)
+
+        folder = drive.CreateFile({'id': folder_id})
+        folder.FetchMetadata(fields='parents')
+
+        parents = folder.get('parents') or []
+        parent_id = parents[0].get('id')
+
+        output_path = f'gdrive://{parent_id}/{OUTPUT_FILENAME}'
+        create_spreadsheet(output_path=output_path, sheets=sheets)
+
+    else:
         create_spreadsheet(output_path=output_path, sheets=sheets)

gitmetrics/drive.py

@@ -98,7 +98,7 @@ def upload_spreadsheet(content, filename, folder):
     retries = 0
     while retries != MAX_UPLOAD_RETRIES:
         try:
-            if drive_file['mimeType'] == SPREADSHEET_MIMETYPE:
+            if drive_file.get('mimeType') == SPREADSHEET_MIMETYPE:
                 drive_file.Upload()
             else:
                 drive_file.Upload({'convert': True})

monthly_extraction_config.yaml

@@ -8,4 +8,3 @@ projects:
   scikit-learn:
   ray:
   airbyte:
-  PyTorchLightning:

weekly_extraction_config.yaml

@@ -11,3 +11,4 @@ projects:
   pycaret:
   snorkel:
   spacy:
+  PyTorchLightning: