Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions .github/workflows/mypy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
name: Type Check

on:
pull_request:
push:
branches:
- main

jobs:
mypy:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.11"

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install .[dev]
pip install mypy

- name: Run mypy
run: |
mypy --config-file augur/mypy.ini
2 changes: 1 addition & 1 deletion augur/application/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from typing import List, Any, Optional
import os
from augur.application.db.models import Config
from augur.application.db.util import execute_session_query, convert_type_of_value
from augur.application.db.helpers import execute_session_query, convert_type_of_value

def get_development_flag_from_config():

Expand Down
26 changes: 26 additions & 0 deletions augur/application/db/helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
from typing import Any, Callable, Optional
from augur.application.db.util import catch_operational_error

def execute_session_query(query: Any, query_type: str = "all") -> Any:
"""
Executes an SQLAlchemy ORM query with retries on OperationalError.

Args:
query: SQLAlchemy ORM query object.
query_type: One of 'all', 'one', 'first'.

Returns:
Query result according to query_type.
"""
func: Optional[Callable[[], Any]] = None

if query_type == "all":
func = query.all
elif query_type == "one":
func = query.one
elif query_type == "first":
func = query.first
else:
raise Exception(f"ERROR: Unsupported query type '{query_type}'")

return catch_operational_error(func)
4 changes: 2 additions & 2 deletions augur/application/db/lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import logging
import sqlalchemy as s
from sqlalchemy import func
from sqlalchemy.exc import DataError

Check warning on line 7 in augur/application/db/lib.py

View workflow job for this annotation

GitHub Actions / runner / pylint

[pylint] reported by reviewdog 🐶 W0611: Unused DataError imported from sqlalchemy.exc (unused-import) Raw Output: augur/application/db/lib.py:7:0: W0611: Unused DataError imported from sqlalchemy.exc (unused-import)
from sqlalchemy.dialects import postgresql
from sqlalchemy.exc import OperationalError
from psycopg2.errors import DeadlockDetected
Expand All @@ -13,7 +13,7 @@
from augur.application.db.models import Config, Repo, Commit, WorkerOauth, Issue, PullRequest, PullRequestReview, ContributorsAlias,UnresolvedCommitEmail, Contributor, CollectionStatus, UserGroup, RepoGroup
from augur.tasks.util.collection_state import CollectionState
from augur.application.db import get_session, get_engine
from augur.application.db.util import execute_session_query, convert_type_of_value
from augur.application.db.helpers import execute_session_query, convert_type_of_value
from augur.application.db.session import remove_duplicates_by_uniques, remove_null_characters_from_list_of_dicts

logger = logging.getLogger("db_lib")
Expand Down Expand Up @@ -167,7 +167,7 @@

try:
working_commits = fetchall_data_from_sql_text(query)
except:

Check warning on line 170 in augur/application/db/lib.py

View workflow job for this annotation

GitHub Actions / runner / pylint

[pylint] reported by reviewdog 🐶 W0702: No exception type(s) specified (bare-except) Raw Output: augur/application/db/lib.py:170:4: W0702: No exception type(s) specified (bare-except)
working_commits = []

return working_commits
Expand All @@ -183,7 +183,7 @@

try:
missing_commit_hashes = fetchall_data_from_sql_text(fetch_missing_hashes_sql)
except:

Check warning on line 186 in augur/application/db/lib.py

View workflow job for this annotation

GitHub Actions / runner / pylint

[pylint] reported by reviewdog 🐶 W0702: No exception type(s) specified (bare-except) Raw Output: augur/application/db/lib.py:186:4: W0702: No exception type(s) specified (bare-except)
missing_commit_hashes = []

return missing_commit_hashes
Expand All @@ -203,7 +203,7 @@
return session.query(CollectionStatus).filter(getattr(CollectionStatus,f"{collection_type}_status" ) == CollectionState.COLLECTING.value).count()


def facade_bulk_insert_commits(logger, records):

Check warning on line 206 in augur/application/db/lib.py

View workflow job for this annotation

GitHub Actions / runner / pylint

[pylint] reported by reviewdog 🐶 W0621: Redefining name 'logger' from outer scope (line 19) (redefined-outer-name) Raw Output: augur/application/db/lib.py:206:31: W0621: Redefining name 'logger' from outer scope (line 19) (redefined-outer-name)

with get_session() as session:

Expand Down Expand Up @@ -275,7 +275,7 @@
raise e


def batch_insert_contributors(logger, data: Union[List[dict], dict]) -> Optional[List[dict]]:

Check warning on line 278 in augur/application/db/lib.py

View workflow job for this annotation

GitHub Actions / runner / pylint

[pylint] reported by reviewdog 🐶 W0621: Redefining name 'logger' from outer scope (line 19) (redefined-outer-name) Raw Output: augur/application/db/lib.py:278:30: W0621: Redefining name 'logger' from outer scope (line 19) (redefined-outer-name)

batch_size = 1000

Expand All @@ -286,7 +286,7 @@



def bulk_insert_dicts(logger, data: Union[List[dict], dict], table, natural_keys: List[str], return_columns: Optional[List[str]] = None, string_fields: Optional[List[str]] = None, on_conflict_update:bool = True) -> Optional[List[dict]]:

Check warning on line 289 in augur/application/db/lib.py

View workflow job for this annotation

GitHub Actions / runner / pylint

[pylint] reported by reviewdog 🐶 W0621: Redefining name 'logger' from outer scope (line 19) (redefined-outer-name) Raw Output: augur/application/db/lib.py:289:22: W0621: Redefining name 'logger' from outer scope (line 19) (redefined-outer-name)

if isinstance(data, list) is False:

Expand Down Expand Up @@ -604,4 +604,4 @@
with get_session() as session:

return session.query(RepoGroup).filter(RepoGroup.rg_name == name).first()

2 changes: 1 addition & 1 deletion augur/application/db/models/augur_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@


from augur.application.db.models.base import Base
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query
from augur.application.db import get_session

DEFAULT_REPO_GROUP_ID = 1
Expand Down
80 changes: 34 additions & 46 deletions augur/application/db/util.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
from sqlalchemy.exc import OperationalError
import time
import logging
from typing import Callable, TypedDict, Any, List, Dict, Optional, Union
from sqlalchemy.exc import OperationalError
from augur.application.db.models.base import Base



def catch_operational_error(func):

def catch_operational_error(func: Callable[[], Any]) -> Any:
"""
Executes a function and retries up to 4 times if OperationalError occurs.
Implements exponential backoff starting with 240 seconds.
"""
attempts = 0
error = None
error: Optional[str] = None
timeout = 240

while attempts < 4:

# do the sleep here instead of instead of in the exception
# so it doesn't sleep after the last failed time
if attempts > 0:
#Do a 30% exponential backoff
time.sleep(timeout)
Expand All @@ -27,64 +29,50 @@

raise Exception(error)


def execute_session_query(query, query_type="all"):

func = None
if query_type == "all":
func = query.all
elif query_type == "one":
func = query.one
elif query_type == "first":
func = query.first
else:
raise Exception(f"ERROR: Unsupported query type '{query_type}'")

return catch_operational_error(func)



def convert_orm_list_to_dict_list(result):
new_list = []
def convert_orm_list_to_dict_list(result: List["Base"]) -> List[Dict[str, Any]]:
"""
Converts a list of ORM model instances to a list of dictionaries.
"""
new_list: List[Dict[str, Any]] = []

for row in result:
row_dict = row.__dict__
try:
del row_dict['_sa_instance_state']
except:
pass

row_dict = dict(row.__dict__) # Copy to avoid mutating the ORM instance
row_dict.pop("_sa_instance_state", None) # Remove SQLAlchemy internal state
new_list.append(row_dict)

return new_list

return new_list

class ConfigDict(TypedDict, total=False):
type: Optional[str]
value: Union[str, int, float, bool]

def convert_type_of_value(config_dict, logger=None):

data_type = config_dict["type"]
def convert_type_of_value(
config_dict: ConfigDict, logger: Optional[logging.Logger] = None
) -> ConfigDict:
"""
Converts the 'value' field in config_dict to the type specified in 'type'.
Supported types: str, int, bool, float.
"""
data_type: Optional[str] = config_dict.get("type")

if data_type == "str" or data_type is None:

Check warning on line 58 in augur/application/db/util.py

View workflow job for this annotation

GitHub Actions / runner / pylint

[pylint] reported by reviewdog 🐶 R1705: Unnecessary "elif" after "return", remove the leading "el" from "elif" (no-else-return) Raw Output: augur/application/db/util.py:58:4: R1705: Unnecessary "elif" after "return", remove the leading "el" from "elif" (no-else-return)
return config_dict

elif data_type == "int":
config_dict["value"] = int(config_dict["value"])

elif data_type == "bool":
value = config_dict["value"]

if value.lower() == "false":
config_dict["value"] = False
else:
config_dict["value"] = True
value = str(config_dict["value"]).lower()
config_dict["value"] = value != "false"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if value is exactly false (string), this will evaluate to true.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

wait did i get this wrong?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the earlier function (Not type safe one) already had that anything not false is True behavior. So according to me this must work i am not sure though .


elif data_type == "float":
config_dict["value"] = float(config_dict["value"])

else:
msg = f"Need to add support for {data_type} types to config"
if logger:
logger.error(f"Need to add support for {data_type} types to config")
logger.error(msg)
else:
print(f"Need to add support for {data_type} types to config")
print(msg)

return config_dict
2 changes: 1 addition & 1 deletion augur/application/logs.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import logging
import logging.config
import logging.handlers
from logging import FileHandler

Check warning on line 6 in augur/application/logs.py

View workflow job for this annotation

GitHub Actions / runner / pylint

[pylint] reported by reviewdog 🐶 W0611: Unused FileHandler imported from logging (unused-import) Raw Output: augur/application/logs.py:6:0: W0611: Unused FileHandler imported from logging (unused-import)
import os
from pathlib import Path
import shutil
Expand All @@ -12,7 +12,7 @@

from augur.application.db.models import Config
from augur.application.config import convert_type_of_value
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query

ROOT_AUGUR_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))

Expand All @@ -26,7 +26,7 @@
# get formatter for the specified log level
def getFormatter(logLevel):

if logLevel == logging.INFO:

Check warning on line 29 in augur/application/logs.py

View workflow job for this annotation

GitHub Actions / runner / pylint

[pylint] reported by reviewdog 🐶 R1705: Unnecessary "elif" after "return", remove the leading "el" from "elif" (no-else-return) Raw Output: augur/application/logs.py:29:4: R1705: Unnecessary "elif" after "return", remove the leading "el" from "elif" (no-else-return)
return logging.Formatter(fmt=SIMPLE_FORMAT_STRING)

elif logLevel == logging.DEBUG:
Expand Down
6 changes: 6 additions & 0 deletions augur/mypy.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[mypy]
files = augur/application/db/util.py

ignore_missing_imports = True
follow_imports = skip
disallow_untyped_defs = True
2 changes: 1 addition & 1 deletion augur/tasks/data_analysis/discourse_analysis/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@

from augur.tasks.init.celery_app import celery_app as celery
from augur.application.db.lib import get_session, get_repo_by_repo_git
from augur.application.db.models import Repo, DiscourseInsight

Check warning on line 12 in augur/tasks/data_analysis/discourse_analysis/tasks.py

View workflow job for this annotation

GitHub Actions / runner / pylint

[pylint] reported by reviewdog 🐶 W0611: Unused Repo imported from augur.application.db.models (unused-import) Raw Output: augur/tasks/data_analysis/discourse_analysis/tasks.py:12:0: W0611: Unused Repo imported from augur.application.db.models (unused-import)
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query
from augur.tasks.init.celery_app import AugurMlRepoCollectionTask

#import os, sys, time, requests, json
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from sqlalchemy.orm.exc import NoResultFound
from augur.application.db.models.augur_data import *
from augur.application.db.models.augur_operations import CollectionStatus
from augur.application.db.util import execute_session_query, convert_orm_list_to_dict_list
from augur.application.db.helpers import execute_session_query
from augur.application.db.lib import execute_sql, get_repo_by_repo_git

class GitCloneError(Exception):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
from .config import FacadeHelper as FacadeHelper
from augur.tasks.util.worker_util import calculate_date_weight_from_timestamps
from augur.application.db.lib import execute_sql, fetchall_data_from_sql_text, remove_working_commits_by_repo_id_and_hashes, remove_commits_by_repo_id_and_hashes, get_repo_by_repo_git, get_session
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query
#from augur.tasks.git.util.facade_worker.facade

def update_repo_log(logger, facade_helper, repos_id,status):
Expand Down
2 changes: 1 addition & 1 deletion augur/tasks/github/contributors.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from augur.tasks.github.util.github_paginator import hit_api
from augur.tasks.github.facade_github.tasks import *
from augur.application.db.models import Contributor
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query
from augur.application.db.lib import bulk_insert_dicts, get_session, batch_insert_contributors
from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth

Expand Down
2 changes: 1 addition & 1 deletion augur/tasks/github/detect_move/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from augur.tasks.github.util.util import parse_json_response
from datetime import datetime
from augur.tasks.util.collection_state import CollectionState
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query
from augur.application.db.lib import bulk_insert_dicts


Expand Down
2 changes: 1 addition & 1 deletion augur/tasks/github/pull_requests/commits_model/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from augur.tasks.github.util.github_data_access import GithubDataAccess, UrlNotFoundException ## URLNotFoundException added to deal with percolation of 404 errors when the commits are not anywhere for a PR already captured.
from augur.application.db.models import *
from augur.tasks.github.util.util import get_owner_repo
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query
from augur.application.db.lib import get_secondary_data_last_collected, get_updated_prs


Expand Down
2 changes: 1 addition & 1 deletion augur/tasks/github/pull_requests/files_model/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from augur.tasks.github.util.github_graphql_data_access import GithubGraphQlDataAccess, NotFoundException, InvalidDataException
from augur.application.db.models import *
from augur.tasks.github.util.util import get_owner_repo
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query
from augur.application.db.lib import get_secondary_data_last_collected, get_updated_prs


Expand Down
2 changes: 1 addition & 1 deletion augur/tasks/github/pull_requests/files_model/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from augur.tasks.github.util.github_task_session import GithubTaskManifest
from augur.tasks.init.celery_app import celery_app as celery
from augur.tasks.init.celery_app import AugurSecondaryRepoCollectionTask
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query

@celery.task(base=AugurSecondaryRepoCollectionTask)
def process_pull_request_files(repo_git: str, full_collection: bool) -> None:
Expand Down
2 changes: 1 addition & 1 deletion augur/tasks/github/pull_requests/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from augur.tasks.github.util.github_task_session import GithubTaskManifest
from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth
from augur.application.db.lib import get_session, get_repo_by_repo_git, bulk_insert_dicts, get_pull_request_reviews_by_repo_id, batch_insert_contributors
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query
from ..messages import process_github_comment_contributors
from augur.application.db.lib import get_secondary_data_last_collected, get_updated_prs, get_core_data_last_collected

Expand Down
2 changes: 1 addition & 1 deletion augur/tasks/github/releases/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from augur.application.db.models import *
from augur.tasks.github.util.util import get_owner_repo
from augur.tasks.github.util.gh_graphql_entities import request_graphql_dict
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query
from augur.application.db.lib import bulk_insert_dicts


Expand Down
2 changes: 1 addition & 1 deletion augur/tasks/util/collection_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from augur.application.logs import AugurLogger
from augur.tasks.init.celery_app import celery_app as celery
from augur.application.db.models import CollectionStatus, Repo
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query
from augur.application.db.lib import get_section
from augur.tasks.github.util.util import get_repo_weight_core, get_repo_weight_by_issue
from augur.application.db import get_engine
Expand Down
2 changes: 1 addition & 1 deletion augur/util/repo_load_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from augur.application.db.engine import DatabaseEngine
from augur.application.db.models import Repo, UserRepo, RepoGroup, UserGroup, User, CollectionStatus
from augur.application.db.models.augur_operations import retrieve_owner_repos
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query

from sqlalchemy import Column, Table, MetaData, or_
from sqlalchemy.sql.operators import ilike_op
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ dev = [
"pytest==6.2.5",
"toml>=0.10.2",
"ipdb==0.13.9",
"mypy>=1.8.0",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if you change this file, you also need to run uv sync and commit the changes to the uv.lock file. this will likely help some of the CI jobs pass

{include-group = "docs"},
]
docs = [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from augur.application.db.session import DatabaseSession
from augur.application.db.models import Config, User
from augur.tasks.github.util.github_paginator import hit_api
from augur.application.db.util import execute_session_query
from augur.application.db.helpers import execute_session_query


logger = logging.getLogger(__name__)
Expand Down
Loading
Loading