Latest Augur Stable Release

augur/application/config.py

@@ -4,7 +4,7 @@
 from typing import List, Any, Optional
 import os
 from augur.application.db.models import Config 
-from augur.application.db.util import execute_session_query
+from augur.application.db.util import execute_session_query, convert_type_of_value
 
 def get_development_flag_from_config():
 
@@ -109,35 +109,6 @@ def get_development_flag():
         }
 
 
-def convert_type_of_value(config_dict, logger=None):
-
-        data_type = config_dict["type"]
-
-        if data_type == "str" or data_type is None:
-            return config_dict
-
-        elif data_type == "int":
-            config_dict["value"] = int(config_dict["value"])
-
-        elif data_type == "bool":
-            value = config_dict["value"]
-
-            if value.lower() == "false":
-                config_dict["value"] = False
-            else:
-                config_dict["value"] = True
-
-        elif data_type == "float":
-            config_dict["value"] = float(config_dict["value"])
-
-        else:
-            if logger:
-                logger.error(f"Need to add support for {data_type} types to config") 
-            else:
-                print(f"Need to add support for {data_type} types to config")
-
-        return config_dict
-
 class AugurConfig():
 
     from augur.application.db.session import DatabaseSession

augur/application/db/lib.py

@@ -4,7 +4,7 @@
 import logging
 import sqlalchemy as s
 from sqlalchemy import func 
 from sqlalchemy.exc import DataError
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.exc import OperationalError
 from psycopg2.errors import DeadlockDetected
@@ -13,41 +13,11 @@
 from augur.application.db.models import Config, Repo, Commit, WorkerOauth, Issue, PullRequest, PullRequestReview, ContributorsAlias,UnresolvedCommitEmail, Contributor, CollectionStatus, UserGroup, RepoGroup
 from augur.tasks.util.collection_state import CollectionState
 from augur.application.db import get_session, get_engine
-from augur.application.db.util import execute_session_query
+from augur.application.db.util import execute_session_query, convert_type_of_value
 from augur.application.db.session import remove_duplicates_by_uniques, remove_null_characters_from_list_of_dicts
 
 logger = logging.getLogger("db_lib")
 
-def convert_type_of_value(config_dict, logger=None):
-
-
-    data_type = config_dict["type"]
-
-    if data_type == "str" or data_type is None:
-        return config_dict
-
-    if data_type == "int":
-        config_dict["value"] = int(config_dict["value"])
-
-    elif data_type == "bool":
-        value = config_dict["value"]
-
-        if value.lower() == "false":
-            config_dict["value"] = False
-        else:
-            config_dict["value"] = True
-
-    elif data_type == "float":
-        config_dict["value"] = float(config_dict["value"])
-
-    else:
-        if logger:
-            logger.error(f"Need to add support for {data_type} types to config") 
-        else:
-            print(f"Need to add support for {data_type} types to config")
-
-    return config_dict
-
 
 def get_section(section_name) -> dict:
     """Get a section of data from the config.
@@ -197,7 +167,7 @@
 
     try:
         working_commits = fetchall_data_from_sql_text(query)
     except:
         working_commits = []
 
     return working_commits
@@ -213,7 +183,7 @@
 
     try:
         missing_commit_hashes = fetchall_data_from_sql_text(fetch_missing_hashes_sql)
     except:
         missing_commit_hashes = []
 
     return missing_commit_hashes
@@ -233,7 +203,7 @@
         return session.query(CollectionStatus).filter(getattr(CollectionStatus,f"{collection_type}_status" ) == CollectionState.COLLECTING.value).count()
 
 
 def facade_bulk_insert_commits(logger, records):
 
     with get_session() as session:
 
@@ -255,24 +225,46 @@
 
                 facade_bulk_insert_commits(logger, firsthalfRecords)
                 facade_bulk_insert_commits(logger, secondhalfRecords)
-            elif len(records) == 1 and isinstance(e,DataError) and "time zone displacement" in f"{e}":
+            elif len(records) == 1:
                 commit_record = records[0]
                 #replace incomprehensible dates with epoch.
                 #2021-10-11 11:57:46 -0500
 
                 # placeholder_date = "1970-01-01 00:00:15 -0500"
-                placeholder_date = commit_record['author_timestamp']
+                placeholder_date = commit_record['cmt_author_timestamp']
+
+                postgres_valid_timezones = {
+                    -1200, -1100, -1000, -930, -900, -800, -700,
+                    -600, -500, -400, -300, -230, -200, -100, 000,
+                    100, 200, 300, 330, 400, 430, 500, 530, 545, 600,
+                    630, 700, 800, 845, 900, 930, 1000, 1030, 1100, 1200,
+                    1245, 1300, 1400
+                }
 
                 # Reconstruct timezone portion of the date string to UTC
-                placeholder_date = re.split("[-+]", placeholder_date)
-                placeholder_date.pop()
-                placeholder_date = "-".join(placeholder_date) + "+0000"
+                placeholder_date_segments = re.split(" ", placeholder_date)
+                tzdata = placeholder_date_segments.pop()
+
+                if ":" in tzdata:
+                    tzdata = tzdata.replace(":", "")
+
+                if int(tzdata) not in postgres_valid_timezones:
+                    tzdata = "+0000"
+                else:
+                    raise e
+
+                placeholder_date_segments.append(tzdata)
+
+                placeholder_date = " ".join(placeholder_date_segments)
 
                 #Check for improper utc timezone offset
                 #UTC timezone offset should be between -14:00 and +14:00
 
-                commit_record['author_timestamp'] = placeholder_date
-                commit_record['committer_timestamp'] = placeholder_date
+                # analyzecommit.generate_commit_record() defines the keys on the commit_record dictionary
+                commit_record['cmt_author_timestamp'] = placeholder_date
+                commit_record['cmt_committer_timestamp'] = placeholder_date
+
+                logger.warning(f"commit with invalid timezone set to UTC: {commit_record['cmt_commit_hash']}")
 
                 session.execute(
                     s.insert(Commit),
@@ -283,7 +275,7 @@
                 raise e
 
 
 def batch_insert_contributors(logger, data: Union[List[dict], dict]) -> Optional[List[dict]]:
 
     batch_size = 1000
 
@@ -294,7 +286,7 @@
 
 
 
 def bulk_insert_dicts(logger, data: Union[List[dict], dict], table, natural_keys: List[str], return_columns: Optional[List[str]] = None, string_fields: Optional[List[str]] = None, on_conflict_update:bool = True) -> Optional[List[dict]]:
 
     if isinstance(data, list) is False:
 

augur/application/db/util.py

@@ -51,10 +51,40 @@
         row_dict = row.__dict__
         try:
             del row_dict['_sa_instance_state']
         except:
             pass
 
         new_list.append(row_dict)
 
     return new_list
 
+
+
+def convert_type_of_value(config_dict, logger=None):
+
+    data_type = config_dict["type"]
+
+    if data_type == "str" or data_type is None:
+        return config_dict
+
+    elif data_type == "int":
+        config_dict["value"] = int(config_dict["value"])
+
+    elif data_type == "bool":
+        value = config_dict["value"]
+
+        if value.lower() == "false":
+            config_dict["value"] = False
+        else:
+            config_dict["value"] = True
+
+    elif data_type == "float":
+        config_dict["value"] = float(config_dict["value"])
+
+    else:
+        if logger:
+            logger.error(f"Need to add support for {data_type} types to config") 
+        else:
+            print(f"Need to add support for {data_type} types to config")
+
+    return config_dict

augur/application/util.py

@@ -24,4 +24,4 @@ def get_all_repos_count(**kwargs):
 
         result = controller.get_repo_count(source="all", **kwargs)
 
-        return result
+        return result