Refactor.

Author: toddbirchard

Pipfile.lock

@@ -1,4 +1,5 @@
 from config import Config
+from loguru import logger
 from .jirafetch import FetchJiraIssues
 from .transform import TransformData
 from .db import Database
@@ -9,7 +10,7 @@ def init_script():
     issues, epics = fetch_jira_issues()
     issues, epics = clean_jira_issues(issues, epics)
     upload = upload_issues(issues, epics)
-    print(upload)
+    logger.info(upload)
 
 
 def fetch_jira_issues():
@@ -22,7 +23,7 @@ def fetch_jira_issues():
 
 def clean_jira_issues(issues, epics):
     """Clean data and create pandas DataFrame."""
-    print('Transforming JIRA issues to tabular data...')
+    logger.info('Transforming JIRA issues to tabular data...')
     transform_data = TransformData()
     issues_df = transform_data.construct_dataframe(issues)
     epics_df = transform_data.construct_dataframe(epics)
@@ -31,7 +32,7 @@ def clean_jira_issues(issues, epics):
 
 def upload_issues(issues, epics):
     """Upload issues table to SQL database."""
-    print("Preparing database upload...")
+    logger.info("Preparing database upload...")
     db = Database(Config)
     epics_upload = db.upload_epics(epics)
     issues_upload = db.upload_issues(issues)

src/__init__.py renamed to jira_database_etl/__init__.py

@@ -0,0 +1,86 @@
+from loguru import logger
+from sqlalchemy import create_engine, text
+from sqlalchemy.types import Integer, Text, String
+import pandas as pd
+
+
+class Database:
+    """Merge Epic metadata and upload JIRA issues."""
+
+    def __init__(self, Config):
+        self.db_uri = Config.db_uri
+        self.db_jira_table = Config.db_jira_table
+        self.db_epic_table = Config.db_epic_table
+        self.engine = create_engine(
+            self.db_uri,
+            echo=False
+        )
+
+    def upload_epics(self, epics_df):
+        """Create SQL table of JIRA epics."""
+        self.__truncate_table(self.db_epic_table)
+        result = self.__upload_dataframe(epics_df, self.db_epic_table)
+        return result
+
+    def upload_issues(self, issues_df):
+        """Create SQL table of JIRA issues & JOIN with epic info."""
+        self.__truncate_table(self.db_jira_table)
+        issues_df = self.__merge_epic_metadata(issues_df)
+        result = self.__upload_dataframe(issues_df, self.db_jira_table)
+        return result
+
+    def __truncate_table(self, table):
+        """Clear existing SQL table."""
+        sql = text(f'TRUNCATE TABLE {table}')
+        self.engine.execute(sql)
+
+    def __merge_epic_metadata(self, jira_issues_df):
+        """Merge epic metadata from existing SQL table."""
+        epics_df = pd.read_sql_table(
+            self.db_epic_table,
+            self.engine
+        )
+        logger.info(epics_df)
+        jira_issues_df = pd.merge(
+            jira_issues_df,
+            epics_df[['key', 'epic_name']],
+            how='left',
+            left_on='epic_link',
+            right_on='key',
+            copy=False
+        )
+        jira_issues_df = jira_issues_df.rename(columns={"key_x": "key", "key_y": "epic_key"})
+        return jira_issues_df
+
+    def __upload_dataframe(self, issues_df, table):
+        """Upload JIRA DataFrame to database."""
+        issues_df.to_sql(
+            table,
+            self.engine,
+            if_exists='replace',
+            chunksize=500,
+            index=False,
+            dtype={
+                "id": Integer,
+                "assignee_name": String(30),
+                "epic_link": String(50),
+                "issuetype": String(50),
+                "key": String(10),
+                "priority_name": String(30),
+                "priority_rank": Integer,
+                "project": String(70),
+                "status": String(30),
+                "summary": Text,
+                "resolution": String(70),
+                "story_points": Integer,
+                "labels": Text,
+                "updated": Integer,
+                "created": Integer,
+                'sprint_status': String(255),
+                'sprint_name': Text,
+                'sprint_goal': Text,
+                "epic_name": String(100)
+            }
+        )
+        success_message = f'Uploaded {len(issues_df)} rows to {table} table.'
+        return success_message

jira_database_etl/db.py

@@ -1,5 +1,6 @@
 import math
 import requests
+from loguru import logger
 
 
 class FetchJiraIssues:
@@ -17,16 +18,20 @@ def __init__(self, config):
 
     def get_issues(self):
         """Fetch JIRA issues which are not Epics."""
-        print('Fetching issues from JIRA...')
-        issues = self.__fetch_all_results(self.jira_issues_jql,
-                                          self.jira_issues_fields)
+        logger.info('Fetching issues from JIRA...')
+        issues = self.__fetch_all_results(
+            self.jira_issues_jql,
+            self.jira_issues_fields
+        )
         return issues
 
     def get_epics(self):
         """Fetch JIRA issues which are Epics."""
-        print('Fetching epics from JIRA...')
-        issues = self.__fetch_all_results(self.jira_epics_jql,
-                                          self.jira_epics_fields)
+        logger.info('Fetching epics from JIRA...')
+        issues = self.__fetch_all_results(
+            self.jira_epics_jql,
+            self.jira_epics_fields
+        )
         return issues
 
     def __get_total_number_of_issues(self, jql):
@@ -35,15 +40,16 @@ def __get_total_number_of_issues(self, jql):
             "jql": jql,
             "maxResults": 0,
             "startAt": 0}
-        req = requests.get(self.jira_endpoint,
-                           headers={"Accept": "application/json"},
-                           params=params,
-                           auth=(self.jira_username, self.jira_api_key))
-        print(req.json())
+        req = requests.get(
+            self.jira_endpoint,
+            headers={"Accept": "application/json"},
+            params=params,
+            auth=(self.jira_username, self.jira_api_key)
+        )
         total_results = req.json().get('total', None)
         if total_results:
             return total_results
-        print('Could not find any issues!')
+        logger.info('Could not find any issues!')
 
     def __fetch_all_results(self, jql, fields):
         """Retrieve all JIRA issues."""
@@ -58,16 +64,18 @@ def fetch_single_page(jql, fields):
                 "startAt": len(issue_arr),
                 "validateQuery": "warn",
                 "fields": fields}
-            req = requests.get(self.jira_endpoint,
-                               headers={"Accept": "application/json"},
-                               params=params,
-                               auth=(self.jira_username, self.jira_api_key))
+            req = requests.get(
+                self.jira_endpoint,
+                headers={"Accept": "application/json"},
+                params=params,
+                auth=(self.jira_username, self.jira_api_key)
+            )
             response = req.json()
             issues = response['issues']
             issues_so_far = len(issue_arr) + self.results_per_page
             if issues_so_far > num_issues:
                 issues_so_far = num_issues
-            print(f'Fetched {issues_so_far} out of {num_issues} total issues.')
+            logger.info(f'Fetched {issues_so_far} out of {num_issues} total issues.')
             issue_arr.extend(issues)
             # Check if additional pages of results exist.
         count = math.ceil(num_issues/self.results_per_page)

src/jirafetch.py renamed to jira_database_etl/jirafetch.py

@@ -14,7 +14,8 @@ def construct_dataframe(self, issues_json):
         jira_issues_df = json_normalize(issue_json_list)
         return jira_issues_df
 
-    def dict_to_json_string(self, issue_dict):
+    @staticmethod
+    def dict_to_json_string(issue_dict):
         """Convert dict to JSON to string."""
         issue_json_string = json.dumps(issue_dict)
         issue_json = json.loads(issue_json_string)
@@ -53,7 +54,8 @@ def make_issue_body(self, issue):
             body['labels'] = self.parse_label_data(issue['fields']['labels'])
         return body
 
-    def parse_sprint_data(self, sprint):
+    @staticmethod
+    def parse_sprint_data(sprint):
         """Parse raw sprint string."""
         sprint_body = {
             'sprint_status': sprint[0].split('state=')[1].split(',', 1)[0],
@@ -62,7 +64,8 @@ def parse_sprint_data(self, sprint):
         }
         return sprint_body
 
-    def parse_label_data(self, labels):
+    @staticmethod
+    def parse_label_data(labels):
         """Parse list of labels."""
         labels = ', '.join(labels)
         return labels

src/transform.py renamed to jira_database_etl/transform.py

@@ -1,5 +1,5 @@
 """Script entry point."""
-from src import main
+from jira_database_etl import init_script
 
 if __name__ == '__main__':
-    main()
+    init_script()