Merge branch 'develop' into KPMP-5807_RenameWSIFileOnMove

Author: rlreamy

data_management/lib/mysql_connection.py

@@ -146,13 +146,12 @@ def get_data(self, sql, query_data=None):
             for row in self.cursor:
                 data.append(row)
             return data
-        except:
-            message = "Error: Can't get data_management data."
-            logger.error(message)
+        except Exception as error:
+            logger.error(str(error))
             requests.post(
                 slack_url,
                 headers={'Content-type': 'application/json', },
-                data='{"text":"' + message + '"}'
+                data='{"text":"' + "Error: " + str(error) + '"}'
             )
         finally:
             self.cursor.close()

data_management/services/dlu_management.py

@@ -101,15 +101,10 @@ def update_dlu_package(self, package_id: str, fields_values: dict):
             values = query_info["values"][0:] + (package_id,)
             query = "UPDATE data_manager_data_v SET " + query_info["set_clause"] + " WHERE dlu_package_id = %s"
             self.db.insert_data(query, values)
-            
-    def get_missing_slides(self, redcap_id: str):
-        return self.db.get_data(
-            "select * from missing_slides_v where spectrack_redcap_record_id = %", 
-            redcap_id,),
 
     def update_missing_slides(self, redcap_id: str):
         return self.db.get_data(
-            "update slide_scan_curation set missing_slides = 1 where redcap_id = %s", redcap_id,
+            "update slide_scan_curation set missing_slides = 1 where redcap_id = %s", (redcap_id,)
         ),
 
     def insert_dlu_file(self, values):
@@ -230,7 +225,7 @@ def get_equal_num_rows(self):
 
     def get_new_slide_manifest_import_rows(self):
         return self.db.get_data("SELECT * FROM slide_manifest_import WHERE image_id NOT IN "
-                                "(SELECT image_id FROM slide_scan_curation where missing_slides = 0)")
+                                "(SELECT image_id FROM slide_scan_curation)")
 
     def get_spectrack_redcap_record_id(self, kit_id):
         result = self.db.get_data("SELECT spectrack_redcap_record_id FROM spectrack_specimen "
@@ -239,6 +234,22 @@ def get_spectrack_redcap_record_id(self, kit_id):
             return result[0]["spectrack_redcap_record_id"]
         else:
             return None
+        
+    def get_redcap_ids_with_null_package_id(self):
+        return self.db.get_data(
+            "select unique redcap_id from slide_scan_curation where dlu_package_id is null and error_message is null and redcap_id is not null", 
+            (None),
+        )
+        
+    def get_package_ids_for_redcap_id(self, redcap_id):
+        return self.db.get_data(
+            "select dlu_package_id from dlu_package_inventory where dlu_subject_id = %s and globus_dlu_status IS NULL", (redcap_id,)
+        )
+        
+    def update_package_ids_in_slide_scan_curation(self, redcap_id, package_id):
+        return self.db.insert_data(
+            "update slide_scan_curation set dlu_package_id = %s where redcap_id = %s and dlu_package_id is null and error_message is null",
+            (package_id, redcap_id,))
 
     def insert_into_slide_scan_curation(self, values):
         query = "INSERT INTO slide_scan_curation (image_id, kit_id, redcap_id, new_file_name, source_file_name, " \
@@ -267,6 +278,14 @@ def is_package_missing_slides(self, package_id):
         return self.db.get_data("SELECT * FROM slide_scan_v WHERE dlu_package_id = %s and missing_slides = 1",
                          (package_id,))
 
+    def slides_marked_missing_by_redcap_id(self, redcap_id: str):
+        return self.db.get_data("SELECT * FROM slide_scan_v WHERE redcap_id = %s AND missing_slides = 1",
+                         (redcap_id,))
+
+    def get_missing_slides_from_view(self, redcap_id: str):
+        return self.db.get_data("select * from missing_slides_v where spectrack_redcap_record_id = %s",
+                                (redcap_id,))
+
     def is_slides_in_error(self, package_id):
         return self.db.get_data("SELECT * FROM slide_scan_curation WHERE dlu_package_id = %s and error_message IS NOT NULL",
                          (package_id,))
@@ -275,6 +294,9 @@ def find_not_approved_filenames(self, package_id):
         return self.db.get_data("SELECT * FROM slide_scan_curation WHERE approve_file_name = 'yes' AND dlu_package_id = %s",
                          (package_id,))
 
+    def update_missing_slide_flag(self, image_id):
+        return self.db.insert_data("UPDATE slide_scan_curation SET missing_slides = 0 WHERE image_id = %s",
+                                   (image_id,))
 
 if __name__ == "__main__":
     dlu_management = DluManagement()

data_management/services/slide_management.py

@@ -74,12 +74,18 @@ def __init__(self, db):
 
     def process_slide_manifest_imports(self):
         new_records = self.db.get_new_slide_manifest_import_rows()
+        redcap_ids_processed = []
         for record in new_records:
             record_in_error = False
             error_message = ""
             kit_id = record["outside_acc"]
             image_id = record["image_id"]
             redcap_id = self.db.get_spectrack_redcap_record_id(kit_id)
+            if redcap_id is None:
+                error_message = "No redcap_id found for kit_id " + kit_id + "; "
+                logger.error(error_message)
+                continue
+
             if record["accession"] is not None:
                 new_file_name = self.determine_new_slide_name(sample_id=record["accession"], kit_id=kit_id,
                                                               stain_info=record["stain"], block_id=record["block_id"])
@@ -99,25 +105,32 @@ def process_slide_manifest_imports(self):
                 source_file_name = file_location.name
                 source_folder_name = file_location.parent.name
 
-            check_missing_slides = self.db.get_missing_slides(redcap_id)
-            if len(check_missing_slides) >= 1:
-
-                slide_scan = SlideScanModel(image_id=image_id, redcap_id=redcap_id, kit_id=kit_id,
-                                        new_file_name=new_file_name, source_file_name=source_file_name,
-                                        source_folder_name=source_folder_name)
-
-                self.db.insert_into_slide_scan_curation(slide_scan.get_dmd_tuple())
-            else:
-                
-                error_message += "There are missing slides for participant " + redcap_id + ";"
-                logger.info(error_message)
+            slide_scan = SlideScanModel(image_id=image_id, redcap_id=redcap_id, kit_id=kit_id,
+                                    new_file_name=new_file_name, source_file_name=source_file_name,
+                                    source_folder_name=source_folder_name)
+            self.db.insert_into_slide_scan_curation(slide_scan.get_dmd_tuple())
+
+            check_missing_slides = self.db.get_missing_slides_from_view(redcap_id)
+            redcap_ids_processed.append(redcap_id)
+            if all(check_missing_slides):
                 self.db.update_missing_slides(redcap_id)
-                
-                # Can't use record_in_error here because we can't set an error message for an image_id that doens't exist
-                self.db.set_error_message_slide_scan_curation_redcap_id(error=error_message, redcap_id=redcap_id)
-                
+
             if record_in_error:
                 self.db.set_error_message_slide_scan_curation(image_id=image_id, error=error_message)
+        logger.info("Processed " + str(len(new_records)) + " new slide_manifest_import records.")
+
+        for redcap_id in redcap_ids_processed:
+            self.update_missing_slides(redcap_id)
+
+    def update_missing_slides(self, redcap_id: str):
+        # This MAY seem redundant, however this will ensure that we unmark any missing slides records that just got
+        # the missing one added
+        missing_slides = self.db.get_missing_slides_from_view(redcap_id)
+        if not missing_slides or len(missing_slides) ==0 :
+            slides_marked_missing = self.db.slides_marked_missing_by_redcap_id(redcap_id)
+            if slides_marked_missing and len(slides_marked_missing) > 0:
+                for slide in slides_marked_missing:
+                    self.db.update_missing_slide_flag(slide['image_id'])
 
     def determine_new_slide_name(self, sample_id: str, kit_id: str, stain_info: str, block_id: str):
         slides_for_kit = self.db.get_slide_manifest_import_by_kit(kit_id, stain_info)
@@ -133,3 +146,18 @@ def determine_new_slide_name(self, sample_id: str, kit_id: str, stain_info: str,
             return None
         else:
             return sample_id + "_" + stain_type + "_" + str(numerator) + "of" + str(denominator) + ".svs"
+    
+    def fill_in_package_ids(self):
+        redcap_id_list = self.db.get_redcap_ids_with_null_package_id()
+        if len(redcap_id_list) != 0:
+            for row in redcap_id_list:
+                redcap_id = row['redcap_id']
+                package_id_list = self.db.get_package_ids_for_redcap_id(redcap_id)
+                if None not in package_id_list and len(package_id_list) == 1:
+                    package_id = package_id_list[0]['dlu_package_id']
+                    self.db.update_package_ids_in_slide_scan_curation(redcap_id=redcap_id, package_id=package_id)
+                    logger.info("Updated package id " + package_id + " for redcap id " + redcap_id)
+                elif len(package_id_list) > 1:
+                    error_message = "Multiple dlu_package_ids found for redcap_id " + redcap_id + ", unable to fill in package id."
+                    logger.info(error_message)
+                    self.db.set_error_message_slide_scan_curation_redcap_id(error=error_message, redcap_id=redcap_id)

data_management/watch_files.py

@@ -118,6 +118,9 @@ def move_packages_to_DLU(self, packages):
                 self.dlu_state.set_package_state(package_id, PackageState.UPLOAD_SUCCEEDED)
                 self.dlu_state.clear_cache()
 
+    def fill_in_null_package_ids(self):
+        self.slide_management.fill_in_package_ids()
+
     def do_wsi_file_renames(self, globus_data_directory: str, package_id: str):
         logger.info("starting rename process")
         error_msg = ""
@@ -201,4 +204,5 @@ def is_directory_valid(self, directory_info, package_id):
     while True:
         dlu_watcher.watch_for_packages()
         dlu_watcher.watch_for_side_manifest_records()
-        time.sleep(60)
+        dlu_watcher.fill_in_null_package_ids()
+        time.sleep(60)