update post processing to match better database structure

fix dangling edge issues

build on dev

Author: =

src/input_adapters/go/go_gaf.py

@@ -65,6 +65,7 @@ def get_all(self) -> Generator[List[ProteinGoTermRelationship], None, None]:
                 ))
                 if count >= self.batch_size:
                     yield pro_go_edges
+                    count = 0
                     pro_go_edges: List[ProteinGoTermRelationship] = []
 
         yield pro_go_edges

src/input_adapters/pharos_arango/set_ligand_activity_flag.py

@@ -28,15 +28,12 @@ def get_all(self) -> Generator[List[ProteinLigandRelationship], None, None]:
 
 passing_activities_query = """FOR pro IN `biolink:Protein`
   FOR chem, rel IN OUTBOUND pro `biolink:interacts_with`
-    FILTER (
-      (pro.idg_family IN ["GPCR", "Nuclear Receptor"] AND LENGTH(rel.act_value[* FILTER CURRENT >= 7]) > 0) OR
-      (pro.idg_family == "Kinase" AND LENGTH(rel.act_value[* FILTER CURRENT >= 7.52288]) > 0) OR
-      (pro.idg_family == "Ion Channel" AND LENGTH(rel.act_value[* FILTER CURRENT >= 5]) > 0) OR
-      (
-        (pro.idg_family == null OR pro.idg_family NOT IN ["Ion Channel", "Kinase", "GPCR", "Nuclear Receptor"])
-        AND LENGTH(rel.act_value[* FILTER CURRENT >= 6]) > 0
-      )
-    )
+    LET act_values = rel.details[* FILTER CURRENT.act_value >= (
+      pro.idg_family == "Kinase" ? 7.52288 :
+      pro.idg_family == "Ion Channel" ? 5 :
+      pro.idg_family IN ["GPCR", "Nuclear Receptor"] ? 7 : 6
+    )]
+    FILTER LENGTH(act_values) > 0
     RETURN DISTINCT {
       protein_id: pro.id,
       chemical_entity_id: chem.id

src/input_adapters/pharos_arango/tdl_input_adapter.py

@@ -83,19 +83,21 @@ def make_set(list_query_result: list):
 """
 
 proteins_with_moa_drugs = """
-for pro in `biolink:Protein`
-  for lig, act in outbound pro `biolink:interacts_with`
+FOR pro IN `biolink:Protein`
+  FOR lig, act IN OUTBOUND pro `biolink:interacts_with`
     FILTER lig.isDrug == TRUE
-    FILTER LENGTH(act.has_moa[* FILTER CURRENT == TRUE]) > 0
-  RETURN distinct pro.id
+    LET has_moa_flags = act.details[* FILTER CURRENT.has_moa == TRUE RETURN CURRENT.has_moa]
+    FILTER LENGTH(has_moa_flags) > 0
+    RETURN DISTINCT pro.id
 """
 
 proteins_with_experimental_f_or_p_go_terms = """
 FOR p IN `biolink:Protein`
   FOR g, r IN OUTBOUND p `ProteinGoTermRelationship`
     FILTER g.is_leaf == true
       AND g.type != 'C'
-      AND 'Experimental evidence code' IN r.category
+    LET evidence_categories = r.evidence[* RETURN CURRENT.category]
+    FILTER 'Experimental evidence code' IN evidence_categories
     RETURN DISTINCT p.id
 """
 

src/output_adapters/arango_output_adapter.py

@@ -48,14 +48,18 @@ def generate_edge_key(from_node, to_node, edge_type):
                     edges.append(edge)
 
                 edge_collection.insert_many(edges, overwrite=True)
-            else:
-                if not db.has_collection(label):
-                    collection = db.create_collection(label)
-                else:
-                    collection = db.collection(label)
 
-                keys = [self.safe_key(obj['id']) for obj in obj_list]
-                existing_nodes = collection.get_many(keys)
+                cursor = db.aql.execute(f""" 
+                        FOR e IN `{edge_collection.name}`
+                          FILTER !DOCUMENT(e._from) || !DOCUMENT(e._to)
+                          REMOVE e IN `{edge_collection.name}`
+                    """)
+                result = cursor.statistics()
+                deleted_count = result.get('modified', 0)
+                if deleted_count > 0:
+                    print(f"Deleted {deleted_count} dangling edges.")
+            else:
+                collection, existing_nodes = self.get_existing_nodes(db, label, obj_list)
                 existing_record_map = {
                     record['id']: record for record in existing_nodes
                 }
@@ -69,6 +73,15 @@ def generate_edge_key(from_node, to_node, edge_type):
 
         return True
 
+    def get_existing_nodes(self, db, label, obj_list):
+        if not db.has_collection(label):
+            collection = db.create_collection(label)
+        else:
+            collection = db.collection(label)
+        keys = [self.safe_key(obj['id']) for obj in obj_list]
+        existing_nodes = collection.get_many(keys)
+        return collection, existing_nodes
+
     def create_or_truncate_datastore(self) -> bool:
         sys_db = self.client.db('_system', username=self.credentials.user,
                             password=self.credentials.password)

src/use_cases/target_graph.yaml

@@ -1,4 +1,4 @@
-source_credentials: &source_credentials ./src/use_cases/secrets/ifxdev_pharos_prod.yaml
+source_credentials: &source_credentials ./src/use_cases/secrets/ifxdev_pharos_dev.yaml
 
 pchembl_cutoff: &pchembl_cutoff 5
 

src/use_cases/target_graph_aql_post.yaml

@@ -1,4 +1,4 @@
-source_credentials: &source_credentials ./src/use_cases/secrets/ifxdev_pharos_prod.yaml
+source_credentials: &source_credentials ./src/use_cases/secrets/ifxdev_pharos_dev.yaml
 database_name: &database_name pharos
 
 labeler: