Feature/2d ftu

package.json

@@ -34,6 +34,7 @@
     "js-yaml": "^4.1.0",
     "node-fetch": "^3.3.0",
     "nunjucks": "^3.2.4",
+    "papaparse": "^5.4.1",
     "semver": "^7.5.3",
     "shelljs": "^0.8.5",
     "tslib": "^2.5.0"

schemas/src/digital-objects/2d-ftu.yaml

@@ -0,0 +1,153 @@
+id: https://purl.humanatlas.io/specs/2d-ftu
+name: ftu-2d
+prefixes:
+  ccf: http://purl.org/ccf/
+  dcterms: http://purl.org/dc/terms/
+  pav: http://purl.org/pav/
+  rdfs: http://www.w3.org/2000/01/rdf-schema#
+  rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns#
+  xsd: http://www.w3.org/2001/XMLSchema#
+  obo: http://purl.obolibrary.org/obo/
+  linkml: https://w3id.org/linkml/
+  UBERON: http://purl.obolibrary.org/obo/UBERON_
+  CL: http://purl.obolibrary.org/obo/CL_
+
+default_prefix: ccf
+default_range: string
+
+imports:
+  - linkml:types
+  - ../shared/metadata-base
+
+settings:
+  uberon: "UBERON"
+  fma: "FMA"
+
+classes:
+  Named:
+    mixin: true
+    slots:
+      - id
+      - label
+      - class_type
+  Instance:
+    mixin: true
+    slots:
+      - typeOf
+    annotations:
+      owl: Individual
+
+  FtuIllustration:
+    class_uri: ccf:FtuIllustration
+    mixins:
+      - Named
+      - Instance
+    slots:
+      - located_in
+      - image_file
+      - illustration_node
+    annotations:
+      owl: NamedIndividual
+
+  FtuIllustrationFile:
+    class_uri: ccf:FtuIllustrationFile
+    mixins:
+      - Named
+      - Instance
+    slots:
+      - file_url
+      - file_format
+    annotations:
+      owl: NamedIndividual
+
+  FtuIllustrationNode:
+    class_uri: ccf:FtuIllustrationNode
+    mixins:
+      - Named
+      - Instance
+    slots:
+      - node_name
+      - part_of_illustration
+    annotations:
+      owl: NamedIndividual
+
+  AnatomicalStructure:
+    slots:
+      - id
+    slot_usage:
+      id:
+        structured_pattern:
+          syntax: "({uberon}|{fma}):\\d+"
+          interpolated: true
+          partial_match: false
+
+  Container:
+    tree_root: true
+    attributes:
+      iri:
+        range: uriorcurie
+      metadata:
+        range: DatasetMetadata
+      data:
+        multivalued: true
+        inlined_as_list: true
+        range: FtuIllustration
+    annotations:
+      owl.template: |-
+        AnnotationAssertion( dct:title {{iri}} "{{metadata.title}}" )
+        AnnotationAssertion( dct:description {{iri}} "{{metadata.description}}" )
+        {% for c in metadata.creators %}
+        AnnotationAssertion( dct:creator {{iri}} "{{c.fullName}} ({{c.orcid}})" )
+        {% endfor %}
+        AnnotationAssertion( schema:version {{iri}} "{{metadata.version}}" )
+        AnnotationAssertion( schema:dateCreated {{iri}} "{{metadata.creation_date}}" )
+        AnnotationAssertion( dct:license {{iri}} "{{metadata.license}}" )
+        AnnotationAssertion( dct:publisher {{iri}} "{{metadata.publisher}}" )
+        AnnotationAssertion( rdfs:seeAlso {{iri}} "{{metadata.see_also}}" )
+
+slots:
+  id:
+    identifier: true
+    range: string
+  label:
+    slot_uri: rdfs:label
+    annotations:
+      owl: AnnotationAssertion
+  class_type:
+    designates_type: true
+  typeOf:
+    multivalued: true
+    range: Named
+    slot_uri: rdf:type
+    annotations:
+      owl: ClassAssertion
+  located_in:
+    slot_uri: ccf:ccf_located_in
+    range: AnatomicalStructure
+    annotations:
+      owl: AnnotationAssertion
+  image_file:
+    range: FtuIllustrationFile
+    multivalued: true
+    inlined_as_list: true
+    annotations:
+      owl: AnnotationAssertion
+  file_url:
+    range: uri
+    annotations:
+      owl: AnnotationAssertion
+  file_format:
+    annotations:
+      owl: AnnotationAssertion
+  illustration_node:
+    range: FtuIllustrationNode
+    multivalued: true
+    inlined_as_list: true
+    annotations:
+      owl: AnnotationAssertion
+  node_name:
+    annotations:
+      owl: AnnotationAssertion
+  part_of_illustration:
+    annotations:
+      owl: AnnotationAssertion

schemas/src/metadata/2d-ftu-metadata.yaml

@@ -0,0 +1,60 @@
+id: https://purl.humanatlas.io/specs/2d-ftu-metadata
+name: ftu-2d-metadata
+prefixes:
+  ccf: http://purl.org/ccf/
+  pav: http://purl.org/pav/
+  dcat: http://www.w3.org/ns/dcat#
+  dct: http://purl.org/dc/terms/
+  foaf: http://xmlns.com/foaf/0.1/
+  schema: http://schema.org/
+  rdfs: http://www.w3.org/2000/01/rdf-schema#
+  xsd: http://www.w3.org/2001/XMLSchema#
+  linkml: https://w3id.org/linkml/
+
+default_prefix: dcat
+default_range: string
+
+imports:
+  - linkml:types
+  - ../shared/metadata-base
+
+classes:
+  Container:
+    tree_root: true
+    class_uri: dcat:Dataset
+    slots:
+      - iri
+      - title
+      - description
+      - creators
+      - project_leads
+      - reviewers
+      - externalReviewers
+      - version
+      - creation_date
+      - license
+      - publisher
+      - funders
+      - hubmapId
+      - doi
+      - citation
+      - citationOverall
+      - datatable
+      - distributions
+    slot_usage:
+      iri:
+        required: true
+      title:
+        required: true
+      description:
+        required: true
+      creators:
+        required: true
+      version:
+        required: true
+      creation_date:
+        required: true
+      license:
+        required: true
+      distributions:
+        required: true

schemas/src/shared/metadata-base.yaml

@@ -39,6 +39,17 @@ classes:
       - downloadUrl
       - accessUrl
       - mediaType
+  DatasetMetadata:
+    class_uri: dcat:Dataset
+    slots:
+      - title
+      - description
+      - creators
+      - version
+      - creation_date
+      - license
+      - publisher
+      - see_also
 
 slots:
   iri:

src/enrichment/enrich-2d-ftu.js

@@ -0,0 +1,116 @@
+import fs from 'fs';
+import { resolve } from 'path';
+import { error, header, info, more } from '../utils/logging.js';
+import { convert, filter, merge, query } from '../utils/robot.js';
+import { throwOnError } from '../utils/sh-exec.js';
+import { 
+  cleanTemporaryFiles, 
+  convertNormalizedMetadataToRdf,
+  convertNormalizedDataToOwl,
+  logOutput 
+} from './utils.js';
+
+export function enrich2dFtuMetadata(context) {
+  const { selectedDigitalObject: obj } = context;
+  const normalizedPath = resolve(obj.path, 'normalized/normalized-metadata.yaml');
+  const enrichedPath = resolve(obj.path, 'enriched/enriched-metadata.ttl');
+  convertNormalizedMetadataToRdf(context, normalizedPath, enrichedPath);
+}
+
+export function enrich2dFtuData(context) {
+  try {
+    const { selectedDigitalObject: obj, processorHome } = context;
+
+    // Convert normalized data to graph data (.ttl)
+    const normalizedPath = resolve(obj.path, 'normalized/normalized.yaml');
+    const baseInputPath = resolve(obj.path, 'enriched/base-input.ttl');
+    convertNormalizedDataToOwl(context, normalizedPath, baseInputPath);
+    logOutput(baseInputPath);
+
+    let inputPaths = []; // variable to hold input files for merging
+
+    const enrichedWithOntologyPath = resolve(obj.path, 'enriched/enriched-with-ontology.owl');
+
+    inputPaths.push(baseInputPath); // Set the enriched path as the initial
+
+    info('Getting concept details from reference ontologies...')
+    const uberonEntitiesPath = collectEntities(context, 'uberon', baseInputPath);
+    if (!isFileEmpty(uberonEntitiesPath)) {
+      info('Extracting UBERON.');
+      const uberonExtractPath = filterClasses(context, 'uberon', uberonEntitiesPath);
+      logOutput(uberonExtractPath);
+      inputPaths.push(uberonExtractPath);
+    }
+
+    const fmaEntitiesPath = collectEntities(context, 'fma', baseInputPath);
+    if (!isFileEmpty(fmaEntitiesPath)) {
+      info('Extracting FMA.');
+      const fmaExtractPath = filterClasses(context, 'fma', fmaEntitiesPath);
+      logOutput(fmaExtractPath);
+      inputPaths.push(fmaExtractPath);
+    }
+
+    const clEntitiesPath = collectEntities(context, 'cl', baseInputPath);
+    if (!isFileEmpty(clEntitiesPath)) {
+      info('Extracting CL.');
+      const clExtractPath = filterClasses(context, 'cl', clEntitiesPath);
+      logOutput(clExtractPath);
+      inputPaths.push(clExtractPath);
+    }
+
+    const pclEntitiesPath = collectEntities(context, 'pcl', baseInputPath);
+    if (!isFileEmpty(pclEntitiesPath)) {
+      info('Extracting PCL.');
+      const pclExtractPath = filterClasses(context, 'pcl', clEntitiesPath);
+      logOutput(pclExtractPath);
+      inputPaths.push(pclExtractPath);
+    }
+
+    info('Merging files:');
+    for (const inputPath of inputPaths) {
+      more(` -> ${inputPath}`);
+    }
+    merge(inputPaths, enrichedWithOntologyPath);
+    logOutput(enrichedWithOntologyPath);
+
+    const enrichedPath = resolve(obj.path, 'enriched/enriched.ttl');
+
+    info(`Creating 2d-ftu: ${enrichedPath}`);
+    convert(enrichedWithOntologyPath, enrichedPath, 'ttl');
+
+  } catch (e) {
+    error(e);
+  } finally {
+    // Clean up
+    info('Cleaning up temporary files...');
+    cleanTemporaryFiles(context);
+    more("Done.")
+  }
+}
+
+function isFileEmpty(path) {
+  return fs.statSync(path).size === 0;
+}
+
+function collectEntities(context, ontologyName, inputPath) {
+  const { selectedDigitalObject: obj, processorHome } = context;
+
+  const queryPath = resolve(processorHome, `src/utils/get-${ontologyName}-terms.sparql`);
+  const outputPath = resolve(obj.path, `enriched/${ontologyName}-terms.csv`);
+
+  query(inputPath, queryPath, outputPath);
+  throwOnError(`sed -i '1d' ${outputPath}`, 'Collect entities failed.');
+
+  return outputPath;
+}
+
+function filterClasses(context, ontologyName, classTermFile) {
+  const { selectedDigitalObject: obj, processorHome } = context;
+
+  const ontologyPath = resolve(processorHome, `mirrors/${ontologyName}.owl`);
+  const outputPath = resolve(obj.path, `enriched/${ontologyName}-filter.owl`);
+
+  filter(ontologyPath, classTermFile, ['rdfs:label', 'http://www.geneontology.org/formats/oboInOwl#id', 'http://purl.obolibrary.org/obo/IAO_0000115'], outputPath);
+
+  return outputPath;
+}

src/enrichment/enrich.js

@@ -2,6 +2,7 @@ import { resolve } from 'path';
 import sh from 'shelljs';
 import { enrichAsctbMetadata, enrichAsctbData } from './enrich-asct-b.js';
 import { enrichRefOrganMetadata, enrichRefOrganData } from './enrich-ref-organ.js';
+import { enrich2dFtuMetadata, enrich2dFtuData } from './enrich-2d-ftu.js';
 import { enrichCollectionMetadata, enrichCollectionData } from './enrich-collection.js';
 import { header } from '../utils/logging.js';
 
@@ -18,6 +19,10 @@ export function enrich(context) {
       enrichRefOrganMetadata(context);
       enrichRefOrganData(context);
       break;
+    case '2d-ftu':
+      enrich2dFtuMetadata(context);
+      enrich2dFtuData(context);
+      break;
     case 'collection':
       enrichCollectionMetadata(context);
       enrichCollectionData(context);