diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/Jbrowse2ConfigReporter.java b/Model/src/main/java/org/apidb/apicommon/model/report/Jbrowse2ConfigReporter.java new file mode 100644 index 000000000..0dc1ada0a --- /dev/null +++ b/Model/src/main/java/org/apidb/apicommon/model/report/Jbrowse2ConfigReporter.java @@ -0,0 +1,193 @@ +package org.gusdb.wdk.model.report.reporter; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.log4j.Logger; +import org.gusdb.wdk.model.WdkModelException; +import org.gusdb.wdk.model.WdkUserException; +import org.gusdb.wdk.model.answer.AnswerValue; +import org.gusdb.wdk.model.answer.stream.RecordStream; +import org.gusdb.wdk.model.answer.stream.RecordStreamFactory; +import org.gusdb.wdk.model.record.RecordInstance; +import org.gusdb.wdk.model.record.TableField; +import org.gusdb.wdk.model.record.TableValue; +import org.gusdb.wdk.model.record.TableValueRow; +import org.gusdb.wdk.model.record.attribute.AttributeField; +import org.gusdb.wdk.model.record.attribute.AttributeValue; +import org.gusdb.wdk.model.report.AbstractReporter; +import org.gusdb.wdk.model.report.PropertiesProvider; +import org.gusdb.wdk.model.report.Reporter; +import org.gusdb.wdk.model.report.ReporterConfigException; +import org.gusdb.wdk.model.report.util.TableCache; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +/** + * @author Steve + * + */ +public class Jbrowse2ConfigReporter extends AbstractReporter { + + private static final Logger LOG = Logger.getLogger(Jbrowse2ConfigReporter.class); + + private TableCache _tableCache; + + @Override + public Jbrowse2ConfigReporter setProperties(PropertiesProvider reporterRef) throws WdkModelException { + super.setProperties(reporterRef); + String cacheTableName = TableCache.getCacheTableName(_properties); + if (cacheTableName != null) { + _tableCache = new TableCache(getQuestion().getRecordClass(), _wdkModel.getAppDb(), cacheTableName); + } + return this; + } + + @Override + public Reporter configure(JSONObject config) throws ReporterConfigException, WdkModelException { + return null; + } + + @Override + public String getHttpContentType() { + return "application/json"; + } + + @Override + public String getDownloadFileName() { + return getQuestion().getName() + "_detail.json"; + } + + @Override + public void write(OutputStream out) throws WdkModelException { + + Map attrFieldMap = getQuestion().getAttributeFieldMap(); + if (!attrFieldMap.containsKey("short_display_name")) throw new WdkModelException(); + AttributeField af = attrFieldMap.get("short_display_name"); + Set afs = new HashSet<>(); + afs.add(af); + + Map tblFieldMap = getQuestion().; + if (!tblFieldMap.containsKey("short_display_name")) throw new WdkModelException(); + TableField tf = tblFieldMap.get("short_display_name"); + Set afs = new HashSet<>(); + afs.add(af); + + // TableField udDependenciesTable = (TableField) fieldMap.get("UdDependencies"); + + try (RecordStream records = RecordStreamFactory.getRecordStream(_baseAnswer, afs, tables)) { + OutputStreamWriter streamWriter = new OutputStreamWriter(out); + JSONWriter writer = new JSONWriter(streamWriter); + + AnswerValue av = _baseAnswer; + writer.object().key("response").object().key("recordset").object().key("id").value( + av.getChecksum()).key("count").value(this.getResultSize()).key("type").value( + av.getAnswerSpec().getQuestion().getRecordClass().getDisplayName()).key("records").array(); + + if (_tableCache != null) { + _tableCache.open(); + } + + // get page based answers with a maximum size (defined in PageAnswerIterator) + int recordCount = 0; + for (RecordInstance record : records) { + + // get VDI ID + String datasetId = record.getPrimaryKey().getValues().get("dataset_id"); + String suffix = "EDAUD_"; + if (!datasetId.startsWith(suffix)) throw new WdkUserException(); + String vdiId = datasetId.substring(suffix.length()); + + // get display name + String shortDisplayName = record.getAttributeValue("short_display_name").getValue(); + + // get organism name for files + TableValue tv = record.getTableValue("UdDependencies"); + if (tv.getNumRows() != 1) throw new WdkUserException(); + TableValueRow tvr = tv.iterator().next(); + if (!tvr.containsKey("identifier")) throw new WdkUserException(); + String orgNameForFiles = tvr.get("identifier").getValue(); + + // count the records processed so far + recordCount++; + writer.endObject(); + streamWriter.flush(); + } + + writer.endArray() // records + .endObject().endObject().endObject(); + streamWriter.flush(); + LOG.info("Totally " + recordCount + " records dumped"); + } catch (WdkUserException | JSONException | SQLException | IOException e) { + throw new WdkModelException("Unable to write JSON report", e); + } finally { + if (_tableCache != null) { + _tableCache.close(); + } + } + } + + /* + { + "assemblyNames": [ + "VDI_ORGANISM_FOR_FILES" + ], + "trackId": "UD_VDI_ID", + "name": "SHORT_DISPLAY_NAME", + "displays": [ + { + "displayId": "wiggle_ApiCommonModel::Model::JBrowseTrackConfig::MultiBigWigTrackConfig::XY=HASH(0x2249320)", + "maxScore": 1000, + "minScore": 1, + "defaultRendering": "multirowxy", + "type": "MultiLinearWiggleDisplay", + "scaleType": "log" + } + ], + "adapter": { + "subadapters": [ + { + "color": "grey", + "name": "Unsporulated oocyst (non-unique)", + "type": "BigWigAdapter", + "bigWigLocation": { + "locationType": "UriLocation", + "uri": "ToxoDB/build-68/EtenellaHoughton2021/bigwig/etenHoughton2021_Reid_RNASeq_ebi_rnaSeq_RSRC/1_Unsporulated_oocyst/non_ +unique_resultsCombinedReps_unlogged.bw" + } + } +*/ + + + private static void formatAttributes(RecordInstance record, Set attributes, JSONWriter writer) + throws WdkModelException, WdkUserException { + if (!attributes.isEmpty()) { + writer.key("fields").array(); + for (AttributeField field : attributes) { + AttributeValue value = record.getAttributeValue(field.getName()); + writer.object().key("name").value(field.getName()).key("value").value(value.getValue()).endObject(); + } + writer.endArray(); + } + } + + /* + - dataset id + - + */ + private Set getAttributes() { + return null; + } + private Set getTables() { + return null; + } + +} + + diff --git a/Service/src/main/java/org/apidb/apicommon/service/ApiWebServiceApplication.java b/Service/src/main/java/org/apidb/apicommon/service/ApiWebServiceApplication.java index e28b5fcb6..e5a1e325b 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/ApiWebServiceApplication.java +++ b/Service/src/main/java/org/apidb/apicommon/service/ApiWebServiceApplication.java @@ -13,6 +13,7 @@ import org.apidb.apicommon.service.services.comments.AttachmentsService; import org.apidb.apicommon.service.services.comments.UserCommentsService; import org.apidb.apicommon.service.services.dataPlotter.ProfileSetService; +import org.apidb.apicommon.service.services.jbrowse.JBrowse2Service; import org.apidb.apicommon.service.services.jbrowse.JBrowseService; import org.apidb.apicommon.service.services.jbrowse.JBrowseUserDatasetsService; import org.eupathdb.common.service.EuPathServiceApplication; @@ -46,6 +47,7 @@ public Set> getClasses() { .add(UserCommentsService.class) .add(TranscriptToggleService.class) .add(JBrowseService.class) + .add(JBrowse2Service.class) .add(JBrowseUserDatasetsService.class) .add(ProfileSetService.class) .add(OrganismMetricsService.class) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java new file mode 100644 index 000000000..d29408f44 --- /dev/null +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -0,0 +1,266 @@ +package org.apidb.apicommon.service.services.jbrowse; + +import java.io.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.BadRequestException; + +import org.apache.log4j.Logger; +import org.gusdb.fgputil.db.runner.SQLRunner; +import org.gusdb.fgputil.db.runner.SQLRunnerException; +import org.gusdb.wdk.model.WdkException; +import org.gusdb.wdk.model.WdkModelException; +import org.gusdb.wdk.model.WdkRuntimeException; +import org.gusdb.wdk.service.service.AbstractWdkService; +import org.json.JSONArray; +import org.json.JSONObject; + +@Path("/jbrowse2") +public class JBrowse2Service extends AbstractWdkService { + + private static final Logger LOG = Logger.getLogger(JBrowse2Service.class); + + private static final String VDI_DATASETS_DIRECTORY_KEY ="VDI_DATASETS_DIRECTORY"; + private static final String VDI_CONTROL_SCHEMA_KEY ="VDI_CONTROL_SCHEMA"; + private static final String VDI_DATASET_SCHEMA_KEY ="VDI_DATASETS_SCHEMA"; + private static final String WEB_SVC_DIR_KEY ="WEBSERVICEMIRROR"; + + private static final String SVC_USER_DATASETS_DIR = "./userDatasetsData"; // hard-coded mount point in the jbrowse2 service + /* + Get config for a single organism. Assumes JSON will easily fit in memory. + */ + @GET + @Path("orgview/{publicOrganismAbbrev}/config.json") + @Produces(MediaType.APPLICATION_JSON) + public Response getJbrowseSingleOrgTracks(@PathParam("publicOrganismAbbrev") String publicOrganismAbbrev, + @QueryParam("trackSets") String trackSetsString) throws IOException, WdkException { + + String errMsg = "Must provide a comma delimited list of tracks in a 'trackSets' query param"; + if (trackSetsString == null) throw new BadRequestException(errMsg); + List trackSetsList = Arrays.asList(trackSetsString.split(",")); + if (trackSetsList.isEmpty()) throw new BadRequestException(errMsg); + + // get static json config, for this organism and set of tracks + String staticConfigJsonString = getStaticConfigJsonString(publicOrganismAbbrev, trackSetsString); + JSONObject staticConfigJson = new JSONObject(staticConfigJsonString); + + // get similar from user datasets + JSONArray udTracks = getUserDatasetTracks(publicOrganismAbbrev, trackSetsList); + + // merge UD tracks into static + staticConfigJson.getJSONArray("tracks").putAll(udTracks); + + // send response + String jsonString = staticConfigJson.toString(); + return Response.ok(jsonString, MediaType.APPLICATION_JSON).build(); + } + + // call out to perl code to produce static config json + String getStaticConfigJsonString(String publicOrganismAbbrev, String trackSetsString) throws IOException { + + String gusHome = getWdkModel().getGusHome(); + String projectId = getWdkModel().getProjectId(); + String buildNumber = getWdkModel().getBuildNumber(); + + List command = new ArrayList<>(); + command.add(gusHome + "/bin/jbrowse2Config"); + command.add("--orgAbbrev"); + command.add(publicOrganismAbbrev); + command.add("--projectId"); + command.add(projectId); + command.add("--buildNumber"); + command.add(buildNumber); + command.add("--webSvcDir"); + command.add(getWdkModel().getProperties().get(WEB_SVC_DIR_KEY)); + command.add("--trackSets"); + command.add(trackSetsString); + + return stringFromCommand(command); + } + + JSONArray getUserDatasetTracks(String publicOrganismAbbrev, List trackSetList) throws WdkModelException { + String buildNumber = getWdkModel().getBuildNumber(); + String projectId = getWdkModel().getProjectId(); + Long userId = getRequestingUser().getUserId(); + String vdiDatasetsSchema = getWdkModel().getProperties().get(VDI_DATASET_SCHEMA_KEY); + String vdiControlSchema = getWdkModel().getProperties().get(VDI_CONTROL_SCHEMA_KEY); + String vdiDatasetsDir = getWdkModel().getProperties().get(VDI_DATASETS_DIRECTORY_KEY); + + String path = String.join("/", vdiDatasetsSchema.toLowerCase(), "build-" + buildNumber, projectId); + String svcUserDataPathString = SVC_USER_DATASETS_DIR + "/" + path; + String wdkUserDatasetsPathString = vdiDatasetsDir + "/" + path; + JSONArray udTracks = new JSONArray(); + + // for now we only have rnaseq UD tracks + if (trackSetList.contains("rnaseq")) { + udTracks.put(getRnaSeqUdTracks(publicOrganismAbbrev, projectId, vdiControlSchema, wdkUserDatasetsPathString, + svcUserDataPathString, userId)); + } + return udTracks; + } + + JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, String vdiControlSchema, + String wdkUserDatasetsPathString, String svcUserDatasetsPathString, Long userId) throws WdkModelException { + + DataSource appDs = getWdkModel().getAppDb().getDataSource(); + String sql = "select distinct user_dataset_id, name " + + "from " + vdiControlSchema + ".AvailableUserDatasets aud, " + + vdiControlSchema + ".dataset_dependency dd " + + "where project_id = '" + projectId + "' " + + "and (type = 'rnaseq' or type = 'bigwigfiles') " + + "and ((is_public = 1 and is_owner = 1) or user_id = " + userId + ") " + + "and dd.dataset_id = aud.user_dataset_id " + + "and dd.identifier = '" + publicOrganismAbbrev + "'"; + + try { + return new SQLRunner(appDs, sql).executeQuery(rs -> { + JSONArray rnaSeqUdTracks = new JSONArray(); + while (rs.next()) { + String datasetId = rs.getString(1); + String name = rs.getString(2); + JSONObject track = createBigwigTrackJson(datasetId, name, publicOrganismAbbrev); + rnaSeqUdTracks.put(track); + List fileNames = getBigwigFileNames(wdkUserDatasetsPathString + "/" + datasetId); + for (String fileName : fileNames) { + + track.getJSONObject("adapter") + .getJSONArray("subadapters") + .put(createBigwigSubadapterJson(datasetId, fileName, svcUserDatasetsPathString)); + } + } + return rnaSeqUdTracks; + }); + } + catch (SQLRunnerException e) { + throw new WdkModelException("Unable to query VDI tables for RNA seq datasets. " + e.getMessage(), e.getCause()); + } + } + + // boilerplate method written by copilot + public static List getBigwigFileNames(String directoryPath) throws SQLRunnerException { + + List bwFiles = new ArrayList<>(); + File directory = new File(directoryPath); + + if (directory.isDirectory()) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + + if (file.isFile() && file.getName().endsWith(".bw")) { + bwFiles.add(file.getName()); + } + } + } + } else { + throw new SQLRunnerException("User Dataset directory not found for path: " + directoryPath); + } + + return bwFiles; + } + +/* + MULTI BIGWIG TRACK EXAMPLE + { + "assemblyNames": [ + "ORG_ABBREV" + ], + "trackId": "VDI_ID", + "name": "VDI_NAME", + "displays": [ + { + "displayId": "wiggle_ApiCommonModel::Model::JBrowseTrackConfig::MultiBigWigTrackConfig::XY=HASH(0x2249320)", + "maxScore": 1000, + "minScore": 1, + "defaultRendering": "multirowxy", + "type": "MultiLinearWiggleDisplay", + "scaleType": "log" + } + ], + "adapter": { + "subadapters": [ + { + "color": "grey", + "name": "FILE_NAME", + "type": "BigWigAdapter", + "bigWigLocation": { + "locationType": "UriLocation", + "uri": "USER_DATASET_PATH/VDI_ID/FILE_NAME" + } + } + } + } + */ + JSONObject createBigwigTrackJson(String vdiId, String vdiName, String organismAbbrev) { + return new JSONObject() + .put("assemblyNames", new JSONArray().put(organismAbbrev)) + .put("trackId", vdiId) + .put("name", vdiName) + .put("displays", new JSONArray() + .put(new JSONObject() + .put("displayId", "wiggle_ApiCommonModel::Model::JBrowseTrackConfig::MultiBigWigTrackConfig::XY=HASH(0x2249320)") + .put("maxScore", 1) + .put("maxScore", 1000) + .put("defaultRendering", "multirowxy") + .put("type", "MultiLinearWiggleDisplay") + .put("scaleType", "log") + ) + ) + .put("adapter", new JSONObject() + .put("subadapters", new JSONArray()) + ); + } + + JSONObject createBigwigSubadapterJson(String vdiId, String fileName, String userDatasetsFilePath) { + JSONObject subAdapter = new JSONObject(); + subAdapter.put("color1", "grey"); + subAdapter.put("name", fileName); + subAdapter.put("type", "BigWigAdapter"); + JSONObject location = new JSONObject().put("locationType", "UriLocation"); + location.put("uri", String.join("/", userDatasetsFilePath, vdiId, fileName)); + subAdapter.put("bigWigLocation", location); + return subAdapter; + } + + String stringFromCommand(List command) throws IOException { + LOG.debug("Running command: " + String.join(" ", command)); + try { + Process p = processFromCommand(command); + + ByteArrayOutputStream stringBuffer = new ByteArrayOutputStream(); + p.getErrorStream().transferTo(stringBuffer); + String errors = stringBuffer.toString(); + + stringBuffer.reset(); + p.getInputStream().transferTo(stringBuffer); + + if (p.waitFor() != 0) { + throw new RuntimeException("Subprocess from [" + String.join(" ", command) + "] returned non-zero. Errors:\n" + errors); + } + + return stringBuffer.toString(); + } + catch (InterruptedException e) { + throw new RuntimeException("Subprocess from [" + String.join(" ", command) + "] was interrupted befor it could complete."); + } + } + Process processFromCommand (List command) throws IOException { + for (int i = 0; i < command.size(); i++) { + if (command.get(i) == null) + throw new WdkRuntimeException( + "Command part at index " + i + " is null. Could be due to unchecked user input."); + } + ProcessBuilder pb = new ProcessBuilder(command); + Map env = pb.environment(); + env.put("GUS_HOME", getWdkModel().getGusHome()); + pb.redirectErrorStream(true); + return pb.start(); + } +}