From aa022a7279ba4085740f85459e2c471c75f9ece2 Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Fri, 21 Feb 2025 16:27:05 -0500 Subject: [PATCH 01/13] initial draft --- .../services/jbrowse/JBrowse2Service.java | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java new file mode 100644 index 000000000..4a7256835 --- /dev/null +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -0,0 +1,211 @@ +package org.apidb.apicommon.service.services.jbrowse; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.gusdb.fgputil.db.runner.SQLRunner; +import org.gusdb.fgputil.db.runner.SQLRunnerException; +import org.gusdb.wdk.model.WdkRuntimeException; +import org.gusdb.wdk.service.service.AbstractWdkService; +import org.json.JSONArray; +import org.json.JSONObject; + +@Path("/jbrowse2") +public class JBrowse2Service extends AbstractWdkService { + + public static String appType = "jbrowse2"; + + @GET + @Path("{tracks}/{publicOrganismAbbrev}/{aaOrNa}/config.json") + @Produces(MediaType.APPLICATION_JSON) + public Response getJbrowseTracks(@PathParam("publicOrganismAbbrev") String publicOrganismAbbrev, + @PathParam("aaOrNa") String aaOrNa, + @PathParam("trackSets") String trackSets) throws IOException { + + boolean isPbrowse = aaOrNa.equals("aa"); + String staticConfigJsonString = getStaticConfigJsonString(publicOrganismAbbrev, isPbrowse, trackSets); + JSONObject staticConfigJson = new JSONObject(staticConfigJsonString); + JSONArray udTracks = getUserDatasetTracks(publicOrganismAbbrev, isPbrowse, trackSets); + staticConfigJson.getJSONArray("tracks").putAll(udTracks); + String jsonString = staticConfigJson.toString(); + return Response.ok(jsonString, MediaType.APPLICATION_JSON).build(); + } + + JSONArray getUserDatasetTracks(String publicOrganismAbbrev, Boolean isPbrowse, String tracksString) { + String buildNumber = getWdkModel().getBuildNumber(); + String projectId = getWdkModel().getProjectId(); + Long userId = getRequestingUser().getUserId(); + String vdiDatasetsDir = getWdkModel().getProperties().get("VDI_DATASETS_DIRECTORY"); + String vdiDatasetsSchema = getWdkModel().getProperties().get("VDI_DATASETS_DIRECTORY"); + String vdiControlSchema = getWdkModel().getProperties().get("VDI_CONTROL_DIRECTORY"); + + +/* +VDI_CONTROL_SCHEMA=VDI_CONTROL_DEV_N +VDI_DATASETS_DIRECTORY=/var/www/Common/userDatasets +VDI_DATASETS_SCHEMA=VDI_DATASETS_DEV_N +/var/www/Common/userDatasets/vdi_datasets_dev_n/build-68/PlasmoDB/ + */ + String udDataPathString = String.join("/", vdiDatasetsDir, vdiDatasetsSchema, "build-" + buildNumber, projectId); + JSONArray udTracks = new JSONArray(); + List trackSetList = Arrays.asList(tracksString.split(",")); + if (trackSetList.contains("rnaseq")) { + udTracks.put(getRnaSeqUdTracks(publicOrganismAbbrev, projectId, vdiControlSchema, vdiDatasetsSchema, + udDataPathString, userId)); + } + return null; + } + + String getStaticConfigJsonString(String publicOrganismAbbrev, boolean isPbrowse, String tracks) throws IOException { + + String gusHome = getWdkModel().getGusHome(); + String projectId = getWdkModel().getProjectId(); + + List command = new ArrayList(); + command.add(gusHome + "/bin/jbrowseTracks"); + command.add(publicOrganismAbbrev); + command.add(projectId); + command.add(String.valueOf(isPbrowse)); + command.add(tracks); + command.add(appType); + command.add("trackListOnly"); + + return stringFromCommand(command); + } + + JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, String vdiControlSchema, String vdiDatasetsSchema, + String udDataPathString, Long userId) { + + DataSource appDs = getWdkModel().getAppDb().getDataSource(); + String sql = "select distinct user_dataset_id, name " + + "from " + vdiControlSchema + ".AvailableUserDatasets " + + "where project_id = '" + projectId + "' " + + "and (type = 'RnaSeq' or type = 'BigWig') " + + "and ((is_public = 1 and is_owner = 1) or user_id = " + userId + ")"; + try { + List userDatasetInfos = new SQLRunner(appDs, sql).executeQuery(rs -> { + List udi = new ArrayList<>(); + while (rs.next()) { + String datasetId = rs.getString(1); + String name = rs.getString(2); + udi.add(new UserDatasetInfo(datasetId, name)); + } + return udi; + }); + + } + catch (SQLRunnerException e) { + throw new PluginModelException("Unable to generate project ID map for organism doc type", e.getCause()); + } + } + +/* + MULTI BIGWIG TRACK EXAMPLE + { + "assemblyNames": [ + "ORG_ABBREV" + ], + "trackId": "VDI_ID", + "name": "VDI_NAME", + "displays": [ + { + "displayId": "wiggle_ApiCommonModel::Model::JBrowseTrackConfig::MultiBigWigTrackConfig::XY=HASH(0x2249320)", + "maxScore": 1000, + "minScore": 1, + "defaultRendering": "multirowxy", + "type": "MultiLinearWiggleDisplay", + "scaleType": "log" + } + ], + "adapter": { + "subadapters": [ + { + "color": "grey", + "name": "FILE_NAME", + "type": "BigWigAdapter", + "bigWigLocation": { + "locationType": "UriLocation", + "uri": "USER_DATASET_PATH/VDI_ID/FILE_NAME" + } + } + } + } + */ + JSONObject createBigwigTrackJson(String vdiId, String vdiName, String fileName, String organismAbbrev, String userDatasetsFilePath) { + JSONObject track = new JSONObject(); + track.put("assemblyNames", new JSONArray().put(organismAbbrev)); + track.put("trackId", vdiId); + track.put("name", vdiName); + JSONObject display = new JSONObject(); + display.put("displayId", "wiggle_ApiCommonModel::Model::JBrowseTrackConfig::MultiBigWigTrackConfig::XY=HASH(0x2249320)"); + display.put("maxScore", 1); + display.put("maxScore", 1000); + display.put("defaultRendering", "multirowxy"); + display.put("type", "MultiLinearWiggleDisplay"); + display.put("scaleType", "log"); + JSONArray displays = new JSONArray().put(display); + track.put("displays", displays); + JSONObject subAdapter = new JSONObject(); + subAdapter.put("color1", "grey"); + subAdapter.put("name", fileName); + subAdapter.put("type", "BigWigAdapter"); + JSONObject location = new JSONObject().put("locationType", "UriLocation"); + location.put("uri", String.join("/", userDatasetsFilePath, vdiId, fileName)); + subAdapter.put("bigWigLocation", location); + return track; + } + + class UserDatasetInfo { + UserDatasetInfo(String id, String name) { + vdiId = id; + vdiName = name; + } + String vdiId; + String vdiName; + List fileNames; + } + + List getUserDatasetInfo() { + return new ArrayList(); + } + + String stringFromCommand(List command) throws IOException { + Process p = processFromCommand(command); + BufferedReader reader = + new BufferedReader(new InputStreamReader(p.getInputStream())); + StringBuilder builder = new StringBuilder(); + String line = null; + while ( (line = reader.readLine()) != null) { + builder.append(line); + builder.append(System.lineSeparator()); + } + return builder.toString(); + } + + Process processFromCommand (List command) throws IOException { + for (int i = 0; i < command.size(); i++) { + if (command.get(i) == null) + throw new WdkRuntimeException( + "Command part at index " + i + " is null. Could be due to unchecked user input."); + } + ProcessBuilder pb = new ProcessBuilder(command); + Map env = pb.environment(); + env.put("GUS_HOME", getWdkModel().getGusHome()); + pb.redirectErrorStream(true); + Process p = pb.start(); + return p; + } +} From 831ab21c41ebc15d105f979d104c792b270c3cba Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Fri, 21 Feb 2025 20:57:01 -0500 Subject: [PATCH 02/13] wip --- .../services/jbrowse/JBrowse2Service.java | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java index 4a7256835..ee8312a24 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -1,6 +1,7 @@ package org.apidb.apicommon.service.services.jbrowse; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; @@ -9,15 +10,13 @@ import java.util.Map; import javax.sql.DataSource; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; +import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.gusdb.fgputil.db.runner.SQLRunner; import org.gusdb.fgputil.db.runner.SQLRunnerException; +import org.gusdb.wdk.model.WdkException; import org.gusdb.wdk.model.WdkRuntimeException; import org.gusdb.wdk.service.service.AbstractWdkService; import org.json.JSONArray; @@ -29,11 +28,11 @@ public class JBrowse2Service extends AbstractWdkService { public static String appType = "jbrowse2"; @GET - @Path("{tracks}/{publicOrganismAbbrev}/{aaOrNa}/config.json") + @Path("orgview/{publicOrganismAbbrev}/{aaOrNa}/config.json") @Produces(MediaType.APPLICATION_JSON) public Response getJbrowseTracks(@PathParam("publicOrganismAbbrev") String publicOrganismAbbrev, @PathParam("aaOrNa") String aaOrNa, - @PathParam("trackSets") String trackSets) throws IOException { + @QueryParam("trackSets") String trackSets) throws IOException, WdkException { boolean isPbrowse = aaOrNa.equals("aa"); String staticConfigJsonString = getStaticConfigJsonString(publicOrganismAbbrev, isPbrowse, trackSets); @@ -44,7 +43,7 @@ public Response getJbrowseTracks(@PathParam("publicOrganismAbbrev") String publi return Response.ok(jsonString, MediaType.APPLICATION_JSON).build(); } - JSONArray getUserDatasetTracks(String publicOrganismAbbrev, Boolean isPbrowse, String tracksString) { + JSONArray getUserDatasetTracks(String publicOrganismAbbrev, Boolean isPbrowse, String tracksString) throws WdkException { String buildNumber = getWdkModel().getBuildNumber(); String projectId = getWdkModel().getProjectId(); Long userId = getRequestingUser().getUserId(); @@ -52,21 +51,14 @@ JSONArray getUserDatasetTracks(String publicOrganismAbbrev, Boolean isPbrowse, S String vdiDatasetsSchema = getWdkModel().getProperties().get("VDI_DATASETS_DIRECTORY"); String vdiControlSchema = getWdkModel().getProperties().get("VDI_CONTROL_DIRECTORY"); - -/* -VDI_CONTROL_SCHEMA=VDI_CONTROL_DEV_N -VDI_DATASETS_DIRECTORY=/var/www/Common/userDatasets -VDI_DATASETS_SCHEMA=VDI_DATASETS_DEV_N -/var/www/Common/userDatasets/vdi_datasets_dev_n/build-68/PlasmoDB/ - */ String udDataPathString = String.join("/", vdiDatasetsDir, vdiDatasetsSchema, "build-" + buildNumber, projectId); JSONArray udTracks = new JSONArray(); List trackSetList = Arrays.asList(tracksString.split(",")); if (trackSetList.contains("rnaseq")) { - udTracks.put(getRnaSeqUdTracks(publicOrganismAbbrev, projectId, vdiControlSchema, vdiDatasetsSchema, + udTracks.put(getRnaSeqUdTracks(publicOrganismAbbrev, projectId, vdiControlSchema, udDataPathString, userId)); } - return null; + return udTracks; } String getStaticConfigJsonString(String publicOrganismAbbrev, boolean isPbrowse, String tracks) throws IOException { @@ -74,7 +66,7 @@ String getStaticConfigJsonString(String publicOrganismAbbrev, boolean isPbrowse, String gusHome = getWdkModel().getGusHome(); String projectId = getWdkModel().getProjectId(); - List command = new ArrayList(); + List command = new ArrayList<>(); command.add(gusHome + "/bin/jbrowseTracks"); command.add(publicOrganismAbbrev); command.add(projectId); @@ -86,8 +78,8 @@ String getStaticConfigJsonString(String publicOrganismAbbrev, boolean isPbrowse, return stringFromCommand(command); } - JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, String vdiControlSchema, String vdiDatasetsSchema, - String udDataPathString, Long userId) { + JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, String vdiControlSchema, + String udDataPathString, Long userId) throws WdkException { DataSource appDs = getWdkModel().getAppDb().getDataSource(); String sql = "select distinct user_dataset_id, name " + @@ -96,20 +88,43 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin "and (type = 'RnaSeq' or type = 'BigWig') " + "and ((is_public = 1 and is_owner = 1) or user_id = " + userId + ")"; try { - List userDatasetInfos = new SQLRunner(appDs, sql).executeQuery(rs -> { - List udi = new ArrayList<>(); + return new SQLRunner(appDs, sql).executeQuery(rs -> { + JSONArray rnaSeqUdTracks = new JSONArray(); while (rs.next()) { String datasetId = rs.getString(1); String name = rs.getString(2); - udi.add(new UserDatasetInfo(datasetId, name)); + List fileNames = getBigwigFileNames(udDataPathString + "/" +datasetId); + for (String fileName : fileNames) { + rnaSeqUdTracks.put(createBigwigTrackJson(datasetId, name, fileName, publicOrganismAbbrev, udDataPathString)); + } } - return udi; + return rnaSeqUdTracks; }); - } catch (SQLRunnerException e) { - throw new PluginModelException("Unable to generate project ID map for organism doc type", e.getCause()); + throw new WdkException("Unable to generate project ID map for organism doc type", e.getCause()); + } + } + + // method written by copilot + public static List getBigwigFileNames(String directoryPath) { + List bwFiles = new ArrayList<>(); + File directory = new File(directoryPath); + + if (directory.isDirectory()) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isFile() && file.getName().endsWith(".bw")) { + bwFiles.add(file.getName()); + } + } + } + } else { + System.out.println("The provided path is not a directory."); } + + return bwFiles; } /* @@ -168,26 +183,12 @@ JSONObject createBigwigTrackJson(String vdiId, String vdiName, String fileName, return track; } - class UserDatasetInfo { - UserDatasetInfo(String id, String name) { - vdiId = id; - vdiName = name; - } - String vdiId; - String vdiName; - List fileNames; - } - - List getUserDatasetInfo() { - return new ArrayList(); - } - String stringFromCommand(List command) throws IOException { Process p = processFromCommand(command); BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream())); StringBuilder builder = new StringBuilder(); - String line = null; + String line; while ( (line = reader.readLine()) != null) { builder.append(line); builder.append(System.lineSeparator()); @@ -205,7 +206,6 @@ Process processFromCommand (List command) throws IOException { Map env = pb.environment(); env.put("GUS_HOME", getWdkModel().getGusHome()); pb.redirectErrorStream(true); - Process p = pb.start(); - return p; + return pb.start(); } } From b10712746c9666d5516ba0e07f601848fc13c510 Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Sat, 22 Feb 2025 10:06:00 -0500 Subject: [PATCH 03/13] wip --- .../services/jbrowse/JBrowse2Service.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java index ee8312a24..a0b73e80f 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -24,32 +24,31 @@ @Path("/jbrowse2") public class JBrowse2Service extends AbstractWdkService { - - public static String appType = "jbrowse2"; + private static final String VDI_DATASET_DIR_KEY = "VDI_DATASETS_DIRECTORY"; + private static final String VDI_CONTROL_SCHEMA_KEY ="VDI_CONTROL_SCHEMA"; + private static final String VDI_DATASET_SCHEMA_KEY ="VDI_DATASETS_SCHEMA"; + private static final String WEB_SVC_DIR_KEY ="WEBSERVICEMIRROR"; @GET - @Path("orgview/{publicOrganismAbbrev}/{aaOrNa}/config.json") + @Path("orgview/{publicOrganismAbbrev}/config.json") @Produces(MediaType.APPLICATION_JSON) - public Response getJbrowseTracks(@PathParam("publicOrganismAbbrev") String publicOrganismAbbrev, - @PathParam("aaOrNa") String aaOrNa, + public Response getJbrowseNaTracks(@PathParam("publicOrganismAbbrev") String publicOrganismAbbrev, @QueryParam("trackSets") String trackSets) throws IOException, WdkException { - - boolean isPbrowse = aaOrNa.equals("aa"); - String staticConfigJsonString = getStaticConfigJsonString(publicOrganismAbbrev, isPbrowse, trackSets); + String staticConfigJsonString = getStaticConfigJsonString(publicOrganismAbbrev, trackSets); JSONObject staticConfigJson = new JSONObject(staticConfigJsonString); - JSONArray udTracks = getUserDatasetTracks(publicOrganismAbbrev, isPbrowse, trackSets); + JSONArray udTracks = getUserDatasetTracks(publicOrganismAbbrev, trackSets); staticConfigJson.getJSONArray("tracks").putAll(udTracks); String jsonString = staticConfigJson.toString(); return Response.ok(jsonString, MediaType.APPLICATION_JSON).build(); } - JSONArray getUserDatasetTracks(String publicOrganismAbbrev, Boolean isPbrowse, String tracksString) throws WdkException { + JSONArray getUserDatasetTracks(String publicOrganismAbbrev, String tracksString) throws WdkException { String buildNumber = getWdkModel().getBuildNumber(); String projectId = getWdkModel().getProjectId(); Long userId = getRequestingUser().getUserId(); - String vdiDatasetsDir = getWdkModel().getProperties().get("VDI_DATASETS_DIRECTORY"); - String vdiDatasetsSchema = getWdkModel().getProperties().get("VDI_DATASETS_DIRECTORY"); - String vdiControlSchema = getWdkModel().getProperties().get("VDI_CONTROL_DIRECTORY"); + String vdiDatasetsDir = getWdkModel().getProperties().get(VDI_DATASET_DIR_KEY); + String vdiDatasetsSchema = getWdkModel().getProperties().get(VDI_DATASET_SCHEMA_KEY); + String vdiControlSchema = getWdkModel().getProperties().get(VDI_CONTROL_SCHEMA_KEY); String udDataPathString = String.join("/", vdiDatasetsDir, vdiDatasetsSchema, "build-" + buildNumber, projectId); JSONArray udTracks = new JSONArray(); @@ -61,19 +60,24 @@ JSONArray getUserDatasetTracks(String publicOrganismAbbrev, Boolean isPbrowse, S return udTracks; } - String getStaticConfigJsonString(String publicOrganismAbbrev, boolean isPbrowse, String tracks) throws IOException { + String getStaticConfigJsonString(String publicOrganismAbbrev, String trackSets) throws IOException { String gusHome = getWdkModel().getGusHome(); String projectId = getWdkModel().getProjectId(); + String buildNumber = getWdkModel().getBuildNumber(); List command = new ArrayList<>(); - command.add(gusHome + "/bin/jbrowseTracks"); + command.add(gusHome + "/bin/jbrowse2config"); + command.add("--orgAbbrev"); command.add(publicOrganismAbbrev); + command.add("--projectId"); command.add(projectId); - command.add(String.valueOf(isPbrowse)); - command.add(tracks); - command.add(appType); - command.add("trackListOnly"); + command.add("--buildNumber"); + command.add(buildNumber); + command.add("--webSvcDir"); + command.add(getWdkModel().getProperties().get(WEB_SVC_DIR_KEY)); + command.add("--trackSets"); + command.add(trackSets); return stringFromCommand(command); } From 077e2ac26b759239101f87b265ad8cf0d1b94ccd Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Sat, 22 Feb 2025 10:08:21 -0500 Subject: [PATCH 04/13] wip --- .../apicommon/service/services/jbrowse/JBrowse2Service.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java index a0b73e80f..fb7b05ed2 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -17,6 +17,7 @@ import org.gusdb.fgputil.db.runner.SQLRunner; import org.gusdb.fgputil.db.runner.SQLRunnerException; import org.gusdb.wdk.model.WdkException; +import org.gusdb.wdk.model.WdkModelException; import org.gusdb.wdk.model.WdkRuntimeException; import org.gusdb.wdk.service.service.AbstractWdkService; import org.json.JSONArray; @@ -111,7 +112,7 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin } // method written by copilot - public static List getBigwigFileNames(String directoryPath) { + public static List getBigwigFileNames(String directoryPath) throws WdkModelException { List bwFiles = new ArrayList<>(); File directory = new File(directoryPath); @@ -125,7 +126,7 @@ public static List getBigwigFileNames(String directoryPath) { } } } else { - System.out.println("The provided path is not a directory."); + throw new WdkModelException("User Dataset directory not found for path: " + directoryPath); } return bwFiles; From f19b132a69e75b2bf5e1e34bca03e4eaf7287489 Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Mon, 24 Feb 2025 11:09:32 -0500 Subject: [PATCH 05/13] wip --- .../service/services/jbrowse/JBrowse2Service.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java index fb7b05ed2..23a7547f8 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -84,14 +84,17 @@ String getStaticConfigJsonString(String publicOrganismAbbrev, String trackSets) } JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, String vdiControlSchema, - String udDataPathString, Long userId) throws WdkException { + String udDataPathString, Long userId) throws WdkModelException { DataSource appDs = getWdkModel().getAppDb().getDataSource(); String sql = "select distinct user_dataset_id, name " + - "from " + vdiControlSchema + ".AvailableUserDatasets " + + "from " + vdiControlSchema + ".AvailableUserDatasets aud, " + + vdiControlSchema + ".dataset_dependency dd " + "where project_id = '" + projectId + "' " + "and (type = 'RnaSeq' or type = 'BigWig') " + - "and ((is_public = 1 and is_owner = 1) or user_id = " + userId + ")"; + "and ((is_public = 1 and is_owner = 1) or user_id = " + userId + ") " + + "and dd.dataset_id = aud.dataset_id " + + " dd.identifier = '" + publicOrganismAbbrev + "'"; try { return new SQLRunner(appDs, sql).executeQuery(rs -> { JSONArray rnaSeqUdTracks = new JSONArray(); @@ -107,12 +110,12 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin }); } catch (SQLRunnerException e) { - throw new WdkException("Unable to generate project ID map for organism doc type", e.getCause()); + throw new WdkModelException("Unable to query VDI tables for RNA seq datasets", e.getCause()); } } // method written by copilot - public static List getBigwigFileNames(String directoryPath) throws WdkModelException { + public static List getBigwigFileNames(String directoryPath) throws SQLRunnerException { List bwFiles = new ArrayList<>(); File directory = new File(directoryPath); @@ -126,7 +129,7 @@ public static List getBigwigFileNames(String directoryPath) throws WdkMo } } } else { - throw new WdkModelException("User Dataset directory not found for path: " + directoryPath); + throw new SQLRunnerException("User Dataset directory not found for path: " + directoryPath); } return bwFiles; From 161b8966cb76dec8105c890708967fca3b75aa92 Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Mon, 24 Feb 2025 13:38:32 -0500 Subject: [PATCH 06/13] wip --- .../services/jbrowse/JBrowse2Service.java | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java index 23a7547f8..e647630f6 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -30,37 +30,31 @@ public class JBrowse2Service extends AbstractWdkService { private static final String VDI_DATASET_SCHEMA_KEY ="VDI_DATASETS_SCHEMA"; private static final String WEB_SVC_DIR_KEY ="WEBSERVICEMIRROR"; + /* + Get config for a single organism. Assumes JSON will easily fit in memory. + */ @GET @Path("orgview/{publicOrganismAbbrev}/config.json") @Produces(MediaType.APPLICATION_JSON) - public Response getJbrowseNaTracks(@PathParam("publicOrganismAbbrev") String publicOrganismAbbrev, + public Response getJbrowseSingleOrgTracks(@PathParam("publicOrganismAbbrev") String publicOrganismAbbrev, @QueryParam("trackSets") String trackSets) throws IOException, WdkException { + + // get static json config, for this organism and set of tracks String staticConfigJsonString = getStaticConfigJsonString(publicOrganismAbbrev, trackSets); JSONObject staticConfigJson = new JSONObject(staticConfigJsonString); + + // get similar from user datasets JSONArray udTracks = getUserDatasetTracks(publicOrganismAbbrev, trackSets); + + // merge UD tracks into static staticConfigJson.getJSONArray("tracks").putAll(udTracks); + + // send response String jsonString = staticConfigJson.toString(); return Response.ok(jsonString, MediaType.APPLICATION_JSON).build(); } - JSONArray getUserDatasetTracks(String publicOrganismAbbrev, String tracksString) throws WdkException { - String buildNumber = getWdkModel().getBuildNumber(); - String projectId = getWdkModel().getProjectId(); - Long userId = getRequestingUser().getUserId(); - String vdiDatasetsDir = getWdkModel().getProperties().get(VDI_DATASET_DIR_KEY); - String vdiDatasetsSchema = getWdkModel().getProperties().get(VDI_DATASET_SCHEMA_KEY); - String vdiControlSchema = getWdkModel().getProperties().get(VDI_CONTROL_SCHEMA_KEY); - - String udDataPathString = String.join("/", vdiDatasetsDir, vdiDatasetsSchema, "build-" + buildNumber, projectId); - JSONArray udTracks = new JSONArray(); - List trackSetList = Arrays.asList(tracksString.split(",")); - if (trackSetList.contains("rnaseq")) { - udTracks.put(getRnaSeqUdTracks(publicOrganismAbbrev, projectId, vdiControlSchema, - udDataPathString, userId)); - } - return udTracks; - } - + // call out to perl code to produce static config json String getStaticConfigJsonString(String publicOrganismAbbrev, String trackSets) throws IOException { String gusHome = getWdkModel().getGusHome(); @@ -83,6 +77,26 @@ String getStaticConfigJsonString(String publicOrganismAbbrev, String trackSets) return stringFromCommand(command); } + JSONArray getUserDatasetTracks(String publicOrganismAbbrev, String tracksString) throws WdkModelException { + String buildNumber = getWdkModel().getBuildNumber(); + String projectId = getWdkModel().getProjectId(); + Long userId = getRequestingUser().getUserId(); + String vdiDatasetsDir = getWdkModel().getProperties().get(VDI_DATASET_DIR_KEY); + String vdiDatasetsSchema = getWdkModel().getProperties().get(VDI_DATASET_SCHEMA_KEY); + String vdiControlSchema = getWdkModel().getProperties().get(VDI_CONTROL_SCHEMA_KEY); + + String udDataPathString = String.join("/", vdiDatasetsDir, vdiDatasetsSchema, "build-" + buildNumber, projectId); + JSONArray udTracks = new JSONArray(); + List trackSetList = Arrays.asList(tracksString.split(",")); + + // for now we only have rnaseq UD tracks + if (trackSetList.contains("rnaseq")) { + udTracks.put(getRnaSeqUdTracks(publicOrganismAbbrev, projectId, vdiControlSchema, + udDataPathString, userId)); + } + return udTracks; + } + JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, String vdiControlSchema, String udDataPathString, Long userId) throws WdkModelException { @@ -114,7 +128,7 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin } } - // method written by copilot + // boilerplate method written by copilot public static List getBigwigFileNames(String directoryPath) throws SQLRunnerException { List bwFiles = new ArrayList<>(); File directory = new File(directoryPath); From e676d615a03402bf8dd5292e6fa1c3d17738f0a2 Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Wed, 26 Feb 2025 17:42:36 -0500 Subject: [PATCH 07/13] wip --- .../services/jbrowse/JBrowse2Service.java | 80 +++++++++++-------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java index e647630f6..57c5fea9f 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -1,9 +1,6 @@ package org.apidb.apicommon.service.services.jbrowse; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; +import java.io.*; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -25,11 +22,11 @@ @Path("/jbrowse2") public class JBrowse2Service extends AbstractWdkService { - private static final String VDI_DATASET_DIR_KEY = "VDI_DATASETS_DIRECTORY"; private static final String VDI_CONTROL_SCHEMA_KEY ="VDI_CONTROL_SCHEMA"; private static final String VDI_DATASET_SCHEMA_KEY ="VDI_DATASETS_SCHEMA"; private static final String WEB_SVC_DIR_KEY ="WEBSERVICEMIRROR"; + private static final String USER_DATASETS_DIR = "./userDatasetsData"; // hard-coded mount point in the jbrowse2 service /* Get config for a single organism. Assumes JSON will easily fit in memory. */ @@ -81,11 +78,10 @@ JSONArray getUserDatasetTracks(String publicOrganismAbbrev, String tracksString) String buildNumber = getWdkModel().getBuildNumber(); String projectId = getWdkModel().getProjectId(); Long userId = getRequestingUser().getUserId(); - String vdiDatasetsDir = getWdkModel().getProperties().get(VDI_DATASET_DIR_KEY); String vdiDatasetsSchema = getWdkModel().getProperties().get(VDI_DATASET_SCHEMA_KEY); String vdiControlSchema = getWdkModel().getProperties().get(VDI_CONTROL_SCHEMA_KEY); - String udDataPathString = String.join("/", vdiDatasetsDir, vdiDatasetsSchema, "build-" + buildNumber, projectId); + String udDataPathString = String.join("/", USER_DATASETS_DIR, vdiDatasetsSchema, "build-" + buildNumber, projectId); JSONArray udTracks = new JSONArray(); List trackSetList = Arrays.asList(tracksString.split(",")); @@ -115,9 +111,12 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin while (rs.next()) { String datasetId = rs.getString(1); String name = rs.getString(2); + JSONObject track = createBigwigTrackJson(datasetId, name, publicOrganismAbbrev); List fileNames = getBigwigFileNames(udDataPathString + "/" +datasetId); for (String fileName : fileNames) { - rnaSeqUdTracks.put(createBigwigTrackJson(datasetId, name, fileName, publicOrganismAbbrev, udDataPathString)); + track.getJSONObject("adapter") + .getJSONArray("subadapters") + .put(createBigwigSubadapterJson(datasetId, fileName, udDataPathString)); } } return rnaSeqUdTracks; @@ -181,20 +180,27 @@ public static List getBigwigFileNames(String directoryPath) throws SQLRu } } */ - JSONObject createBigwigTrackJson(String vdiId, String vdiName, String fileName, String organismAbbrev, String userDatasetsFilePath) { - JSONObject track = new JSONObject(); - track.put("assemblyNames", new JSONArray().put(organismAbbrev)); - track.put("trackId", vdiId); - track.put("name", vdiName); - JSONObject display = new JSONObject(); - display.put("displayId", "wiggle_ApiCommonModel::Model::JBrowseTrackConfig::MultiBigWigTrackConfig::XY=HASH(0x2249320)"); - display.put("maxScore", 1); - display.put("maxScore", 1000); - display.put("defaultRendering", "multirowxy"); - display.put("type", "MultiLinearWiggleDisplay"); - display.put("scaleType", "log"); - JSONArray displays = new JSONArray().put(display); - track.put("displays", displays); + JSONObject createBigwigTrackJson(String vdiId, String vdiName, String organismAbbrev) { + return new JSONObject() + .put("assemblyNames", new JSONArray().put(organismAbbrev)) + .put("trackId", vdiId) + .put("name", vdiName) + .put("displays", new JSONArray() + .put(new JSONObject() + .put("displayId", "wiggle_ApiCommonModel::Model::JBrowseTrackConfig::MultiBigWigTrackConfig::XY=HASH(0x2249320)") + .put("maxScore", 1) + .put("maxScore", 1000) + .put("defaultRendering", "multirowxy") + .put("type", "MultiLinearWiggleDisplay") + .put("scaleType", "log") + ) + ) + .put("adapter", new JSONObject() + .put("subadapters", new JSONArray()) + ); + } + + JSONObject createBigwigSubadapterJson(String vdiId, String fileName, String userDatasetsFilePath) { JSONObject subAdapter = new JSONObject(); subAdapter.put("color1", "grey"); subAdapter.put("name", fileName); @@ -202,22 +208,30 @@ JSONObject createBigwigTrackJson(String vdiId, String vdiName, String fileName, JSONObject location = new JSONObject().put("locationType", "UriLocation"); location.put("uri", String.join("/", userDatasetsFilePath, vdiId, fileName)); subAdapter.put("bigWigLocation", location); - return track; + return subAdapter; } String stringFromCommand(List command) throws IOException { - Process p = processFromCommand(command); - BufferedReader reader = - new BufferedReader(new InputStreamReader(p.getInputStream())); - StringBuilder builder = new StringBuilder(); - String line; - while ( (line = reader.readLine()) != null) { - builder.append(line); - builder.append(System.lineSeparator()); + try { + Process p = processFromCommand(command); + + ByteArrayOutputStream stringBuffer = new ByteArrayOutputStream(); + p.getErrorStream().transferTo(stringBuffer); + String errors = stringBuffer.toString(); + + stringBuffer.reset(); + p.getInputStream().transferTo(stringBuffer); + + if (p.waitFor() != 0) { + throw new RuntimeException("Subprocess from [" + String.join(" ", command) + "] returned non-zero. Errors:\n" + errors); + } + + return stringBuffer.toString(); + } + catch (InterruptedException e) { + throw new RuntimeException("Subprocess from [" + String.join(" ", command) + "] was interrupted befor it could complete."); } - return builder.toString(); } - Process processFromCommand (List command) throws IOException { for (int i = 0; i < command.size(); i++) { if (command.get(i) == null) From fd084161228af300cec5c52e3b83bf068ce5c48f Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Wed, 26 Feb 2025 18:01:33 -0500 Subject: [PATCH 08/13] wip --- .../org/apidb/apicommon/service/ApiWebServiceApplication.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Service/src/main/java/org/apidb/apicommon/service/ApiWebServiceApplication.java b/Service/src/main/java/org/apidb/apicommon/service/ApiWebServiceApplication.java index e28b5fcb6..e5a1e325b 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/ApiWebServiceApplication.java +++ b/Service/src/main/java/org/apidb/apicommon/service/ApiWebServiceApplication.java @@ -13,6 +13,7 @@ import org.apidb.apicommon.service.services.comments.AttachmentsService; import org.apidb.apicommon.service.services.comments.UserCommentsService; import org.apidb.apicommon.service.services.dataPlotter.ProfileSetService; +import org.apidb.apicommon.service.services.jbrowse.JBrowse2Service; import org.apidb.apicommon.service.services.jbrowse.JBrowseService; import org.apidb.apicommon.service.services.jbrowse.JBrowseUserDatasetsService; import org.eupathdb.common.service.EuPathServiceApplication; @@ -46,6 +47,7 @@ public Set> getClasses() { .add(UserCommentsService.class) .add(TranscriptToggleService.class) .add(JBrowseService.class) + .add(JBrowse2Service.class) .add(JBrowseUserDatasetsService.class) .add(ProfileSetService.class) .add(OrganismMetricsService.class) From faa7bba60cde3c585e368a8312c4d80c73f7ae54 Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Thu, 27 Feb 2025 11:53:06 -0500 Subject: [PATCH 09/13] wip --- .../services/jbrowse/JBrowse2Service.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java index 57c5fea9f..df672c84a 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -10,6 +10,7 @@ import javax.ws.rs.*; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.BadRequestException; import org.gusdb.fgputil.db.runner.SQLRunner; import org.gusdb.fgputil.db.runner.SQLRunnerException; @@ -34,14 +35,19 @@ public class JBrowse2Service extends AbstractWdkService { @Path("orgview/{publicOrganismAbbrev}/config.json") @Produces(MediaType.APPLICATION_JSON) public Response getJbrowseSingleOrgTracks(@PathParam("publicOrganismAbbrev") String publicOrganismAbbrev, - @QueryParam("trackSets") String trackSets) throws IOException, WdkException { + @QueryParam("trackSets") String trackSetsString) throws IOException, WdkException { + String errMsg = "Must provide a comma delimited list of tracks in a 'trackSets' query param"; + if (trackSetsString == null) throw new BadRequestException(errMsg); + List trackSetsList = Arrays.asList(trackSetsString.split(",")); + if (trackSetsList.isEmpty()) throw new BadRequestException(errMsg); + // get static json config, for this organism and set of tracks - String staticConfigJsonString = getStaticConfigJsonString(publicOrganismAbbrev, trackSets); + String staticConfigJsonString = getStaticConfigJsonString(publicOrganismAbbrev, trackSetsString); JSONObject staticConfigJson = new JSONObject(staticConfigJsonString); // get similar from user datasets - JSONArray udTracks = getUserDatasetTracks(publicOrganismAbbrev, trackSets); + JSONArray udTracks = getUserDatasetTracks(publicOrganismAbbrev, trackSetsList); // merge UD tracks into static staticConfigJson.getJSONArray("tracks").putAll(udTracks); @@ -52,14 +58,14 @@ public Response getJbrowseSingleOrgTracks(@PathParam("publicOrganismAbbrev") Str } // call out to perl code to produce static config json - String getStaticConfigJsonString(String publicOrganismAbbrev, String trackSets) throws IOException { + String getStaticConfigJsonString(String publicOrganismAbbrev, String trackSetsString) throws IOException { String gusHome = getWdkModel().getGusHome(); String projectId = getWdkModel().getProjectId(); String buildNumber = getWdkModel().getBuildNumber(); List command = new ArrayList<>(); - command.add(gusHome + "/bin/jbrowse2config"); + command.add(gusHome + "/bin/jbrowse2Config"); command.add("--orgAbbrev"); command.add(publicOrganismAbbrev); command.add("--projectId"); @@ -69,12 +75,12 @@ String getStaticConfigJsonString(String publicOrganismAbbrev, String trackSets) command.add("--webSvcDir"); command.add(getWdkModel().getProperties().get(WEB_SVC_DIR_KEY)); command.add("--trackSets"); - command.add(trackSets); + command.add(trackSetsString); return stringFromCommand(command); } - JSONArray getUserDatasetTracks(String publicOrganismAbbrev, String tracksString) throws WdkModelException { + JSONArray getUserDatasetTracks(String publicOrganismAbbrev, List trackSetList) throws WdkModelException { String buildNumber = getWdkModel().getBuildNumber(); String projectId = getWdkModel().getProjectId(); Long userId = getRequestingUser().getUserId(); @@ -83,7 +89,6 @@ JSONArray getUserDatasetTracks(String publicOrganismAbbrev, String tracksString) String udDataPathString = String.join("/", USER_DATASETS_DIR, vdiDatasetsSchema, "build-" + buildNumber, projectId); JSONArray udTracks = new JSONArray(); - List trackSetList = Arrays.asList(tracksString.split(",")); // for now we only have rnaseq UD tracks if (trackSetList.contains("rnaseq")) { From 2f19f58eaada732dfbfa88c770010273df246973 Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Thu, 27 Feb 2025 14:11:32 -0500 Subject: [PATCH 10/13] wip --- .../service/services/jbrowse/JBrowse2Service.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java index df672c84a..0cc56799f 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -12,6 +12,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.BadRequestException; +import org.apache.log4j.Logger; import org.gusdb.fgputil.db.runner.SQLRunner; import org.gusdb.fgputil.db.runner.SQLRunnerException; import org.gusdb.wdk.model.WdkException; @@ -23,6 +24,9 @@ @Path("/jbrowse2") public class JBrowse2Service extends AbstractWdkService { + + private static final Logger LOG = Logger.getLogger(JBrowse2Service.class); + private static final String VDI_CONTROL_SCHEMA_KEY ="VDI_CONTROL_SCHEMA"; private static final String VDI_DATASET_SCHEMA_KEY ="VDI_DATASETS_SCHEMA"; private static final String WEB_SVC_DIR_KEY ="WEBSERVICEMIRROR"; @@ -106,10 +110,10 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin "from " + vdiControlSchema + ".AvailableUserDatasets aud, " + vdiControlSchema + ".dataset_dependency dd " + "where project_id = '" + projectId + "' " + - "and (type = 'RnaSeq' or type = 'BigWig') " + + "and (type = 'rnaseq' or type = 'bigwigfiles') " + "and ((is_public = 1 and is_owner = 1) or user_id = " + userId + ") " + - "and dd.dataset_id = aud.dataset_id " + - " dd.identifier = '" + publicOrganismAbbrev + "'"; + "and dd.dataset_id = aud.user_dataset_id " + + "and dd.identifier = '" + publicOrganismAbbrev + "'"; try { return new SQLRunner(appDs, sql).executeQuery(rs -> { JSONArray rnaSeqUdTracks = new JSONArray(); @@ -128,7 +132,7 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin }); } catch (SQLRunnerException e) { - throw new WdkModelException("Unable to query VDI tables for RNA seq datasets", e.getCause()); + throw new WdkModelException("Unable to query VDI tables for RNA seq datasets. " + e.getMessage(), e.getCause()); } } @@ -217,6 +221,7 @@ JSONObject createBigwigSubadapterJson(String vdiId, String fileName, String user } String stringFromCommand(List command) throws IOException { + LOG.debug("Running command: " + String.join(" ", command)); try { Process p = processFromCommand(command); From 25a2dab9d134c4a8b16d8e4a473bb6cdb90df141 Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Thu, 6 Mar 2025 12:28:41 -0500 Subject: [PATCH 11/13] correct UD dir paths --- .../services/jbrowse/JBrowse2Service.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java index 0cc56799f..be98e9dfc 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -27,11 +27,12 @@ public class JBrowse2Service extends AbstractWdkService { private static final Logger LOG = Logger.getLogger(JBrowse2Service.class); + private static final String VDI_DATASETS_DIRECTORY_KEY ="VDI_DATASETS_DIRECTORY"; private static final String VDI_CONTROL_SCHEMA_KEY ="VDI_CONTROL_SCHEMA"; private static final String VDI_DATASET_SCHEMA_KEY ="VDI_DATASETS_SCHEMA"; private static final String WEB_SVC_DIR_KEY ="WEBSERVICEMIRROR"; - private static final String USER_DATASETS_DIR = "./userDatasetsData"; // hard-coded mount point in the jbrowse2 service + private static final String SVC_USER_DATASETS_DIR = "./userDatasetsData"; // hard-coded mount point in the jbrowse2 service /* Get config for a single organism. Assumes JSON will easily fit in memory. */ @@ -90,20 +91,22 @@ JSONArray getUserDatasetTracks(String publicOrganismAbbrev, List trackSe Long userId = getRequestingUser().getUserId(); String vdiDatasetsSchema = getWdkModel().getProperties().get(VDI_DATASET_SCHEMA_KEY); String vdiControlSchema = getWdkModel().getProperties().get(VDI_CONTROL_SCHEMA_KEY); + String vdiDatasetsDir = getWdkModel().getProperties().get(VDI_DATASETS_DIRECTORY_KEY); - String udDataPathString = String.join("/", USER_DATASETS_DIR, vdiDatasetsSchema, "build-" + buildNumber, projectId); + String svcUserDataPathString = String.join("/", SVC_USER_DATASETS_DIR, vdiDatasetsSchema, "build-" + buildNumber, projectId); + String wdkUserDatasetsPathString = String.join("/", vdiDatasetsDir, vdiDatasetsSchema, "build-" + buildNumber, projectId); JSONArray udTracks = new JSONArray(); // for now we only have rnaseq UD tracks if (trackSetList.contains("rnaseq")) { - udTracks.put(getRnaSeqUdTracks(publicOrganismAbbrev, projectId, vdiControlSchema, - udDataPathString, userId)); + udTracks.put(getRnaSeqUdTracks(publicOrganismAbbrev, projectId, vdiControlSchema, wdkUserDatasetsPathString, + svcUserDataPathString, userId)); } return udTracks; } JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, String vdiControlSchema, - String udDataPathString, Long userId) throws WdkModelException { + String svcUserDataPathString, String wdkUserDatasetsPathString, Long userId) throws WdkModelException { DataSource appDs = getWdkModel().getAppDb().getDataSource(); String sql = "select distinct user_dataset_id, name " + @@ -121,11 +124,11 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin String datasetId = rs.getString(1); String name = rs.getString(2); JSONObject track = createBigwigTrackJson(datasetId, name, publicOrganismAbbrev); - List fileNames = getBigwigFileNames(udDataPathString + "/" +datasetId); + List fileNames = getBigwigFileNames(wdkUserDatasetsPathString + "/" +datasetId); for (String fileName : fileNames) { track.getJSONObject("adapter") .getJSONArray("subadapters") - .put(createBigwigSubadapterJson(datasetId, fileName, udDataPathString)); + .put(createBigwigSubadapterJson(datasetId, fileName, svcUserDataPathString)); } } return rnaSeqUdTracks; From c43a3347cda41387a8885a88929472e43056b61c Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Thu, 6 Mar 2025 13:57:12 -0500 Subject: [PATCH 12/13] debug --- .../services/jbrowse/JBrowse2Service.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java index be98e9dfc..d29408f44 100644 --- a/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java +++ b/Service/src/main/java/org/apidb/apicommon/service/services/jbrowse/JBrowse2Service.java @@ -93,8 +93,9 @@ JSONArray getUserDatasetTracks(String publicOrganismAbbrev, List trackSe String vdiControlSchema = getWdkModel().getProperties().get(VDI_CONTROL_SCHEMA_KEY); String vdiDatasetsDir = getWdkModel().getProperties().get(VDI_DATASETS_DIRECTORY_KEY); - String svcUserDataPathString = String.join("/", SVC_USER_DATASETS_DIR, vdiDatasetsSchema, "build-" + buildNumber, projectId); - String wdkUserDatasetsPathString = String.join("/", vdiDatasetsDir, vdiDatasetsSchema, "build-" + buildNumber, projectId); + String path = String.join("/", vdiDatasetsSchema.toLowerCase(), "build-" + buildNumber, projectId); + String svcUserDataPathString = SVC_USER_DATASETS_DIR + "/" + path; + String wdkUserDatasetsPathString = vdiDatasetsDir + "/" + path; JSONArray udTracks = new JSONArray(); // for now we only have rnaseq UD tracks @@ -106,7 +107,7 @@ JSONArray getUserDatasetTracks(String publicOrganismAbbrev, List trackSe } JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, String vdiControlSchema, - String svcUserDataPathString, String wdkUserDatasetsPathString, Long userId) throws WdkModelException { + String wdkUserDatasetsPathString, String svcUserDatasetsPathString, Long userId) throws WdkModelException { DataSource appDs = getWdkModel().getAppDb().getDataSource(); String sql = "select distinct user_dataset_id, name " + @@ -117,6 +118,7 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin "and ((is_public = 1 and is_owner = 1) or user_id = " + userId + ") " + "and dd.dataset_id = aud.user_dataset_id " + "and dd.identifier = '" + publicOrganismAbbrev + "'"; + try { return new SQLRunner(appDs, sql).executeQuery(rs -> { JSONArray rnaSeqUdTracks = new JSONArray(); @@ -124,11 +126,13 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin String datasetId = rs.getString(1); String name = rs.getString(2); JSONObject track = createBigwigTrackJson(datasetId, name, publicOrganismAbbrev); - List fileNames = getBigwigFileNames(wdkUserDatasetsPathString + "/" +datasetId); + rnaSeqUdTracks.put(track); + List fileNames = getBigwigFileNames(wdkUserDatasetsPathString + "/" + datasetId); for (String fileName : fileNames) { - track.getJSONObject("adapter") + + track.getJSONObject("adapter") .getJSONArray("subadapters") - .put(createBigwigSubadapterJson(datasetId, fileName, svcUserDataPathString)); + .put(createBigwigSubadapterJson(datasetId, fileName, svcUserDatasetsPathString)); } } return rnaSeqUdTracks; @@ -141,6 +145,7 @@ JSONArray getRnaSeqUdTracks(String publicOrganismAbbrev, String projectId, Strin // boilerplate method written by copilot public static List getBigwigFileNames(String directoryPath) throws SQLRunnerException { + List bwFiles = new ArrayList<>(); File directory = new File(directoryPath); @@ -148,6 +153,7 @@ public static List getBigwigFileNames(String directoryPath) throws SQLRu File[] files = directory.listFiles(); if (files != null) { for (File file : files) { + if (file.isFile() && file.getName().endsWith(".bw")) { bwFiles.add(file.getName()); } From f4270d5bf0f130f6c89148d276dfc8968aa60ee8 Mon Sep 17 00:00:00 2001 From: steve-fischer-200 Date: Thu, 18 Sep 2025 17:41:48 -0400 Subject: [PATCH 13/13] draft of Jbrowse2ConfigReporter --- .../model/report/Jbrowse2ConfigReporter.java | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 Model/src/main/java/org/apidb/apicommon/model/report/Jbrowse2ConfigReporter.java diff --git a/Model/src/main/java/org/apidb/apicommon/model/report/Jbrowse2ConfigReporter.java b/Model/src/main/java/org/apidb/apicommon/model/report/Jbrowse2ConfigReporter.java new file mode 100644 index 000000000..0dc1ada0a --- /dev/null +++ b/Model/src/main/java/org/apidb/apicommon/model/report/Jbrowse2ConfigReporter.java @@ -0,0 +1,193 @@ +package org.gusdb.wdk.model.report.reporter; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.log4j.Logger; +import org.gusdb.wdk.model.WdkModelException; +import org.gusdb.wdk.model.WdkUserException; +import org.gusdb.wdk.model.answer.AnswerValue; +import org.gusdb.wdk.model.answer.stream.RecordStream; +import org.gusdb.wdk.model.answer.stream.RecordStreamFactory; +import org.gusdb.wdk.model.record.RecordInstance; +import org.gusdb.wdk.model.record.TableField; +import org.gusdb.wdk.model.record.TableValue; +import org.gusdb.wdk.model.record.TableValueRow; +import org.gusdb.wdk.model.record.attribute.AttributeField; +import org.gusdb.wdk.model.record.attribute.AttributeValue; +import org.gusdb.wdk.model.report.AbstractReporter; +import org.gusdb.wdk.model.report.PropertiesProvider; +import org.gusdb.wdk.model.report.Reporter; +import org.gusdb.wdk.model.report.ReporterConfigException; +import org.gusdb.wdk.model.report.util.TableCache; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONWriter; + +/** + * @author Steve + * + */ +public class Jbrowse2ConfigReporter extends AbstractReporter { + + private static final Logger LOG = Logger.getLogger(Jbrowse2ConfigReporter.class); + + private TableCache _tableCache; + + @Override + public Jbrowse2ConfigReporter setProperties(PropertiesProvider reporterRef) throws WdkModelException { + super.setProperties(reporterRef); + String cacheTableName = TableCache.getCacheTableName(_properties); + if (cacheTableName != null) { + _tableCache = new TableCache(getQuestion().getRecordClass(), _wdkModel.getAppDb(), cacheTableName); + } + return this; + } + + @Override + public Reporter configure(JSONObject config) throws ReporterConfigException, WdkModelException { + return null; + } + + @Override + public String getHttpContentType() { + return "application/json"; + } + + @Override + public String getDownloadFileName() { + return getQuestion().getName() + "_detail.json"; + } + + @Override + public void write(OutputStream out) throws WdkModelException { + + Map attrFieldMap = getQuestion().getAttributeFieldMap(); + if (!attrFieldMap.containsKey("short_display_name")) throw new WdkModelException(); + AttributeField af = attrFieldMap.get("short_display_name"); + Set afs = new HashSet<>(); + afs.add(af); + + Map tblFieldMap = getQuestion().; + if (!tblFieldMap.containsKey("short_display_name")) throw new WdkModelException(); + TableField tf = tblFieldMap.get("short_display_name"); + Set afs = new HashSet<>(); + afs.add(af); + + // TableField udDependenciesTable = (TableField) fieldMap.get("UdDependencies"); + + try (RecordStream records = RecordStreamFactory.getRecordStream(_baseAnswer, afs, tables)) { + OutputStreamWriter streamWriter = new OutputStreamWriter(out); + JSONWriter writer = new JSONWriter(streamWriter); + + AnswerValue av = _baseAnswer; + writer.object().key("response").object().key("recordset").object().key("id").value( + av.getChecksum()).key("count").value(this.getResultSize()).key("type").value( + av.getAnswerSpec().getQuestion().getRecordClass().getDisplayName()).key("records").array(); + + if (_tableCache != null) { + _tableCache.open(); + } + + // get page based answers with a maximum size (defined in PageAnswerIterator) + int recordCount = 0; + for (RecordInstance record : records) { + + // get VDI ID + String datasetId = record.getPrimaryKey().getValues().get("dataset_id"); + String suffix = "EDAUD_"; + if (!datasetId.startsWith(suffix)) throw new WdkUserException(); + String vdiId = datasetId.substring(suffix.length()); + + // get display name + String shortDisplayName = record.getAttributeValue("short_display_name").getValue(); + + // get organism name for files + TableValue tv = record.getTableValue("UdDependencies"); + if (tv.getNumRows() != 1) throw new WdkUserException(); + TableValueRow tvr = tv.iterator().next(); + if (!tvr.containsKey("identifier")) throw new WdkUserException(); + String orgNameForFiles = tvr.get("identifier").getValue(); + + // count the records processed so far + recordCount++; + writer.endObject(); + streamWriter.flush(); + } + + writer.endArray() // records + .endObject().endObject().endObject(); + streamWriter.flush(); + LOG.info("Totally " + recordCount + " records dumped"); + } catch (WdkUserException | JSONException | SQLException | IOException e) { + throw new WdkModelException("Unable to write JSON report", e); + } finally { + if (_tableCache != null) { + _tableCache.close(); + } + } + } + + /* + { + "assemblyNames": [ + "VDI_ORGANISM_FOR_FILES" + ], + "trackId": "UD_VDI_ID", + "name": "SHORT_DISPLAY_NAME", + "displays": [ + { + "displayId": "wiggle_ApiCommonModel::Model::JBrowseTrackConfig::MultiBigWigTrackConfig::XY=HASH(0x2249320)", + "maxScore": 1000, + "minScore": 1, + "defaultRendering": "multirowxy", + "type": "MultiLinearWiggleDisplay", + "scaleType": "log" + } + ], + "adapter": { + "subadapters": [ + { + "color": "grey", + "name": "Unsporulated oocyst (non-unique)", + "type": "BigWigAdapter", + "bigWigLocation": { + "locationType": "UriLocation", + "uri": "ToxoDB/build-68/EtenellaHoughton2021/bigwig/etenHoughton2021_Reid_RNASeq_ebi_rnaSeq_RSRC/1_Unsporulated_oocyst/non_ +unique_resultsCombinedReps_unlogged.bw" + } + } +*/ + + + private static void formatAttributes(RecordInstance record, Set attributes, JSONWriter writer) + throws WdkModelException, WdkUserException { + if (!attributes.isEmpty()) { + writer.key("fields").array(); + for (AttributeField field : attributes) { + AttributeValue value = record.getAttributeValue(field.getName()); + writer.object().key("name").value(field.getName()).key("value").value(value.getValue()).endObject(); + } + writer.endArray(); + } + } + + /* + - dataset id + - + */ + private Set getAttributes() { + return null; + } + private Set getTables() { + return null; + } + +} + +