Skip to content

Fix data import from URL #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nicodr97 merged 7 commits into from
Jul 1, 2025
Merged

Fix data import from URL #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ee217aa
refactor: simplify conditionals
nicodr97 Jun 26, 2025
ab4cdc7
fix: remove repetitive slash from url
nicodr97 Jun 26, 2025
9a0c6d2
chore: autoformat
nicodr97 Jun 26, 2025
8c45826
fix: add missing download script for urls
nicodr97 Jun 26, 2025
b38a77b
chore: autoformat
nicodr97 Jun 26, 2025
88c87e9
feat: do not show warning of missing file for internal tools
nicodr97 Jun 26, 2025
140ef4f
feat: add wget to default pre-installed tools
nicodr97 Jun 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,8 @@ services:
- /var/run/docker.sock:/var/run/docker.sock:rprivate
- ./sge/configuration:/tmp/configuration:rprivate
- ./sge/configuration:/etc/gridengine/configuration:rprivate
- ./sge/shared_scripts:/shared_scripts:rshared
- ./volumes/shared_data:/shared_data:rshared
- ./volumes/shared_scripts:/shared_scripts:rshared
devices:
- '/dev/fuse:/dev/fuse'
cap_add:
Expand Down
178 changes: 146 additions & 32 deletions front_end/openVRE/install/database/tools.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[
{
"_id": "tool_skeleton",
"name": "Tool Skeleton",
Expand All @@ -13,60 +14,85 @@
},
"status": 1,
"external": true,
"keywords": ["next gen seq", "dna"],
"keywords_tool": ["Tool skeleton", "Sample keyword"],
"keywords": [
"next gen seq",
"dna"
],
"keywords_tool": [
"Tool skeleton",
"Sample keyword"
],
"infrastructure": {
"memory": 12,
"cpus": 1,
"executable": "/home/vre_template_tool/VRE_RUNNER",
"container_image": "dockerhub/tool-skeleton:latest",
"clouds": {
"local": {
"launcher": "docker_SGE",
"default_cloud": true,
"queue": "testq",
"executable": "/home/vre_template_tool/VRE_RUNNER",
"executable_type": "docker"
"memory": 12,
"cpus": 1,
"executable": "/home/vre_template_tool/VRE_RUNNER",
"container_image": "dockerhub/tool-skeleton:latest",
"clouds": {
"local": {
"launcher": "docker_SGE",
"default_cloud": true,
"queue": "testq",
"executable": "/home/vre_template_tool/VRE_RUNNER",
"executable_type": "docker"
}
}
}
},
},
"input_files": [
{
"name": "fasta1",
"description": "FASTA input file 1",
"help": "This is the input file 1 help tooltip",
"file_type": ["FASTA"],
"data_type": ["dna_sequence"],
"file_type": [
"FASTA"
],
"data_type": [
"dna_sequence"
],
"required": true,
"allow_multiple": false
},
{
"name": "fasta2",
"description": "FASTA input file 2",
"help": "This is the input file 2 help tooltip",
"file_type": ["FASTA"],
"data_type": ["dna_sequence"],
"file_type": [
"FASTA"
],
"data_type": [
"dna_sequence"
],
"required": false,
"allow_multiple": false
}
],
"input_files_combinations": [
{
"description": "Start tool with 1 FASTA FILE",
"input_files": ["fasta1"]
"input_files": [
"fasta1"
]
},
{
"description": "Start tool with 2 FASTA FILES",
"input_files": ["fasta1", "fasta2"]
"input_files": [
"fasta1",
"fasta2"
]
}
],
"input_files_combinations_internal": [
[
{ "dna_sequence": 1 },
{ "dna_sequence": 1 }
{
"dna_sequence": 1
},
{
"dna_sequence": 1
}
],
[
{ "dna_sequence": 1 }
{
"dna_sequence": 1
}
]
],
"arguments": [
Expand All @@ -83,8 +109,14 @@
"help": " Tooltip for this argument",
"type": "enum_multiple",
"enum_items": {
"name": ["option 1 name", "option 2 name"],
"description": ["option1", "option2"]
"name": [
"option 1 name",
"option 2 name"
],
"description": [
"option1",
"option2"
]
},
"required": true,
"default": ""
Expand Down Expand Up @@ -124,10 +156,92 @@
}
}
],
"sites": [
{
"site_id": "local",
"status": 1
}
]
"sites": [
{
"site_id": "local",
"status": 1
}
]
},
{
"_id": "wget",
"title": "Remote file retrieving from URLs",
"name": "Get remote file",
"short_description": "Implementation of a the remote file retriever WGET.",
"url": "https://www.gnu.org/software/wget/",
"owner": {
"institution": "Barcelona Supercomputing Center",
"author": "Laia Codo",
"contact": "laia.codo@bsc.es",
"url": "",
"user": "laia.codo@bsc.es"
},
"external": false,
"keywords": [],
"infrastructure": {
"memory": 2,
"cpus": 1,
"executable": "/shared_scripts/download_from_url.sh",
"clouds": {
"local": {
"launcher": "SGE",
"default_cloud": true,
"queue": "local.q"
}
}
},
"input_files": [],
"arguments": [
{
"name": "url",
"description": "URL of the file to be downloaded",
"help": "URL",
"type": "string",
"required": true,
"allow_multiple": true
},
{
"name": "output",
"description": "Name of the output file",
"help": "Name of the output file",
"type": "string",
"allow_multiple": false,
"default": ""
},
{
"name": "working_dir",
"description": "Temporal dir",
"help": "",
"type": "string",
"allow_multiple": false,
"default": ""
},
{
"name": "untar",
"description": "Untar output file if ",
"help": "",
"type": "string",
"allow_multiple": false,
"default": ""
}
],
"control_file": ".summary.json",
"output_files": [
{
"name": "file",
"required": true,
"allow_multiple": false,
"file_type": "",
"data_type": "",
"file": {
"meta_data": {
"validated": false,
"description": "Data imported from URL",
"tool": "wget",
"visible": true
}
}
}
]
}
]
52 changes: 22 additions & 30 deletions front_end/openVRE/public/phplib/classes/Tooljob.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -859,7 +859,28 @@ public function prepareExecution($tool, $metadata, $metadata_pub = [])
$launcher = $this->launcher;
$cloudName = $this->cloudName;

if ($tool['external'] !== false) {
if ($tool['external'] === false) {
switch ($launcher) {
case "SGE":
$cmd = $this->setBashCmd_withoutApp($tool, $metadata);
if (!$cmd) {
return 0;
}

$submissionFilename = $this->createSubmitFile_SGE($cmd);
if (!is_file($submissionFilename)) {
return 0;
}

break;

default:
$_SESSION['errorData']['Error'][] = "Internal Tool '$this->toolId' not properly registered. Launcher for '$this->toolId' is set to \"$launcher\". Case not implemented.";
return 0;
}

return 1;
} else {
$configFilename = $this->setConfiguration_file($tool);
if ($configFilename == "0") {
return 0;
Expand Down Expand Up @@ -891,7 +912,6 @@ public function prepareExecution($tool, $metadata, $metadata_pub = [])

case "docker_SGE":
$cmd = $this->setBashCommandDockerSge($tool);
#$_SESSION['errorData']['Error'][] = "CMD3 " . $cmd;
if (!$cmd) {
return 0;
}
Expand Down Expand Up @@ -930,8 +950,6 @@ public function prepareExecution($tool, $metadata, $metadata_pub = [])
break;

case "Slurm":
//$_SESSION['errorData']['Internal Error'][]="Cannot set tool command line. Case still not implemented.";

$username = $_POST['username'];
$cmd = $this->setHPCRequest($cloudName, $tool, $username);
if (!$cmd) {
Expand All @@ -946,32 +964,6 @@ public function prepareExecution($tool, $metadata, $metadata_pub = [])

return 1;
}

if ($tool['external'] === false) {
switch ($launcher) {
case "SGE":
$cmd = $this->setBashCmd_withoutApp($tool, $metadata);
if (!$cmd) {
return 0;
}

$submissionFilename = $this->createSubmitFile_SGE($cmd);
if (!is_file($submissionFilename)) {
return 0;
}

break;

case "PMES":
//TODO

default:
$_SESSION['errorData']['Error'][] = "Internal Tool '$this->toolId' not properly registered. Launcher for '$this->toolId' is set to \"$launcher\". Case not implemented.";
return 0;
}

return 1;
}
}

protected function setBashCmd_SGE($tool)
Expand Down
Loading