diff --git a/docker-compose.yml b/docker-compose.yml index b4df0240..f5eb8d32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -163,8 +163,8 @@ services: - /var/run/docker.sock:/var/run/docker.sock:rprivate - ./sge/configuration:/tmp/configuration:rprivate - ./sge/configuration:/etc/gridengine/configuration:rprivate + - ./sge/shared_scripts:/shared_scripts:rshared - ./volumes/shared_data:/shared_data:rshared - - ./volumes/shared_scripts:/shared_scripts:rshared devices: - '/dev/fuse:/dev/fuse' cap_add: diff --git a/front_end/openVRE/install/database/tools.json b/front_end/openVRE/install/database/tools.json index 9a9e3f8f..f0ec0513 100644 --- a/front_end/openVRE/install/database/tools.json +++ b/front_end/openVRE/install/database/tools.json @@ -1,3 +1,4 @@ +[ { "_id": "tool_skeleton", "name": "Tool Skeleton", @@ -13,30 +14,40 @@ }, "status": 1, "external": true, - "keywords": ["next gen seq", "dna"], - "keywords_tool": ["Tool skeleton", "Sample keyword"], + "keywords": [ + "next gen seq", + "dna" + ], + "keywords_tool": [ + "Tool skeleton", + "Sample keyword" + ], "infrastructure": { - "memory": 12, - "cpus": 1, - "executable": "/home/vre_template_tool/VRE_RUNNER", - "container_image": "dockerhub/tool-skeleton:latest", - "clouds": { - "local": { - "launcher": "docker_SGE", - "default_cloud": true, - "queue": "testq", - "executable": "/home/vre_template_tool/VRE_RUNNER", - "executable_type": "docker" + "memory": 12, + "cpus": 1, + "executable": "/home/vre_template_tool/VRE_RUNNER", + "container_image": "dockerhub/tool-skeleton:latest", + "clouds": { + "local": { + "launcher": "docker_SGE", + "default_cloud": true, + "queue": "testq", + "executable": "/home/vre_template_tool/VRE_RUNNER", + "executable_type": "docker" + } } - } - }, + }, "input_files": [ { "name": "fasta1", "description": "FASTA input file 1", "help": "This is the input file 1 help tooltip", - "file_type": ["FASTA"], - "data_type": ["dna_sequence"], + "file_type": [ + "FASTA" + ], + "data_type": [ + "dna_sequence" + ], "required": true, "allow_multiple": false }, @@ -44,8 +55,12 @@ "name": "fasta2", "description": "FASTA input file 2", "help": "This is the input file 2 help tooltip", - "file_type": ["FASTA"], - "data_type": ["dna_sequence"], + "file_type": [ + "FASTA" + ], + "data_type": [ + "dna_sequence" + ], "required": false, "allow_multiple": false } @@ -53,20 +68,31 @@ "input_files_combinations": [ { "description": "Start tool with 1 FASTA FILE", - "input_files": ["fasta1"] + "input_files": [ + "fasta1" + ] }, { "description": "Start tool with 2 FASTA FILES", - "input_files": ["fasta1", "fasta2"] + "input_files": [ + "fasta1", + "fasta2" + ] } ], "input_files_combinations_internal": [ [ - { "dna_sequence": 1 }, - { "dna_sequence": 1 } + { + "dna_sequence": 1 + }, + { + "dna_sequence": 1 + } ], [ - { "dna_sequence": 1 } + { + "dna_sequence": 1 + } ] ], "arguments": [ @@ -83,8 +109,14 @@ "help": " Tooltip for this argument", "type": "enum_multiple", "enum_items": { - "name": ["option 1 name", "option 2 name"], - "description": ["option1", "option2"] + "name": [ + "option 1 name", + "option 2 name" + ], + "description": [ + "option1", + "option2" + ] }, "required": true, "default": "" @@ -124,10 +156,92 @@ } } ], - "sites": [ - { - "site_id": "local", - "status": 1 - } - ] + "sites": [ + { + "site_id": "local", + "status": 1 + } + ] + }, + { + "_id": "wget", + "title": "Remote file retrieving from URLs", + "name": "Get remote file", + "short_description": "Implementation of a the remote file retriever WGET.", + "url": "https://www.gnu.org/software/wget/", + "owner": { + "institution": "Barcelona Supercomputing Center", + "author": "Laia Codo", + "contact": "laia.codo@bsc.es", + "url": "", + "user": "laia.codo@bsc.es" + }, + "external": false, + "keywords": [], + "infrastructure": { + "memory": 2, + "cpus": 1, + "executable": "/shared_scripts/download_from_url.sh", + "clouds": { + "local": { + "launcher": "SGE", + "default_cloud": true, + "queue": "local.q" + } + } + }, + "input_files": [], + "arguments": [ + { + "name": "url", + "description": "URL of the file to be downloaded", + "help": "URL", + "type": "string", + "required": true, + "allow_multiple": true + }, + { + "name": "output", + "description": "Name of the output file", + "help": "Name of the output file", + "type": "string", + "allow_multiple": false, + "default": "" + }, + { + "name": "working_dir", + "description": "Temporal dir", + "help": "", + "type": "string", + "allow_multiple": false, + "default": "" + }, + { + "name": "untar", + "description": "Untar output file if ", + "help": "", + "type": "string", + "allow_multiple": false, + "default": "" + } + ], + "control_file": ".summary.json", + "output_files": [ + { + "name": "file", + "required": true, + "allow_multiple": false, + "file_type": "", + "data_type": "", + "file": { + "meta_data": { + "validated": false, + "description": "Data imported from URL", + "tool": "wget", + "visible": true + } + } + } + ] } +] \ No newline at end of file diff --git a/front_end/openVRE/public/phplib/classes/Tooljob.php b/front_end/openVRE/public/phplib/classes/Tooljob.php index d645fed3..21b7fd3f 100644 --- a/front_end/openVRE/public/phplib/classes/Tooljob.php +++ b/front_end/openVRE/public/phplib/classes/Tooljob.php @@ -859,7 +859,28 @@ public function prepareExecution($tool, $metadata, $metadata_pub = []) $launcher = $this->launcher; $cloudName = $this->cloudName; - if ($tool['external'] !== false) { + if ($tool['external'] === false) { + switch ($launcher) { + case "SGE": + $cmd = $this->setBashCmd_withoutApp($tool, $metadata); + if (!$cmd) { + return 0; + } + + $submissionFilename = $this->createSubmitFile_SGE($cmd); + if (!is_file($submissionFilename)) { + return 0; + } + + break; + + default: + $_SESSION['errorData']['Error'][] = "Internal Tool '$this->toolId' not properly registered. Launcher for '$this->toolId' is set to \"$launcher\". Case not implemented."; + return 0; + } + + return 1; + } else { $configFilename = $this->setConfiguration_file($tool); if ($configFilename == "0") { return 0; @@ -891,7 +912,6 @@ public function prepareExecution($tool, $metadata, $metadata_pub = []) case "docker_SGE": $cmd = $this->setBashCommandDockerSge($tool); - #$_SESSION['errorData']['Error'][] = "CMD3 " . $cmd; if (!$cmd) { return 0; } @@ -930,8 +950,6 @@ public function prepareExecution($tool, $metadata, $metadata_pub = []) break; case "Slurm": - //$_SESSION['errorData']['Internal Error'][]="Cannot set tool command line. Case still not implemented."; - $username = $_POST['username']; $cmd = $this->setHPCRequest($cloudName, $tool, $username); if (!$cmd) { @@ -946,32 +964,6 @@ public function prepareExecution($tool, $metadata, $metadata_pub = []) return 1; } - - if ($tool['external'] === false) { - switch ($launcher) { - case "SGE": - $cmd = $this->setBashCmd_withoutApp($tool, $metadata); - if (!$cmd) { - return 0; - } - - $submissionFilename = $this->createSubmitFile_SGE($cmd); - if (!is_file($submissionFilename)) { - return 0; - } - - break; - - case "PMES": - //TODO - - default: - $_SESSION['errorData']['Error'][] = "Internal Tool '$this->toolId' not properly registered. Launcher for '$this->toolId' is set to \"$launcher\". Case not implemented."; - return 0; - } - - return 1; - } } protected function setBashCmd_SGE($tool) diff --git a/front_end/openVRE/public/phplib/getdata.inc.php b/front_end/openVRE/public/phplib/getdata.inc.php index 06d2b2b2..424a2628 100644 --- a/front_end/openVRE/public/phplib/getdata.inc.php +++ b/front_end/openVRE/public/phplib/getdata.inc.php @@ -5,29 +5,30 @@ ///////////////////////////////// // upload file from local -function getData_fromLocal() { - // set destination working_directory/uploads - $dataDirPath = getAttr_fromGSFileId($_SESSION['User']['dataDir'], "path"); - $localWorkingDir = "$dataDirPath/uploads"; - $workingDir = $GLOBALS['dataDir']."/".$localWorkingDir; - $workingDirId = getGSFileId_fromPath($localWorkingDir); - - // check source file/s - if (empty($_FILES)) { - $_SESSION['errorData']['upload'][] = "ERROR: Receiving blank. Please select a file to upload"; - die("ERROR: Recieving blank. Please select a file to upload0"); - } - - // check target directory - if ($workingDirId == "0" || !is_dir($workingDir)) { - $_SESSION['errorData']['upload'][] = "Target server directory '".basename($localWorkingDir)."' does not exist. Please, login again."; - die("Target server directory '".basename($localWorkingDir)."' does not exist. Please, login again.0"); - } - - $fileIds = []; - // upload each source file +function getData_fromLocal() +{ + // set destination working_directory/uploads + $dataDirPath = getAttr_fromGSFileId($_SESSION['User']['dataDir'], "path"); + $localWorkingDir = "$dataDirPath/uploads"; + $workingDir = $GLOBALS['dataDir'] . "/" . $localWorkingDir; + $workingDirId = getGSFileId_fromPath($localWorkingDir); + + // check source file/s + if (empty($_FILES)) { + $_SESSION['errorData']['upload'][] = "ERROR: Receiving blank. Please select a file to upload"; + die("ERROR: Recieving blank. Please select a file to upload0"); + } + + // check target directory + if ($workingDirId == "0" || !is_dir($workingDir)) { + $_SESSION['errorData']['upload'][] = "Target server directory '" . basename($localWorkingDir) . "' does not exist. Please, login again."; + die("Target server directory '" . basename($localWorkingDir) . "' does not exist. Please, login again.0"); + } + + $fileIds = []; + // upload each source file $errorCode = $_FILES['file']['error']; - if ($errorCode) { + if ($errorCode) { $errMsg = [ 0 => "[UPLOAD_ERR_OK]: There is no error, the file uploaded with success", 1 => "[UPLOAD_ERR_INI_SIZE]: The uploaded file exceeds the upload_max_filesize directive in php.ini", @@ -40,8 +41,8 @@ function getData_fromLocal() { ]; if (isset($errMsg[$errorCode])) { - $_SESSION['errorData']['upload'][] = "ERROR [code $errorCode] ".$errMsg[$errorCode]; - die("ERROR [code $errorCode] ".$errMsg[$errorCode]."0"); + $_SESSION['errorData']['upload'][] = "ERROR [code $errorCode] " . $errMsg[$errorCode]; + die("ERROR [code $errorCode] " . $errMsg[$errorCode] . "0"); } $_SESSION['errorData']['upload'][] = "Unknown upload error"; @@ -50,13 +51,13 @@ function getData_fromLocal() { $size = $_FILES['file']['size']; if (!$size || $size == 0) { - $_SESSION['errorData']['upload'][] = "ERROR: ".$_FILES['file']['name']." file size is zero"; - die("ERROR: ".$_FILES['file']['name']." file size is zero 0"); + $_SESSION['errorData']['upload'][] = "ERROR: " . $_FILES['file']['name'] . " file size is zero"; + die("ERROR: " . $_FILES['file']['name'] . " file size is zero 0"); } if ($size > return_bytes(ini_get('upload_max_filesize')) || $size > return_bytes(ini_get('post_max_size'))) { - $_SESSION['errorData']['upload'][] = "ERROR: File size $size larger than UPLOAD_MAX_FILESIZE (".ini_get('upload_max_filesize').") "; - die("ERROR: File size $size larger than UPLOAD_MAX_FILESIZE (".ini_get('upload_max_filesize').") 0"); + $_SESSION['errorData']['upload'][] = "ERROR: File size $size larger than UPLOAD_MAX_FILESIZE (" . ini_get('upload_max_filesize') . ") "; + die("ERROR: File size $size larger than UPLOAD_MAX_FILESIZE (" . ini_get('upload_max_filesize') . ") 0"); } $usedDisk = (int) getUsedDiskSpace(); @@ -65,8 +66,8 @@ function getData_fromLocal() { $_SESSION['errorData']['upload'][] = "ERROR: Cannot upload file. Not enough space left in the workspace"; die("ERROR: Cannot upload file. Not enough space left in the workspace"); } - - $filePath = "$workingDir/".cleanName($_FILES['file']['name']); + + $filePath = "$workingDir/" . cleanName($_FILES['file']['name']); //do not overwrite, rename if (is_file($filePath)) { foreach (range(1, 99) as $N) { // TODO: should be changed @@ -77,7 +78,7 @@ function getData_fromLocal() { $name = $filePath; } - $tmpFilePath = $name .'_'. $N . $extension; + $tmpFilePath = $name . '_' . $N . $extension; if (!is_file($tmpFilePath)) { $filePath = $tmpFilePath; break; @@ -118,7 +119,7 @@ function getData_fromLocal() { array_push($fileIds, $fileId); - print implode(",", $fileIds); + print implode(",", $fileIds); } ///////////////////////////////// @@ -127,14 +128,16 @@ function getData_fromLocal() { // upload file from URL via CURL -function getData_fromUrl($url, $meta = null) { - [$toolArgs, $toolOuts, $output_dir] = prepare_getData_fromURL($url, "uploads", $GLOBALS['BASEURL']."/getdata/uploadForm.php#load_from_url", $meta); - getData_wget_asyncron($toolArgs, $toolOuts, $output_dir, $GLOBALS['BASEURL']."/getdata/uploadForm.php#load_from_url"); +function getData_fromUrl($url, $meta = null) +{ + [$toolArgs, $toolOuts, $output_dir] = prepare_getData_fromURL($url, "uploads", $GLOBALS['BASEURL'] . "/getdata/uploadForm.php#load_from_url", $meta); + getData_wget_asyncron($toolArgs, $toolOuts, $output_dir, $GLOBALS['BASEURL'] . "/getdata/uploadForm.php#load_from_url"); } // prepare target directory and file metadata -function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { +function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) +{ //parse out username and password from URL, if any $user = 0; $pass = 0; @@ -142,7 +145,7 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { if (preg_match('/(.*\/\/)(.*):(.*)@(.*)/', $url, $matches)) { $user = $matches[2]; $pass = $matches[3]; - $url_withCredentials = $matches[1].urlencode($user).":".urlencode($pass)."@".$matches[4]; + $url_withCredentials = $matches[1] . urlencode($user) . ":" . urlencode($pass) . "@" . $matches[4]; $url = $matches[1] . $matches[4]; } @@ -151,9 +154,9 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, TRUE); curl_setopt($ch, CURLOPT_HEADER, TRUE); - curl_setopt($ch, CURLOPT_VERBOSE,TRUE); + curl_setopt($ch, CURLOPT_VERBOSE, TRUE); curl_setopt($ch, CURLOPT_NOBODY, TRUE); - if($user && $pass){ + if ($user && $pass) { curl_setopt($ch, CURLOPT_USERPWD, "$user:$pass"); } @@ -161,10 +164,10 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { $status = curl_getinfo($ch, CURLINFO_HTTP_CODE); if ($status != 200 && !preg_match('/^3/', $status)) { $msg = "Resource URL '$url' is not valid or unaccessible. Status: $status"; - if($referer == "die") { + if ($referer == "die") { die($msg); } - + $_SESSION['errorData']['Error'][] = $msg; redirect($referer); } @@ -175,7 +178,7 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { if (!$filename) { $msg = "Resource URL ('$url') has not a valid HTTP header. Filename not found"; - if($referer == "die") { + if ($referer == "die") { die($msg); } @@ -188,7 +191,7 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { $diskLimit = (int) $_SESSION['User']['diskQuota']; if ($size == 0) { $msg = "Resource URL ('$url') is pointing to an empty resource (size = 0)"; - if($referer == "die") { + if ($referer == "die") { die($msg); } @@ -197,13 +200,13 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { } if ($size > ($diskLimit - $usedDisk)) { - $msg = "Cannot import file. There will be not enough space left in the workspace (size = ".getSize($size).")"; + $msg = "Cannot import file. There will be not enough space left in the workspace (size = " . getSize($size) . ")"; if ($referer == "die") { - $_SESSION['errorData']['Error'][] = $msg; - redirect($GLOBALS['BASEURL']."workspace/"); + $_SESSION['errorData']['Error'][] = $msg; + redirect($GLOBALS['BASEURL'] . "workspace/"); } - $_SESSION['errorData']['Error'][] = $msg; + $_SESSION['errorData']['Error'][] = $msg; redirect($referer); } @@ -211,7 +214,7 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { // setting output directory $dataDirPath = getAttr_fromGSFileId($_SESSION['User']['dataDir'], "path"); $localWorkingDir = "{$dataDirPath}/{$outdir}"; - $workingDir = $GLOBALS['dataDir']."/".$localWorkingDir; + $workingDir = $GLOBALS['dataDir'] . "/" . $localWorkingDir; $workingDirId = getGSFileId_fromPath($localWorkingDir); if ($workingDirId == "0") { @@ -219,25 +222,27 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { $workingDirId = createGSDirBNS($localWorkingDir, 1); $_SESSION['errorData']['Info'][] = "Creating '$outdir' directory: $localWorkingDir ($workingDirId)"; if ($workingDirId == "0") { - $msg = "Cannot create repository directory in $dataDirPath"; - if($referer == "die") { - die($msg); - } - - $_SESSION['errorData']['Error'][] = $msg; - redirect($referer); + $msg = "Cannot create repository directory in $dataDirPath"; + if ($referer == "die") { + die($msg); + } + + $_SESSION['errorData']['Error'][] = $msg; + redirect($referer); } - $IsMetadataAdded = addMetadataBNS($workingDirId, ["expiration" => -1, - "description"=> "Remote personal data"]); + $IsMetadataAdded = addMetadataBNS($workingDirId, [ + "expiration" => -1, + "description" => "Remote personal data" + ]); if ($IsMetadataAdded == "0") { - $msg = "Cannot set '$outdir' directory $localWorkingDir"; - if($referer == "die") { - die($msg); - } + $msg = "Cannot set '$outdir' directory $localWorkingDir"; + if ($referer == "die") { + die($msg); + } - $_SESSION['errorData']['Error'][] = $msg; - redirect($referer); + $_SESSION['errorData']['Error'][] = $msg; + redirect($referer); } if (!is_dir($workingDir)) { @@ -246,21 +251,21 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { } if (!is_dir($workingDir)) { - $msg = "Target server directory '$localWorkingDir' is not a directory. Your user account is corrupted. Please, report to helpdesk@multiscalegenomics.eu"; - if($referer == "die") { + $msg = "Target server directory '$localWorkingDir' is not a directory. Your user account is corrupted. Please, report to helpdesk@multiscalegenomics.eu"; + if ($referer == "die") { die($msg); } $_SESSION['errorData']['Error'][] = $msg; redirect($referer); } - + // Check file already registered $filePath = "$workingDir/$filename"; $filePathLocal = "$localWorkingDir/$filename"; $fileId = getGSFileId_fromPath($filePathLocal); if ($fileId) { - $_SESSION['errorData']['Error'][]="Resource file ('".$url."') is already available in the workspace: $filePath"; + $_SESSION['errorData']['Error'][] = "Resource file ('" . $url . "') is already available in the workspace: $filePath"; redirect("../getdata/editFile.php?fn[]=$fileId"); } //output_dir will be where fn is expected to be created @@ -268,15 +273,16 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { // working_dir will be set in user temporal dir. Checking it // TODO Or NO! maybe we decide to run directly on uploads/ - $dirTmp = $GLOBALS['dataDir']."/".$dataDirPath."/".$GLOBALS['tmpUser_dir']; + $dirTmp = $GLOBALS['dataDir'] . "/" . $dataDirPath . "/" . $GLOBALS['tmpUser_dir']; if (!is_dir($dirTmp) && !mkdir($dirTmp, 0775, true)) { $_SESSION['errorData']['error'][] = "Cannot create temporal file '$dirTmp'.Please, try it later."; } // setting tool arguments $toolArgs = [ - "url" => $toolArgs["url"] = $url_withCredentials ?: $url, - "output" => $filePath]; // Tool is responsible to create outputs in the output_dir + "url" => $toolArgs["url"] = $url_withCredentials ?: $url, + "output" => $filePath + ]; // Tool is responsible to create outputs in the output_dir // setting tool outputs -- metadata to save in DMP during tool output_file registration $descrip = "File imported from URL '$url'"; @@ -284,17 +290,18 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { [$fileExtension, $compressed, $fileBaseName] = getFileExtension($filePath); $filetypes = getFileTypeFromExtension($fileExtension); $filetype = array_keys($filetypes)[0] ?? ""; - $fileOut = ["name" => "file", - "file_path" => $filePath, - "data_type" => "", - "file_type" => $filetype, - "sources" => [0], - "taxon_id" => $taxon, - "meta_data" => [ - "validated" => false, - "compressed" => $compressed, - "description" => $descrip - ] + $fileOut = [ + "name" => "file", + "file_path" => $filePath, + "data_type" => "", + "file_type" => $filetype, + "sources" => [0], + "taxon_id" => $taxon, + "meta_data" => [ + "validated" => false, + "compressed" => $compressed, + "description" => $descrip + ] ]; $toolOuts = ["output_files" => [$fileOut]]; @@ -303,18 +310,19 @@ function prepare_getData_fromURL($url, $outdir, $referer, $meta = []) { //function getData_wget($url,$outdir,$referer,$meta=array()) { -function getData_wget_asyncron($toolArgs, $toolOuts, $output_dir, $referer) { - $toolId = "wget"; - $toolInputs = []; - $filePath = $toolOuts['output_files'][0]["file_path"]; - $logName = basename($filePath). ".log"; - - //TODO: FIXME START - This is a temporal fix. In future, files should not be downloaded, only registered - $pid = launchToolInternal($toolId, $toolInputs, $toolArgs, $toolOuts, $output_dir, $logName); - $outdir = basename($output_dir); - - if ($pid == 0) { - $msg ="File imported from URL '".basename($filePath)."' cannot be imported. Error occurred while preparing the job 'Get remote file'"; +function getData_wget_asyncron($toolArgs, $toolOuts, $output_dir, $referer) +{ + $toolId = "wget"; + $toolInputs = []; + $filePath = $toolOuts['output_files'][0]["file_path"]; + $logName = basename($filePath) . ".log"; + + //TODO: FIXME START - This is a temporal fix. In future, files should not be downloaded, only registered + $pid = launchToolInternal($toolId, $toolInputs, $toolArgs, $toolOuts, $output_dir, $logName); + $outdir = basename($output_dir); + + if ($pid == 0) { + $msg = "File imported from URL '" . basename($filePath) . "' cannot be imported. Error occurred while preparing the job 'Get remote file'"; if ($referer == "die") { die($msg); } @@ -323,62 +331,63 @@ function getData_wget_asyncron($toolArgs, $toolOuts, $output_dir, $referer) { redirect($referer); } - $_SESSION['errorData']['Info'][] = "File from URL '".basename($filePath)."' is being imported into the '$outdir' folder below. Please, edit its metadata once the import has finished"; - redirect($GLOBALS['BASEURL']."/workspace/"); -} + $_SESSION['errorData']['Info'][] = "File from URL '" . basename($filePath) . "' is being imported into the '$outdir' folder below. Please, edit its metadata once the import has finished"; + redirect($GLOBALS['BASEURL'] . "workspace/"); +} ///////////////////////////////// /////// BUILD FILE TEXT ///////////////////////////////// -function getData_fromTXT() { - $filename = $_REQUEST['filename']; - $data = $_REQUEST['txtdata']; - - // getting working directory - $dataDirPath = getAttr_fromGSFileId($_SESSION['User']['dataDir'], "path"); - $localWorkingDir = $dataDirPath."/uploads"; - - $workingDir = $GLOBALS['dataDir']."/".$localWorkingDir; - $workingDirId = getGSFileId_fromPath($localWorkingDir); - - // check target directory - if ($workingDirId == "0" || !is_dir($workingDir)){ - die("ERROR: Target server directory '".basename($localWorkingDir)."' does not exist. Please, login again."); - } - - $filePath = "$workingDir/".cleanName($filename); - $size = strlen($data); - - if ($size == 0) { - die("ERROR: ".$filename." file size is zero"); - } - - $usedDisk = (int) getUsedDiskSpace(); - $diskLimit = (int) $_SESSION['User']['diskQuota']; - if ($size > ($diskLimit - $usedDisk)) { - die("ERROR: Cannot upload file. Not enough space left in the workspace"); - } - - if (is_file($filePath)) { - foreach (range(1, 99) as $suffixNumber) { // TODO: should be changed to a better solution - if ($pos = strrpos($filePath, '.')) { - $name = substr($filePath, 0, $pos); - $ext = substr($filePath, $pos); - } else { - $name = $filePath; - } - - $tmpFilename = $name .'_'. $suffixNumber . $ext; - if (!is_file($tmpFilename)) { - $filePath = $tmpFilename; - break; - } - } - } - - $file = fopen($filePath, "w+"); - fputs($file, $data); - fclose($file); +function getData_fromTXT() +{ + $filename = $_REQUEST['filename']; + $data = $_REQUEST['txtdata']; + + // getting working directory + $dataDirPath = getAttr_fromGSFileId($_SESSION['User']['dataDir'], "path"); + $localWorkingDir = $dataDirPath . "/uploads"; + + $workingDir = $GLOBALS['dataDir'] . "/" . $localWorkingDir; + $workingDirId = getGSFileId_fromPath($localWorkingDir); + + // check target directory + if ($workingDirId == "0" || !is_dir($workingDir)) { + die("ERROR: Target server directory '" . basename($localWorkingDir) . "' does not exist. Please, login again."); + } + + $filePath = "$workingDir/" . cleanName($filename); + $size = strlen($data); + + if ($size == 0) { + die("ERROR: " . $filename . " file size is zero"); + } + + $usedDisk = (int) getUsedDiskSpace(); + $diskLimit = (int) $_SESSION['User']['diskQuota']; + if ($size > ($diskLimit - $usedDisk)) { + die("ERROR: Cannot upload file. Not enough space left in the workspace"); + } + + if (is_file($filePath)) { + foreach (range(1, 99) as $suffixNumber) { // TODO: should be changed to a better solution + if ($pos = strrpos($filePath, '.')) { + $name = substr($filePath, 0, $pos); + $ext = substr($filePath, $pos); + } else { + $name = $filePath; + } + + $tmpFilename = $name . '_' . $suffixNumber . $ext; + if (!is_file($tmpFilename)) { + $filePath = $tmpFilename; + break; + } + } + } + + $file = fopen($filePath, "w+"); + fputs($file, $data); + fclose($file); if (!is_file($filePath)) { die("ERROR: Uploaded file not correctly stored."); @@ -412,15 +421,16 @@ function getData_fromTXT() { // ONGOING: import data from Resository to Public dir. // Step 0: if TAR is given, files are unbundled and a directory is registered -function getData_fromRepository_ToPublic($params=array()) { //url, untar, data_type, file_type, other metadata +function getData_fromRepository_ToPublic($params = array()) +{ //url, untar, data_type, file_type, other metadata // Get params $url = $params['url']; - $extract_uncompress = (isset($params['extract_uncompress'])? $params['extract_uncompress']:false); // true, false + $extract_uncompress = (isset($params['extract_uncompress']) ? $params['extract_uncompress'] : false); // true, false - $datatype = (isset($params['data_type'])&& $params['data_type']? $params['data_type']:""); - $filetype = (isset($params['file_type'])&& $params['file_type']? $params['file_type']:""); - $descrip = (isset($params['description'])&& $params['description']? $params['description']:""); + $datatype = (isset($params['data_type']) && $params['data_type'] ? $params['data_type'] : ""); + $filetype = (isset($params['file_type']) && $params['file_type'] ? $params['file_type'] : ""); + $descrip = (isset($params['description']) && $params['description'] ? $params['description'] : ""); // Get URL headers @@ -435,23 +445,23 @@ function getData_fromRepository_ToPublic($params=array()) { //url, untar, data_t $status = curl_getinfo($ch, CURLINFO_HTTP_CODE); $url_effective = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL); - if ($status != 200 && !preg_match('/^3/',$status) ){ + if ($status != 200 && !preg_match('/^3/', $status)) { $_SESSION['errorData']['Error'][] = "Resource URL ('$url') is not valid or unaccessible. Status: $status"; redirect($_SERVER['HTTP_REFERER']); } // Get filename from header - if (! $filename){ - if (preg_match('/^Content-Disposition: .*?filename=(?[^\s]+|\x22[^\x22]+\x22)\x3B?.*$/m', $curl_data,$m)){ - $filename = trim($m['f'],' ";'); - }elseif (preg_match('/^Content-Length:\s*(\d+)/m', $curl_data,$m)){ - $hasLength = $m[1]; - if ($hasLength){ - $filename = basename($url); - } - } - } - if (!$filename){ + if (! $filename) { + if (preg_match('/^Content-Disposition: .*?filename=(?[^\s]+|\x22[^\x22]+\x22)\x3B?.*$/m', $curl_data, $m)) { + $filename = trim($m['f'], ' ";'); + } elseif (preg_match('/^Content-Length:\s*(\d+)/m', $curl_data, $m)) { + $hasLength = $m[1]; + if ($hasLength) { + $filename = basename($url); + } + } + } + if (!$filename) { $_SESSION['errorData']['Error'][] = "Resource URL ('$url') is not pointing to a valid filename"; redirect($_SERVER['HTTP_REFERER']); } @@ -461,12 +471,11 @@ function getData_fromRepository_ToPublic($params=array()) { //url, untar, data_t $size = curl_getinfo($ch, CURLINFO_CONTENT_LENGTH_DOWNLOAD); $usedDisk = (int)getUsedDiskSpace(); $diskLimit = (int)$_SESSION['User']['diskQuota']; - if ($size == 0 ) { + if ($size == 0) { $_SESSION['errorData']['Error'][] = "Resource URL ('$url') is pointing to an empty resource (size = 0)"; redirect($_SERVER['HTTP_REFERER']); - } - if ($size > ($diskLimit-$usedDisk) ) { + if ($size > ($diskLimit - $usedDisk)) { $_SESSION['errorData']['Error'][] = "Cannot import file. There will be not enough space left in the workspace (size = $size)"; redirect($_SERVER['HTTP_REFERER']); } @@ -474,142 +483,147 @@ function getData_fromRepository_ToPublic($params=array()) { //url, untar, data_t // Check repository from workspace - $dataDirPath = getAttr_fromGSFileId($_SESSION['User']['dataDir'],"path"); - $wd = $dataDirPath."/repository"; - $wdP = $GLOBALS['dataDir']."/".$wd; + $dataDirPath = getAttr_fromGSFileId($_SESSION['User']['dataDir'], "path"); + $wd = $dataDirPath . "/repository"; + $wdP = $GLOBALS['dataDir'] . "/" . $wd; $wdId = getGSFileId_fromPath($wd); - if ($wdId == "0" || !is_dir($wdP)){ - $_SESSION['errorData']['Error'][]="Target server directory '$wd' is not a directory. Your user account is corrupted. Please, report to helpdesk@multiscalegenomics.eu"; + if ($wdId == "0" || !is_dir($wdP)) { + $_SESSION['errorData']['Error'][] = "Target server directory '$wd' is not a directory. Your user account is corrupted. Please, report to helpdesk@multiscalegenomics.eu"; redirect($_SERVER['HTTP_REFERER']); } // Set output file/folder - list($fileExtension,$compressExtension,$fileBaseName) = getFileExtension($filename); - $compression = ($compressExtension && isset($GLOBALS['compressions'][$compressExtension])? $GLOBALS['compressions'][$compressExtension] : 0); - $output = ($compression && $extract_uncompress && preg_match('/TAR/',$compression)? hash('md5', $url, false):$filename); + list($fileExtension, $compressExtension, $fileBaseName) = getFileExtension($filename); + $compression = ($compressExtension && isset($GLOBALS['compressions'][$compressExtension]) ? $GLOBALS['compressions'][$compressExtension] : 0); + $output = ($compression && $extract_uncompress && preg_match('/TAR/', $compression) ? hash('md5', $url, false) : $filename); + + print "($fileExtension,$compressExtension,$fileBaseName) = getFileExtension($filename)\n
"; + print "output (file or folder) = $output\n"; -print "($fileExtension,$compressExtension,$fileBaseName) = getFileExtension($filename)\n
"; -print "output (file or folder) = $output\n"; - // Check output file/folder already registered $fnP = "$wdP/$output"; $fn = "$wd/$output"; $fnId = getGSFileId_fromPath($fn); - if ($fnId){ + if ($fnId) { // file already here - $_SESSION['errorData']['Error'][]="Resource file ('$url') is already available in the workspace: $fnP"; + $_SESSION['errorData']['Error'][] = "Resource file ('$url') is already available in the workspace: $fnP"; redirect("../getdata/editFile.php?fn[]=$fnId"); - //Do asyncronous download file (internal tool wget) + //Do asyncronous download file (internal tool wget) + + } else { - }else{ - //FIXME START - This is a temporal fix. In future, files should not be downloaded, only registered //output_dir will be where fn is expeted to be created: repository $output_dir = $wdP; - + // working_dir will be set in user temporal dir. Checking it - $dirTmp = $GLOBALS['dataDir']."/".$dataDirPath."/".$GLOBALS['tmpUser_dir']; - if (! is_dir($dirTmp)){ - if(!mkdir($dirTmp, 0775, true)) { - $_SESSION['errorData']['error'][]="Cannot create temporal file $dirTmp . Please, try it later."; - $resp['state']=0; - #break; - } + $dirTmp = $GLOBALS['dataDir'] . "/" . $dataDirPath . "/" . $GLOBALS['tmpUser_dir']; + if (! is_dir($dirTmp)) { + if (!mkdir($dirTmp, 0775, true)) { + $_SESSION['errorData']['error'][] = "Cannot create temporal file $dirTmp . Please, try it later."; + $resp['state'] = 0; + #break; + } } // choosing interanl tool - $toolId = "wget"; + $toolId = "wget"; // setting tool inputs - $toolInputs= array(); + $toolInputs = array(); // setting tool arguments. Tool is responsible to create outputs in the output_dir $toolArgs = array( - "url" => $url, - "output" => $fnP - ); - if ($compression and $extract_uncompress){ - $compressors = explode(",",$compression); - foreach($compressors as $c){ - if ($c == "TAR"){ - $toolArgs['archiver'] = $c; - continue; - }else{ - $toolArgs['compressor'] = $c; - continue; - } - } -// $toolArgs['uncompress_cmd'] = " tar -xz -C "; -// $toolArgs['uncompress_cmd'] = " tar -xj "; -// $toolArgs['uncompress_cmd'] = " gunzip -c "; -// $toolArgs['uncompress_cmd'] = " bzip2 "; - } + "url" => $url, + "output" => $fnP + ); + if ($compression and $extract_uncompress) { + $compressors = explode(",", $compression); + foreach ($compressors as $c) { + if ($c == "TAR") { + $toolArgs['archiver'] = $c; + continue; + } else { + $toolArgs['compressor'] = $c; + continue; + } + } + // $toolArgs['uncompress_cmd'] = " tar -xz -C "; + // $toolArgs['uncompress_cmd'] = " tar -xj "; + // $toolArgs['uncompress_cmd'] = " gunzip -c "; + // $toolArgs['uncompress_cmd'] = " bzip2 "; + } // setting tool output metadata. It will be registerd after tool execution - if (!$descrip){ - $descrip="Remote file extracted from $url"; - } - if (!$filetype){ - $filetypes = getFileTypeFromExtension($fileExtension); - $filetype =(isset(array_keys($filetypes)[0])?array_keys($filetypes)[0]:""); - } - $validated = ($filetype != "" && $datatype != ""? true:false); // Can lead to problems - - $fileOut_type = ($compression && $extract_uncompress && preg_match('/TAR/',$compression)? "dir":"file"); - - $fileOut=array( - "name" => "file", - "type" => $fileOut_type, - "file_path" => $fnP, - "data_type" => $datatype, - "file_type" => $filetype, - "sources" => [0], - "source_url" => $url, - "meta_data" => array( - "validated" => $validated, - "compressed" => $compressed, - "description" => $descrip, - ) - ); - if (isset($params['oeb_dataset_id'])){ $fileOut['meta_data']["oeb_dataset_id"] = $params['oeb_dataset_id'];} - if (isset($params['oeb_community_ids'])){ $fileOut['meta_data']["oeb_community_ids"] = $params['oeb_community_ids'];} - $toolOuts = Array ("output_files" => Array($fileOut)); - + if (!$descrip) { + $descrip = "Remote file extracted from $url"; + } + if (!$filetype) { + $filetypes = getFileTypeFromExtension($fileExtension); + $filetype = (isset(array_keys($filetypes)[0]) ? array_keys($filetypes)[0] : ""); + } + $validated = ($filetype != "" && $datatype != "" ? true : false); // Can lead to problems + + $fileOut_type = ($compression && $extract_uncompress && preg_match('/TAR/', $compression) ? "dir" : "file"); + + $fileOut = array( + "name" => "file", + "type" => $fileOut_type, + "file_path" => $fnP, + "data_type" => $datatype, + "file_type" => $filetype, + "sources" => [0], + "source_url" => $url, + "meta_data" => array( + "validated" => $validated, + "compressed" => $compressed, + "description" => $descrip, + ) + ); + if (isset($params['oeb_dataset_id'])) { + $fileOut['meta_data']["oeb_dataset_id"] = $params['oeb_dataset_id']; + } + if (isset($params['oeb_community_ids'])) { + $fileOut['meta_data']["oeb_community_ids"] = $params['oeb_community_ids']; + } + $toolOuts = array("output_files" => array($fileOut)); + // setting logName - $logName = basename($fnP). ".log"; - + $logName = basename($fnP) . ".log"; + //calling internal tool - $pid = launchToolInternal($toolId,$toolInputs,$toolArgs,$toolOuts,$output_dir,$logName); + $pid = launchToolInternal($toolId, $toolInputs, $toolArgs, $toolOuts, $output_dir, $logName); - if ($pid == 0){ - $_SESSION['errorData']['Error'][]="Resource file '".basename($fnP)."' cannot be imported. Error occurred while preparing the job 'Get remote file'"; + if ($pid == 0) { + $_SESSION['errorData']['Error'][] = "Resource file '" . basename($fnP) . "' cannot be imported. Error occurred while preparing the job 'Get remote file'"; redirect($_SERVER['HTTP_REFERER']); - }else{ - $_SESSION['errorData']['Info'][] ="Remote file '".basename($fnP)."' imported into the 'repository' folder below. Please, edit its metadata once the job has finished"; - redirect($GLOBALS['BASEURL']."workspace/"); + } else { + $_SESSION['errorData']['Info'][] = "Remote file '" . basename($fnP) . "' imported into the 'repository' folder below. Please, edit its metadata once the job has finished"; + redirect($GLOBALS['BASEURL'] . "workspace/"); } - } + } } -function process_URL($url){ +function process_URL($url) +{ $response = [ - "status" => false, - "size" => 0, - "filename" => false, - "effective_url" => false + "status" => false, + "size" => 0, + "filename" => false, + "effective_url" => false ]; $headers_data = get_headers($url, 1); if ($headers_data === false) { $_SESSION['errorData']['Error'][] = "Resource URL ('$url') is not valid or unaccessible. Server not found"; - return false; + return false; } // corrects url when 301/302 redirect(s) lead(s) to 200 @@ -630,7 +644,7 @@ function process_URL($url){ $response['size'] = isset($headers_data['Content-Disposition']) && preg_match("/filename=.+/", $headers_data['Content-Disposition']) && $headers_data['Content-Length'] ? $headers_data['Content-Length'] : $response['size']; - + $status = substr($headers_data[0], 9, 3); if (!preg_match('/(200)/', $headers_data[0]) && !preg_match('/^3/', $status)) { $_SESSION['errorData']['Error'][] = "Resource URL ('$url') is not valid or unaccessible. Status: $status"; @@ -641,7 +655,8 @@ function process_URL($url){ } // import from Repository (URL) to user workspace -function getData_fromRepository($url, $datatype, $filetype, $description, $oeb_dataset_id, $oeb_community_ids) { +function getData_fromRepository($url, $datatype, $filetype, $description, $oeb_dataset_id, $oeb_community_ids) +{ $url_data = process_URL($url); $status = $url_data['status']; if ($status != 200 && !preg_match('/^3/', $status)) { @@ -671,7 +686,7 @@ function getData_fromRepository($url, $datatype, $filetype, $description, $oeb_d // setting repository directory $dataDirPath = getAttr_fromGSFileId($_SESSION['User']['dataDir'], "path"); $localWorkingDir = "$dataDirPath/repository"; - $workingDir = $GLOBALS['dataDir']."/".$localWorkingDir; + $workingDir = $GLOBALS['dataDir'] . "/" . $localWorkingDir; $workingDirId = getGSFileId_fromPath($localWorkingDir); if ($workingDirId == "0") { @@ -680,15 +695,17 @@ function getData_fromRepository($url, $datatype, $filetype, $description, $oeb_d $_SESSION['errorData']['Info'][] = "Creating repository directory: $localWorkingDir ($workingDirId)"; if ($workingDirId == "0") { - $_SESSION['errorData']['Internal error'][] = "Cannot create repository directory in $dataDirPath"; - redirect($_SERVER['HTTP_REFERER']); + $_SESSION['errorData']['Internal error'][] = "Cannot create repository directory in $dataDirPath"; + redirect($_SERVER['HTTP_REFERER']); } - $addedMetadata = addMetadataBNS($workingDirId, ["expiration" => -1, - "description"=> "Remote personal data"]); + $addedMetadata = addMetadataBNS($workingDirId, [ + "expiration" => -1, + "description" => "Remote personal data" + ]); if ($addedMetadata == "0") { - $_SESSION['errorData']['Internal error'][] = "Cannot set 'repository' directory $localWorkingDir"; - redirect($_SERVER['HTTP_REFERER']); + $_SESSION['errorData']['Internal error'][] = "Cannot set 'repository' directory $localWorkingDir"; + redirect($_SERVER['HTTP_REFERER']); } if (!is_dir($workingDir)) { @@ -697,10 +714,10 @@ function getData_fromRepository($url, $datatype, $filetype, $description, $oeb_d } if (!is_dir($workingDir)) { - $_SESSION['errorData']['Error'][] = "Target server directory '$localWorkingDir' is not a directory. Your user account is corrupted. Please, report to ..."; + $_SESSION['errorData']['Error'][] = "Target server directory '$localWorkingDir' is not a directory. Your user account is corrupted. Please, report to ..."; redirect($_SERVER['HTTP_REFERER']); } - + // Check file already registered $filePath = "$workingDir/$filename"; $localFilePath = "$localWorkingDir/$filename"; @@ -711,7 +728,7 @@ function getData_fromRepository($url, $datatype, $filetype, $description, $oeb_d } //asyncronous download file (internal tool wget) - + //FIXME START - This is a temporal fix. In future, files should not be downloaded, only registered //output_dir will be where fn is expeted to be created: repository @@ -719,7 +736,7 @@ function getData_fromRepository($url, $datatype, $filetype, $description, $oeb_d // working_dir will be set in user temporal dir. Checking it // TODO Or NO! maybe we decide to run directly on uploads/ - $tmpDir = $GLOBALS['dataDir']."/".$dataDirPath."/".$GLOBALS['tmpUser_dir']; + $tmpDir = $GLOBALS['dataDir'] . "/" . $dataDirPath . "/" . $GLOBALS['tmpUser_dir']; if (!is_dir($tmpDir)) { if (!mkdir($tmpDir, 0775, true)) { $_SESSION['errorData']['error'][] = "Cannot create temporal file $tmpDir . Please, try it later."; @@ -727,18 +744,18 @@ function getData_fromRepository($url, $datatype, $filetype, $description, $oeb_d } } - $toolId = "wget"; - $toolInputs= []; + $toolId = "wget"; + $toolInputs = []; $toolArgs = [ - "url" => $url, - "output" => $filePath - ]; + "url" => $url, + "output" => $filePath + ]; // setting tool outputs. Metadata will be saved in DB during tool output_file registration $description = $description ?: "Remote file extracted from $url"; if (!$filetype) { - [$fileExtension, $compressed, $fileBaseName] = getFileExtension($filePath); + [$fileExtension, $compressed, $fileBaseName] = getFileExtension($filePath); $filetypes = getFileTypeFromExtension($fileExtension); $filetype = array_keys($filetypes)[0] ?? ""; } @@ -759,25 +776,25 @@ function getData_fromRepository($url, $datatype, $filetype, $description, $oeb_d ] ]; - if (isset($oeb_dataset_id)) { + if (isset($oeb_dataset_id)) { $fileOut['meta_data']["oeb_dataset_id"] = $oeb_dataset_id; } - if (isset($oeb_community_ids)) { + if (isset($oeb_community_ids)) { $fileOut['meta_data']["oeb_community_ids"] = $oeb_community_ids; } $toolOuts = ["output_files" => [$fileOut]]; - $logName = basename($filePath). ".log"; + $logName = basename($filePath) . ".log"; $pid = launchToolInternal($toolId, $toolInputs, $toolArgs, $toolOuts, $output_dir, $logName); if ($pid == 0) { - $_SESSION['errorData']['Error'][] = "Resource file '".basename($filePath)."' cannot be imported. Error occurred while preparing the job 'Get remote file'"; + $_SESSION['errorData']['Error'][] = "Resource file '" . basename($filePath) . "' cannot be imported. Error occurred while preparing the job 'Get remote file'"; redirect($_SERVER['HTTP_REFERER']); } - $_SESSION['errorData']['Info'][] = "Remote file '".basename($filePath)."' imported into the 'repository' folder below. Please, edit its metadata once the job has finished"; - redirect($GLOBALS['BASEURL']."workspace/"); + $_SESSION['errorData']['Info'][] = "Remote file '" . basename($filePath) . "' imported into the 'repository' folder below. Please, edit its metadata once the job has finished"; + redirect($GLOBALS['BASEURL'] . "workspace/"); //FIXME END } @@ -790,91 +807,92 @@ function getData_fromRepository($url, $datatype, $filetype, $description, $oeb_d // list sampleData -function getSampleDataList($status=1,$filter_tool_status=true) { +function getSampleDataList($status = 1, $filter_tool_status = true) +{ $ft; - if ($filter_tool_status){ - $fa = indexArray($GLOBALS['toolsCol']->find(array('status' => 1),array('_id' => 1))); - $fu = indexArray($GLOBALS['visualizersCol']->find(array('status' => 1),array('_id' => 1))); - $tools_active = array_keys(array_merge($fa,$fu)); + if ($filter_tool_status) { + $fa = indexArray($GLOBALS['toolsCol']->find(array('status' => 1), array('_id' => 1))); + $fu = indexArray($GLOBALS['visualizersCol']->find(array('status' => 1), array('_id' => 1))); + $tools_active = array_keys(array_merge($fa, $fu)); // if common/anon user, list sampledata for active tools - if ($_SESSION['User']['Type'] == 3 || $_SESSION['User']['Type'] == 2){ + if ($_SESSION['User']['Type'] == 3 || $_SESSION['User']['Type'] == 2) { $ft = $GLOBALS['sampleDataCol']->find(array( - '$or' => array( - array("status" => $status, "tool"=> array('$not' => array('$exists' => 1)) ), - array("status" => $status, "tool"=> array('$in' => $tools_active)) - ) - ),array('_id' => 1)); - - // if admin user, list sampledata regardless tool status - }elseif ($_SESSION['User']['Type'] == 0){ - $ft = $GLOBALS['sampleDataCol']->find(array('status' => $status),array('_id' => 1)); - - // if tool dev user, list sampledata for active tools + its own tools - }elseif ($_SESSION['User']['Type'] == 1){ - $fr = $GLOBALS['toolsCol']->find(array('status' => 3,'_id'=> array('$in'=>$_SESSION['User']['ToolsDev'])),array('_id' => 1)); + '$or' => array( + array("status" => $status, "tool" => array('$not' => array('$exists' => 1))), + array("status" => $status, "tool" => array('$in' => $tools_active)) + ) + ), array('_id' => 1)); + + // if admin user, list sampledata regardless tool status + } elseif ($_SESSION['User']['Type'] == 0) { + $ft = $GLOBALS['sampleDataCol']->find(array('status' => $status), array('_id' => 1)); + + // if tool dev user, list sampledata for active tools + its own tools + } elseif ($_SESSION['User']['Type'] == 1) { + $fr = $GLOBALS['toolsCol']->find(array('status' => 3, '_id' => array('$in' => $_SESSION['User']['ToolsDev'])), array('_id' => 1)); $tools_owned = array_keys(iterator_to_array($fr)); $ft = $GLOBALS['sampleDataCol']->find(array( - '$or' => array( - array("status" => $status, "tool"=> array('$not' => array('$exists' => 1)) ), - array("status" => $status, "tool"=> array('$in' => array_merge($tools_active,$tools_owned))) - ) - ),array('_id' => 1)); - + '$or' => array( + array("status" => $status, "tool" => array('$not' => array('$exists' => 1))), + array("status" => $status, "tool" => array('$in' => array_merge($tools_active, $tools_owned))) + ) + ), array('_id' => 1)); } - - }else{ + } else { // list active sample data sets, regardless tool status - $ft = $GLOBALS['sampleDataCol']->find(array('status' => $status),array('_id' => 1)); + $ft = $GLOBALS['sampleDataCol']->find(array('status' => $status), array('_id' => 1)); } - return iterator_to_array($ft); - + return iterator_to_array($ft); } // get sampleData -function getSampleData($sampleData) { +function getSampleData($sampleData) +{ return $GLOBALS['sampleDataCol']->findOne(['_id' => $sampleData]); } // import sampleData into into current WS user -function getData_fromSampleData($params = []) { +function getData_fromSampleData($params = []) +{ if (!is_array($params['sampleData'])) { $params['sampleData'] = [$params['sampleData']]; } foreach ($params['sampleData'] as $sampleName) { $_SESSION['errorData']['Info'][] = "Importing exemple dataset for '$sampleName'"; - $dataDir = $_SESSION['User']['id'] ."/".$_SESSION['User']['activeProject']; + $dataDir = $_SESSION['User']['id'] . "/" . $_SESSION['User']['activeProject']; if (setUserWorkSpace_sampleData($sampleName, $dataDir) == "0") { $_SESSION['errorData']['Warning'][] = "Cannot fully inject exemple dataset into user workspace."; - redirect($GLOBALS['URL']."/getdata/sampleDataList.php"); + redirect($GLOBALS['URL'] . "/getdata/sampleDataList.php"); } $_SESSION['errorData']['Info'][] = "Example data successfuly imported."; - header("Location:".$GLOBALS['URL']."/workspace/"); + header("Location:" . $GLOBALS['URL'] . "/workspace/"); } } -function getData_fromEGA($datasetIds, $fileIds, $filenames, $fileSizes) { +function getData_fromEGA($datasetIds, $fileIds, $filenames, $fileSizes) +{ $datasetIdsArray = explode(',', $datasetIds); $fileIdsArray = explode(',', $fileIds); $filenamesArray = explode(',', $filenames); $fileSizesArray = explode(',', $fileSizes); $dataDirPath = getAttr_fromGSFileId($_SESSION['User']['dataDir'], "path"); - $localWorkingDir = $dataDirPath."/uploads"; + $localWorkingDir = $dataDirPath . "/uploads"; for ($i = 0; $i < count($fileIdsArray); $i++) { $filePath = "{$datasetIdsArray[$i]}/{$filenamesArray[$i]}"; $insertData = [ 'owner' => $_SESSION['User']['id'], 'size' => $fileSizesArray[$i], - 'mtime' => new MongoDB\BSON\UTCDateTime(strtotime("now")*1000) + 'mtime' => new MongoDB\BSON\UTCDateTime(strtotime("now") * 1000) ]; - + $metaData = [ 'data_type' => "variants", 'data_source' => "EGA", @@ -893,6 +911,6 @@ function getData_fromEGA($datasetIds, $fileIds, $filenames, $fileSizes) { echo "EGA file uploaded"; } - + return; } diff --git a/front_end/openVRE/public/phplib/projects.inc.php b/front_end/openVRE/public/phplib/projects.inc.php index c5d54eb0..8dd61343 100644 --- a/front_end/openVRE/public/phplib/projects.inc.php +++ b/front_end/openVRE/public/phplib/projects.inc.php @@ -1,198 +1,212 @@ us reporting this error"; + // set project default + if (!$projectDir) { + $_SESSION['errorData']['Internal'][] = "Cannot create user workspace $homeDir. No project directory name given. Please, contact us reporting this error"; + } + if (! count($projectData)) { + $projectData = array( + "name" => $GLOBALS['project_default'], + "description" => "This is my first VRE project, automatically created when entering into '" . $GLOBALS['NAME'] . "'. It is going to include my first data and executions.", + "keywords" => "default" + ); } - if (! count($projectData)){ - $projectData = array( - "name" => $GLOBALS['project_default'], - "description" => "This is my first VRE project, automatically created when entering into '".$GLOBALS['NAME']."'. It is going to include my first data and executions.", - "keywords" => "default" - ); - } - // create worskspace data - $dataDirId = setUserWorkSpace($homeDir,$projectDir,$sampleData,$projectData,$verbose,$asRoot); + // create worskspace data + $dataDirId = setUserWorkSpace($homeDir, $projectDir, $sampleData, $projectData, $verbose, $asRoot); - return $dataDirId; - } + return $dataDirId; +} -function setUserWorkSpace($homeDir,$projectDir,$sampleData="",$projectData,$verbose=FALSE,$asRoot=0) { +function setUserWorkSpace($homeDir, $projectDir, $sampleData = "", $projectData, $verbose = FALSE, $asRoot = 0) +{ - // set sampleData default - if ($sampleData == "") - $sampleData = $GLOBALS['sampleData_default']; + // set sampleData default + if ($sampleData == "") + $sampleData = $GLOBALS['sampleData_default']; + + if ($verbose) + $_SESSION['errorData']['Info'][] = "Preparing user workspace named '$homeDir' with sample data '$sampleData'"; - if ($verbose) - $_SESSION['errorData']['Info'][] = "Preparing user workspace named '$homeDir' with sample data '$sampleData'"; + //creating user home directory + if (!is_dir($GLOBALS['dataDir']) || !is_writable($GLOBALS['dataDir'])) { + header('Location: ' . $GLOBALS['BASEURL'] . '/htmlib/errordb.php?msg=Cannot access VRE data. Make sure data device is accessible and writable.'); + die(1); + } - //creating user home directory - if (!is_dir($GLOBALS['dataDir']) || !is_writable($GLOBALS['dataDir'])){ - header('Location: '.$GLOBALS['BASEURL'].'/htmlib/errordb.php?msg=Cannot access VRE data. Make sure data device is accessible and writable.'); - die(1); - } + $homeDirP = $GLOBALS['dataDir'] . "/$homeDir"; + $homeDirId = getGSFileId_fromPath($homeDir, $asRoot); - $homeDirP = $GLOBALS['dataDir']."/$homeDir"; - $homeDirId = getGSFileId_fromPath($homeDir,$asRoot); - - if (! isGSDirBNS($GLOBALS['filesCol'],$homeDirId) || ! is_dir($homeDirP)){ + if (! isGSDirBNS($GLOBALS['filesCol'], $homeDirId) || ! is_dir($homeDirP)) { - $homeDirId = createGSDirBNS($homeDir,1); + $homeDirId = createGSDirBNS($homeDir, 1); if ($verbose) - $_SESSION['errorData']['Info'][] = "Creating main user directory: $homeDirP ($homeDirId)"; + $_SESSION['errorData']['Info'][] = "Creating main user directory: $homeDirP ($homeDirId)"; - if ($homeDirId == "0" ){ + if ($homeDirId == "0") { $_SESSION['errorData']['Error'][] = "Cannot create main user directory $homeDir"; return 0; } - $r = addMetadataBNS($homeDirId,Array("expiration"=>-1, - "description"=> "Root user data")); - if ($r == "0" ){ + $r = addMetadataBNS($homeDirId, array( + "expiration" => -1, + "description" => "Root user data" + )); + if ($r == "0") { $_SESSION['errorData']['Error'][] = "Cannot set main user directory $homeDir"; return 0; } if (!is_dir($homeDirP)) - mkdir($homeDirP, 0775) or die("Cannot write main user directory $homeDirP"); - } + mkdir($homeDirP, 0775) or die("Cannot write main user directory $homeDirP"); + } $GLOBALS['filesCol']->updateOne( array('_id' => $homeDirId), - array('$set' => array( - 'lastAccess' => moment() + array( + '$set' => array( + 'lastAccess' => moment() ) ) - ); + ); - // creating user workspace for given project - - $dataDir = "$homeDir/$projectDir"; - $dataDirP = $GLOBALS['dataDir']."/$dataDir"; - $dataDirId = getGSFileId_fromPath($dataDir,$asRoot); + // creating user workspace for given project - if (! isGSDirBNS($GLOBALS['filesCol'],$dataDirId) || ! is_dir($dataDirP)){ + $dataDir = "$homeDir/$projectDir"; + $dataDirP = $GLOBALS['dataDir'] . "/$dataDir"; + $dataDirId = getGSFileId_fromPath($dataDir, $asRoot); + + if (! isGSDirBNS($GLOBALS['filesCol'], $dataDirId) || ! is_dir($dataDirP)) { //creating project directory - $dataDirId = createProjectDir($dataDir,$dataDirP,$projectData,$asRoot); - - if ($verbose) - $_SESSION['errorData']['Info'][] = "Creating main project directory: $dataDirP ($dataDirId)"; + $dataDirId = createProjectDir($dataDir, $dataDirP, $projectData, $asRoot); + + if ($verbose) + $_SESSION['errorData']['Info'][] = "Creating main project directory: $dataDirP ($dataDirId)"; - if ($dataDirId == "0" ){ - $_SESSION['errorData']['Error'][]="Error creating default project '".$GLOBALS['project_default']."'"; + if ($dataDirId == "0") { + $_SESSION['errorData']['Error'][] = "Error creating default project '" . $GLOBALS['project_default'] . "'"; return 0; } //creating uploads directory - if ($sampleData != "0"){ - $upDirId = createGSDirBNS($dataDir."/uploads",1); - if ($verbose) - $_SESSION['errorData']['Info'][] = "Creating uploads directory: $dataDir/uploads ($upDirId)"; - - if ($upDirId == "0" ){ - $_SESSION['errorData']['Error'][] = "Cannot create uploads directory in $dataDir ($dataDirId) "; - return 0; + if ($sampleData != "0") { + $upDirId = createGSDirBNS($dataDir . "/uploads", 1); + if ($verbose) + $_SESSION['errorData']['Info'][] = "Creating uploads directory: $dataDir/uploads ($upDirId)"; + + if ($upDirId == "0") { + $_SESSION['errorData']['Error'][] = "Cannot create uploads directory in $dataDir ($dataDirId) "; + return 0; } - $r = addMetadataBNS($upDirId,Array("expiration" => -1, - "description"=> "Uploaded personal data")); - if ($r == "0" ){ - $_SESSION['errorData']['Error'][] = "Cannot set uploads directory $dataDir/uploads"; - return 0; - } - if (!is_dir("$dataDirP/uploads")) - mkdir("$dataDirP/uploads", 0775); - } + $r = addMetadataBNS($upDirId, array( + "expiration" => -1, + "description" => "Uploaded personal data" + )); + if ($r == "0") { + $_SESSION['errorData']['Error'][] = "Cannot set uploads directory $dataDir/uploads"; + return 0; + } + if (!is_dir("$dataDirP/uploads")) + mkdir("$dataDirP/uploads", 0775); + } - //creating repository directory - - if ($sampleData != "0"){ - $repDirId = createGSDirBNS($dataDir."/repository",1); - if ($verbose) - $_SESSION['errorData']['Info'][] = "Creating repository directory: $dataDir/repository ($repDirId)"; - - if ($repDirId == "0" ){ - $_SESSION['errorData']['Error'][] = "Cannot create repository directory in $dataDir ($dataDirId) "; - return 0; - } - $r = addMetadataBNS($repDirId,Array("expiration" => -1, - "description"=> "Remote personal data")); - if ($r == "0"){ - $_SESSION['errorData']['Error'][] = "Cannot set uploads directory $dataDir/repository"; - return 0; - } - if (!is_dir("$dataDirP/repository")) - mkdir("$dataDirP/repository", 0775); - } - - //creating rstudio directory - if ($sampleData != "0"){ - $repDirId = createGSDirBNS($dataDir."/rstudio",1); - if ($verbose) - $_SESSION['errorData']['Info'][] = "Creating rstudio directory: $dataDir/rstudio ($repDirId)"; - if ($repDirId == "0" ){ - $_SESSION['errorData']['Error'][] = "Cannot create repository directory in $dataDir ($dataDirId) "; - return 0; + //creating repository directory + + if ($sampleData != "0") { + $repDirId = createGSDirBNS($dataDir . "/repository", 1); + if ($verbose) + $_SESSION['errorData']['Info'][] = "Creating repository directory: $dataDir/repository ($repDirId)"; + + if ($repDirId == "0") { + $_SESSION['errorData']['Error'][] = "Cannot create repository directory in $dataDir ($dataDirId) "; + return 0; + } + $r = addMetadataBNS($repDirId, array( + "expiration" => -1, + "description" => "Remote personal data" + )); + if ($r == "0") { + $_SESSION['errorData']['Error'][] = "Cannot set uploads directory $dataDir/repository"; + return 0; + } + if (!is_dir("$dataDirP/repository")) + mkdir("$dataDirP/repository", 0775); + } + + //creating rstudio directory + if ($sampleData != "0") { + $repDirId = createGSDirBNS($dataDir . "/rstudio", 1); + if ($verbose) + $_SESSION['errorData']['Info'][] = "Creating rstudio directory: $dataDir/rstudio ($repDirId)"; + if ($repDirId == "0") { + $_SESSION['errorData']['Error'][] = "Cannot create repository directory in $dataDir ($dataDirId) "; + return 0; } - $r = addMetadataBNS($repDirId,Array("expiration" => -1, - "description"=> "RStudio personal data")); - if ($r == "0"){ - $_SESSION['errorData']['Error'][] = "Cannot set rstudio directory $dataDir/rstudio"; - return 0; + $r = addMetadataBNS($repDirId, array( + "expiration" => -1, + "description" => "RStudio personal data" + )); + if ($r == "0") { + $_SESSION['errorData']['Error'][] = "Cannot set rstudio directory $dataDir/rstudio"; + return 0; } if (!is_dir("$dataDirP/rstudio")) - mkdir("$dataDirP/rstudio", 0775); - } + mkdir("$dataDirP/rstudio", 0775); + } - // creating other directories not registered in mongo + // creating other directories not registered in mongo - if (!is_dir("$dataDirP/.jbrowse") || !is_dir("$dataDirP/.tadkit") || !is_dir("$dataDirP/.dev") || !is_dir($dataDirP."/".$GLOBALS['tmpUser_dir']) ){ + if (!is_dir("$dataDirP/.jbrowse") || !is_dir("$dataDirP/.tadkit") || !is_dir("$dataDirP/.dev") || !is_dir($dataDirP . "/" . $GLOBALS['tmpUser_dir'])) { mkdir("$dataDirP/.jbrowse", 0775); mkdir("$dataDirP/.tadkit", 0775); mkdir("$dataDirP/.dev", 0775); - mkdir("$dataDirP/".$GLOBALS['tmpUser_dir'], 0775); + mkdir("$dataDirP/" . $GLOBALS['tmpUser_dir'], 0775); } // injecting sample data - - if ($sampleData != "0"){ - - $r = setUserWorkSpace_sampleData($sampleData,$dataDir,$verbose); - if ($r=="0"){ - $_SESSION['errorData']['Warning'][] = "Cannot fully inject sample data '$sampleData' into user workspace."; - }else{ - if ($verbose) - $_SESSION['errorData']['Info'][] = "Sample data '$sampleData' successfully injected into user workspace."; - } - } + + if ($sampleData != "0") { + + $r = setUserWorkSpace_sampleData($sampleData, $dataDir, $verbose); + if ($r == "0") { + $_SESSION['errorData']['Warning'][] = "Cannot fully inject sample data '$sampleData' into user workspace."; + } else { + if ($verbose) + $_SESSION['errorData']['Info'][] = "Sample data '$sampleData' successfully injected into user workspace."; + } + } } $GLOBALS['filesCol']->updateOne( array('_id' => $dataDirId), - array('$set' => array( - 'lastAccess' => moment() + array( + '$set' => array( + 'lastAccess' => moment() ) ) - ); + ); - return $dataDirId; + return $dataDirId; } -function setUserWorkSpace_sampleData($sampleName, $dataDir, $verbose = TRUE) { +function setUserWorkSpace_sampleData($sampleName, $dataDir, $verbose = TRUE) +{ // path for sample data set $sampleData = getSampleData($sampleName); if (!$sampleData) { @@ -201,97 +215,97 @@ function setUserWorkSpace_sampleData($sampleName, $dataDir, $verbose = TRUE) { } // validate sample Data integrity - $datafolders = scanDir($GLOBALS['sampleData']."/".$sampleData['sample_path']); + $datafolders = scanDir($GLOBALS['sampleData'] . "/" . $sampleData['sample_path']); if (!in_array("uploads", $datafolders)) { - $_SESSION['errorData']['Warning'][] = "Sample data '".$sampleData['name']."' has no 'uploads' folder"; + $_SESSION['errorData']['Warning'][] = "Sample data '" . $sampleData['name'] . "' has no 'uploads' folder"; return 0; } - $metadataPath = $GLOBALS['sampleData']."/".$sampleData['sample_path']."/.sample_metadata.json"; + $metadataPath = $GLOBALS['sampleData'] . "/" . $sampleData['sample_path'] . "/.sample_metadata.json"; if (!is_file($metadataPath)) { - $_SESSION['errorData']['Warning'][] = "Sample data '".$sampleData['name']."' has no metadata (.sample_metadata.json) to load -> $metadataPath "; + $_SESSION['errorData']['Warning'][] = "Sample data '" . $sampleData['name'] . "' has no metadata (.sample_metadata.json) to load -> $metadataPath "; return 0; } // read sample Data metadata $metadataArray = json_decode(file_get_contents($metadataPath), true); if (count($metadataArray) == 0) { - $_SESSION['errorData']['Warning'][] = "Sample data '".$sampleData['name']."' has malformated json in '$metadataPath'"; + $_SESSION['errorData']['Warning'][] = "Sample data '" . $sampleData['name'] . "' has malformated json in '$metadataPath'"; return 0; } foreach ($metadataArray as $metadata) { - if (!isset($metadata['file_path'])) { - $_SESSION['errorData']['Warning'][] = "Wrong sample data '".$sampleData['name']."' metadata contains elements without 'file_path' attribute. Ignoring them."; + if (!isset($metadata['file_path'])) { + $_SESSION['errorData']['Warning'][] = "Wrong sample data '" . $sampleData['name'] . "' metadata contains elements without 'file_path' attribute. Ignoring them."; continue; - } - - if (save_fromSampleDataMetadata($metadata, $dataDir, $sampleName, "folder", $verbose) == "0") { - $_SESSION['errorData']['Warning'][] = "Failed to inject sample data '".$metadata['file_path']."'"; } - // TODO: check if it is necessary + if (save_fromSampleDataMetadata($metadata, $dataDir, $sampleName, "folder", $verbose) == "0") { + $_SESSION['errorData']['Warning'][] = "Failed to inject sample data '" . $metadata['file_path'] . "'"; + } + + // TODO: check if it is necessary // looking for files in the folder - $sampleDataPath = $GLOBALS['sampleData']."/".$sampleData['sample_path']."/".$metadata['file_path']; - $metaFilePath = "$sampleDataPath/.sample_metadata.json"; + $sampleDataPath = $GLOBALS['sampleData'] . "/" . $sampleData['sample_path'] . "/" . $metadata['file_path']; + $metaFilePath = "$sampleDataPath/.sample_metadata.json"; - if (!is_file($metaFilePath)) { - $_SESSION['errorData']['Warning'][] = "Sample data '".$sampleData['name']."' has no metadata in $sampleDataPath to load. Empty directory."; + if (!is_file($metaFilePath)) { + $_SESSION['errorData']['Warning'][] = "Sample data '" . $sampleData['name'] . "' has no metadata in $sampleDataPath to load. Empty directory."; continue; - } + } - $metadataArray = json_decode(file_get_contents($metaFilePath), true); - if (count($metadataArray) == 0 ) { - $_SESSION['errorData']['Warning'][] = "Sample data '".$sampleData['name']."' has malformated json in folder '$sampleDataPath'"; + $metadataArray = json_decode(file_get_contents($metaFilePath), true); + if (count($metadataArray) == 0) { + $_SESSION['errorData']['Warning'][] = "Sample data '" . $sampleData['name'] . "' has malformated json in folder '$sampleDataPath'"; continue; - } + } - foreach ($metadataArray as $meta_file) { + foreach ($metadataArray as $meta_file) { if (!isset($meta_file['file_path'])) { - $_SESSION['errorData']['Warning'][] = "Sample data '".$sampleData['name']."' contains elements without 'file_path' attribute. Ignoring them."; + $_SESSION['errorData']['Warning'][] = "Sample data '" . $sampleData['name'] . "' contains elements without 'file_path' attribute. Ignoring them."; continue; } if (save_fromSampleDataMetadata($meta_file, $dataDir, $sampleName, "file", $verbose) == "0") { - $_SESSION['errorData']['Warning'][] = "Failed to inject sample data '".$meta_file['file_path']."'"; + $_SESSION['errorData']['Warning'][] = "Failed to inject sample data '" . $meta_file['file_path'] . "'"; } - } + } } return 1; -} +} -function save_fromSampleDataMetadata($metadata, $dataDir, $sampleName, $type, $verbose = TRUE) { - $sampleData = getSampleData($sampleName); - $sampleDataPath = $GLOBALS['sampleData']."/".$sampleData['sample_path']."/".$metadata['file_path']; - $dataDirPath = $GLOBALS['dataDir']."/$dataDir"; - $userDataPath = $dataDirPath."/".$metadata['file_path']; +function save_fromSampleDataMetadata($metadata, $dataDir, $sampleName, $type, $verbose = TRUE) +{ + $sampleData = getSampleData($sampleName); + $sampleDataPath = $GLOBALS['sampleData'] . "/" . $sampleData['sample_path'] . "/" . $metadata['file_path']; + $dataDirPath = $GLOBALS['dataDir'] . "/$dataDir"; + $userDataPath = $dataDirPath . "/" . $metadata['file_path']; - // Saving to disk + // Saving to disk if ($type == "file") { if (!is_file($sampleDataPath)) { if (is_dir($sampleDataPath)) { - $_SESSION['errorData']['Warning'][] = "Sample data file '".$metadata['file_path']."' is a subfolder. Not supported. Ignoring it."; + $_SESSION['errorData']['Warning'][] = "Sample data file '" . $metadata['file_path'] . "' is a subfolder. Not supported. Ignoring it."; return 0; } - $_SESSION['errorData']['Warning'][] = "Sample data file '".$metadata['file_path']."' not in Sample Data directory ($sampleDataPath). Ignoring it."; + $_SESSION['errorData']['Warning'][] = "Sample data file '" . $metadata['file_path'] . "' not in Sample Data directory ($sampleDataPath). Ignoring it."; return 0; } if (!is_file($userDataPath)) { copy($sampleDataPath, $userDataPath); } - } elseif ($type == "folder") { if (!is_dir($sampleDataPath)) { if (is_file($sampleDataPath)) { - $_SESSION['errorData']['Warning'][] = "Sample data folder '".$metadata['file_path']."' not grouped below any folder. Ignoring it."; + $_SESSION['errorData']['Warning'][] = "Sample data folder '" . $metadata['file_path'] . "' not grouped below any folder. Ignoring it."; return 0; } - $_SESSION['errorData']['Warning'][] = "Sample data folder '".$metadata['file_path']."' not in Sample Data directory. Ignoring it."; + $_SESSION['errorData']['Warning'][] = "Sample data folder '" . $metadata['file_path'] . "' not in Sample Data directory. Ignoring it."; return 0; } @@ -299,18 +313,20 @@ function save_fromSampleDataMetadata($metadata, $dataDir, $sampleName, $type, $v mkdir($userDataPath, 0775); } - $sampleMetadataFiles= array_filter(scandir($sampleDataPath), function($i) {return preg_match('/^\.\w+/', $i);}); + $sampleMetadataFiles = array_filter(scandir($sampleDataPath), function ($i) { + return preg_match('/^\.\w+/', $i); + }); if (count($sampleMetadataFiles)) { foreach ($sampleMetadataFiles as $metadataFile) { if ($metadataFile == ".sample_metadata.json") { continue; } - copy($sampleDataPath."/".$metadataFile, $userDataPath."/".$metadataFile); + copy($sampleDataPath . "/" . $metadataFile, $userDataPath . "/" . $metadataFile); } } } else { - $_SESSION['errorData']['Internal'][] = "Sample data '".$metadata['file_path']."' cannot be injected."; + $_SESSION['errorData']['Internal'][] = "Sample data '" . $metadata['file_path'] . "' cannot be injected."; return 0; } @@ -319,18 +335,18 @@ function save_fromSampleDataMetadata($metadata, $dataDir, $sampleName, $type, $v } // adapt sample data metadata - $metadata['file_path'] = "$dataDir/".$metadata['file_path']; - $metadata['user_id'] = dirname($dataDir); - $metadata['meta_data']['validated'] = true; - if (isset($metadata['meta_data']['submission_file'])) { - $metadata['meta_data']['submission_file'] = "$dataDirPath/".$metadata['meta_data']['submission_file']; - } + $metadata['file_path'] = "$dataDir/" . $metadata['file_path']; + $metadata['user_id'] = dirname($dataDir); + $metadata['meta_data']['validated'] = true; + if (isset($metadata['meta_data']['submission_file'])) { + $metadata['meta_data']['submission_file'] = "$dataDirPath/" . $metadata['meta_data']['submission_file']; + } - if (isset($metadata['meta_data']['log_file'])) { - $metadata['meta_data']['log_file'] = "$dataDirPath/".$metadata['meta_data']['log_file']; + if (isset($metadata['meta_data']['log_file'])) { + $metadata['meta_data']['log_file'] = "$dataDirPath/" . $metadata['meta_data']['log_file']; } - - if (isset($metadata['meta_data']['associated_files'])) { + + if (isset($metadata['meta_data']['associated_files'])) { $associatedFileIds = []; foreach ($metadata['meta_data']['associated_files'] as $associatedFile) { $assocPath = "$dataDir/$associatedFile"; @@ -339,9 +355,9 @@ function save_fromSampleDataMetadata($metadata, $dataDir, $sampleName, $type, $v } $metadata['meta_data']['associated_files'] = $associatedFileIds; - } + } - if (isset($metadata['sources'])) { + if (isset($metadata['sources'])) { $associatedFileIds = []; foreach ($metadata['sources'] as $source) { $sourcePath = "$dataDir/$source"; @@ -349,18 +365,18 @@ function save_fromSampleDataMetadata($metadata, $dataDir, $sampleName, $type, $v array_push($associatedFileIds, $sourceId); } - $metadata['sources'] = $associatedFileIds; - } + $metadata['sources'] = $associatedFileIds; + } - // validate sample data metadata - if (preg_match('/uploads/', $metadata['file_path']) || preg_match('/repository/', $metadata['file_path'])) { + // validate sample data metadata + if (preg_match('/uploads/', $metadata['file_path']) || preg_match('/repository/', $metadata['file_path'])) { [$responseCode, $validatedMetadata] = validateMugFile($metadata, false); - } else { - [$responseCode ,$validatedMetadata] = validateMugFile($metadata, true); - } + } else { + [$responseCode, $validatedMetadata] = validateMugFile($metadata, true); + } - if ($responseCode == 0) { - $_SESSION['errorData']['Warning'][] = "Sample data '".$metadata['file_path']."' not injected. Its metadata is not valid. Ignoring it"; + if ($responseCode == 0) { + $_SESSION['errorData']['Warning'][] = "Sample data '" . $metadata['file_path'] . "' not injected. Its metadata is not valid. Ignoring it"; return 0; } @@ -368,241 +384,244 @@ function save_fromSampleDataMetadata($metadata, $dataDir, $sampleName, $type, $v $fileId = getGSFileId_fromPath($validatedMetadata['file_path'], 1); if ($fileId) { if ($verbose) { - $_SESSION['errorData']['Info'][] = "Sample data already in your workspace. Data '".basename($metadata['file_path'])."'"; + $_SESSION['errorData']['Info'][] = "Sample data already in your workspace. Data '" . basename($metadata['file_path']) . "'"; } - } else { - //convert metadata from MuGfile to VREfile - [$file, $metadata] = getVREfile_fromFile($validatedMetadata); + } else { + //convert metadata from MuGfile to VREfile + [$file, $metadata] = getVREfile_fromFile($validatedMetadata); - //saving metadata - if ($type == "folder") { + //saving metadata + if ($type == "folder") { $newId = createGSDirBNS($validatedMetadata['file_path'], 1); - if ($newId == "0") { - $_SESSION['errorData']['Error'][] = "Cannot register data sample '".$validatedMetadata['file_path']."'"; - return 0; + if ($newId == "0") { + $_SESSION['errorData']['Error'][] = "Cannot register data sample '" . $validatedMetadata['file_path'] . "'"; + return 0; } - if (addMetadataBNS($newId, $metadata) == "0") { - $_SESSION['errorData']['login'][] = "Cannot register data sample '".$validatedMetadata['file_path']."'"; + if (addMetadataBNS($newId, $metadata) == "0") { + $_SESSION['errorData']['login'][] = "Cannot register data sample '" . $validatedMetadata['file_path'] . "'"; return 0; } if ($verbose) { - $_SESSION['errorData']['Info'][]= "Sample data imported in your workspace. New Project: '".basename($metadata['file_path'])."'"; + $_SESSION['errorData']['Info'][] = "Sample data imported in your workspace. New Project: '" . basename($metadata['file_path']) . "'"; } - - } elseif ($type == "file") { + } elseif ($type == "file") { $newId = uploadGSFileBNS($validatedMetadata['file_path'], $userDataPath, $file, $metadata, FALSE, 1); - if ($newId == "0") { - $_SESSION['errorData']['Error'][] = "Cannot register data sample '".$validatedMetadata['file_path']."'"; + if ($newId == "0") { + $_SESSION['errorData']['Error'][] = "Cannot register data sample '" . $validatedMetadata['file_path'] . "'"; return 0; } - + if (isset($metadata['file_path']) && preg_match('/uploads/', $metadata['file_path'])) { if ($verbose) { - $_SESSION['errorData']['Info'][]= "Sample data imported in your uploads folder. New File: '".basename($metadata['file_path'])."'"; + $_SESSION['errorData']['Info'][] = "Sample data imported in your uploads folder. New File: '" . basename($metadata['file_path']) . "'"; } } - } - } + } + } return 1; } -function getFilesToDisplay($dirSelection,$filter_data_types=array()) { +function getFilesToDisplay($dirSelection, $filter_data_types = array()) +{ - $filesAll=Array(); + $filesAll = array(); - // Register recent outputs & extract pending files - $filesPending= processPendingFiles($_SESSION['User']['_id']); + // Register recent outputs & extract pending files + $filesPending = processPendingFiles($_SESSION['User']['_id']); // Retrieve files from Mongo - $files=getGSFilesFromDir($dirSelection,1); + $files = getGSFilesFromDir($dirSelection, 1); - if (!$files){ - $_SESSION['errorData']['Error'][]="Cannot update dashboard."; - return $filesAll; - } + if (!$files) { + $_SESSION['errorData']['Error'][] = "Cannot update dashboard."; + return $filesAll; + } - // Filter user pending files not belonging to active project - if ($filesPending){ - foreach ($filesPending as $r){ - if ($r['project'] != $_SESSION['User']['activeProject']){ - unset($filesPending[$r['_id']]); - } + // Filter user pending files not belonging to active project + if ($filesPending) { + foreach ($filesPending as $r) { + if ($r['project'] != $_SESSION['User']['activeProject']) { + unset($filesPending[$r['_id']]); + } + } } - } // Merge pending files and mongo data - if ($filesPending){ - foreach ($filesPending as $r){ + if ($filesPending) { + foreach ($filesPending as $r) { - // Update $files[parentId][files] - if (!isset($filesPending[$r['_id']]['parentDir'])) { - $_SESSION['errorData']['Error'][]="Pending file ".$filesPending[$r['_id']]['path']." has no parentDir"; - continue; - } - $parentId = $filesPending[$r['_id']]['parentDir']; - - if (!isset($files[$parentId])){ - if ($r['pending']){ - unset($filesPending[$r['_id']]); - }else{ - $_SESSION['errorData']['Error'][] ="Cannot display '".$filesPending[$r['_id']]['path']."'. FS inconsistency. Its parent folder ($parentId) does not exist anymore or is unaccessible."; - unset($filesPending[$r['_id']]); + // Update $files[parentId][files] + if (!isset($filesPending[$r['_id']]['parentDir'])) { + $_SESSION['errorData']['Error'][] = "Pending file " . $filesPending[$r['_id']]['path'] . " has no parentDir"; + continue; } - continue; + $parentId = $filesPending[$r['_id']]['parentDir']; + + if (!isset($files[$parentId])) { + if ($r['pending']) { + unset($filesPending[$r['_id']]); + } else { + $_SESSION['errorData']['Error'][] = "Cannot display '" . $filesPending[$r['_id']]['path'] . "'. FS inconsistency. Its parent folder ($parentId) does not exist anymore or is unaccessible."; + unset($filesPending[$r['_id']]); + } + continue; + } + array_push($files[$parentId]['files'], $r['_id']); } - array_push($files[$parentId]['files'],$r['_id']); - } - $filesAll=array_merge($files,$filesPending); - }else{ - $filesAll=$files; - } + $filesAll = array_merge($files, $filesPending); + } else { + $filesAll = $files; + } - return $filesAll; + return $filesAll; } -function filterFiles_by_dataType($filesAll,$filter_data_types=array()) { +function filterFiles_by_dataType($filesAll, $filter_data_types = array()) +{ - // Filter files by data_types + // Filter files by data_types - if ($filter_data_types || is_array($filter_data_types)){ - $filesFiltered = array(); - $dirs_filtered = array(); - //filter out files with unselected data_types - foreach ($filesAll as $fn => $file ){ - if (isset($file['data_type']) and in_array($file['data_type'],$filter_data_types) ){ - $filesFiltered[$fn] = $filesAll[$fn]; - array_push($dirs_filtered,$file['parentDir']); - } - } - //filter out empty dirs - foreach ($filesAll as $fn => $file ){ - if (isset($file['parentDir']) and in_array($file['_id'],$dirs_filtered) ){ - $filesFiltered[$fn] = $filesAll[$fn]; - } + if ($filter_data_types || is_array($filter_data_types)) { + $filesFiltered = array(); + $dirs_filtered = array(); + //filter out files with unselected data_types + foreach ($filesAll as $fn => $file) { + if (isset($file['data_type']) and in_array($file['data_type'], $filter_data_types)) { + $filesFiltered[$fn] = $filesAll[$fn]; + array_push($dirs_filtered, $file['parentDir']); + } + } + //filter out empty dirs + foreach ($filesAll as $fn => $file) { + if (isset($file['parentDir']) and in_array($file['_id'], $dirs_filtered)) { + $filesFiltered[$fn] = $filesAll[$fn]; + } + } + $filesAll = $filesFiltered; } - $filesAll = $filesFiltered; - } - return $filesAll; + return $filesAll; } //add datatable tree nodes and hidden cols values -function addTreeTableNodesToFiles($filesAll){ - $n=1; - foreach ($filesAll as $r){ +function addTreeTableNodesToFiles($filesAll) +{ + $n = 1; + foreach ($filesAll as $r) { // Add Tree Nodes - if (isset($r['files'])){ - if (isset($filesAll[$r['_id']]['tree_id']) && $filesAll[$r['_id']]['tree_id']) - continue; - - $filesAll[$r['_id']]['tree_id'] = $n; - $filesAll[$r['_id']]['size'] = calcGSUsedSpaceDir($r['_id']); - $filesAll[$r['_id']]['size_parent'] = $filesAll[$r['_id']]['size']; - $filesAll[$r['_id']]['mtime_parent']=(isset($r['atime'])? $r['atime']->toDateTime()->format('U') : $r['mtime']); - $i=1; - foreach ($r['files'] as $rr){ - $filesAll[$rr]['tree_id'] = "$n.$i"; - $filesAll[$rr]['tree_id_parent']= $n; - $filesAll[$rr]['size_parent'] = $filesAll[$r['_id']]['size_parent']; - $filesAll[$rr]['mtime_parent'] = $filesAll[$r['_id']]['mtime_parent']; - $i++; - } - $n++; - }else{ - if (isset($r['pending']) ){ + if (isset($r['files'])) { + if (isset($filesAll[$r['_id']]['tree_id']) && $filesAll[$r['_id']]['tree_id']) + continue; + + $filesAll[$r['_id']]['tree_id'] = $n; + $filesAll[$r['_id']]['size'] = calcGSUsedSpaceDir($r['_id']); + $filesAll[$r['_id']]['size_parent'] = $filesAll[$r['_id']]['size']; + $filesAll[$r['_id']]['mtime_parent'] = (isset($r['atime']) ? $r['atime']->toDateTime()->format('U') : $r['mtime']); + $i = 1; + foreach ($r['files'] as $rr) { + $filesAll[$rr]['tree_id'] = "$n.$i"; + $filesAll[$rr]['tree_id_parent'] = $n; + $filesAll[$rr]['size_parent'] = $filesAll[$r['_id']]['size_parent']; + $filesAll[$rr]['mtime_parent'] = $filesAll[$r['_id']]['mtime_parent']; + $i++; + } + $n++; + } else { + if (isset($r['pending'])) { $dir = $r['parentDir']; - $filesAll[$dir]['pending']="true"; + $filesAll[$dir]['pending'] = "true"; } } - } + } return $filesAll; - } -function printTable($filesAll=Array() ) { - $autorefresh=0; - ?> +function printTable($filesAll = array()) +{ + $autorefresh = 0; +?> - + ?>
- \n"; } - // Convert the PHP array to JSON and output it to the browser console - ?> - - - + + + $node) { @@ -610,72 +629,70 @@ function printLastJobs($filesAll=Array() ) { } array_multisort($timestamps, SORT_DESC, $filesAll); - ?> +?> - find(array('external' => true, 'status' => array('$in' => [$status, 3]))); - if($_SESSION['User']['Type'] == 1) { - - $tools_list = iterator_to_array($tl, false); + if ($_SESSION['User']['Type'] == 1) { - foreach($tools_list as $key => $tool) { + $tools_list = iterator_to_array($tl, false); - if($tool["status"] == 3 && !in_array($tool["_id"], $_SESSION['User']["ToolsDev"])) { - unset($tools_list[$key]); - } + foreach ($tools_list as $key => $tool) { - } + if ($tool["status"] == 3 && !in_array($tool["_id"], $_SESSION['User']["ToolsDev"])) { + unset($tools_list[$key]); + } + } $tl = $tools_list; - } $arrTools = []; - foreach($tl as $tool){ - if(isset($tool["input_files_combinations_internal"])) { + foreach ($tl as $tool) { + if (isset($tool["input_files_combinations_internal"])) { $combinations = $tool["input_files_combinations_internal"]; foreach ($combinations as $comb) { if (sizeof($comb) == 1) { @@ -684,888 +701,882 @@ function getToolsByDT($data_type, $status = 1) { $aux = array($tool["_id"], $tool["name"]); $arrTools[] = $aux; } - } + } } } - } else if(sizeof($tool["input_files"]) == 1) { + } else if (sizeof($tool["input_files"]) == 1) { - if(in_array($data_type, $tool["input_files"][0]["data_type"])) { + if (in_array($data_type, $tool["input_files"][0]["data_type"])) { $aux = array($tool["_id"], $tool["name"]); $arrTools[] = $aux; - } - } - } return $arrTools; - } -function formatData($data) { - //_id id_URL - if (!isset($data['_id'])) - return $data; - $data['_id_URL'] = urlencode($data['_id']); - //mtime atime - if (isset($data['mtime'])){ - if (is_object($data['mtime'])){ - $data['mtime']=$data['mtime']->toDateTime()->format('U'); - } - $data['mtime'] = datefmt_format(getDateTimeFormat(), $data['mtime']); - - $hoursleft = (time() - (int) $data['mtime']) / 3600; - - //if($hoursleft < 1) $data['lastuploaded'] = ''; - if($hoursleft < 1) $data['lastuploaded'] = ' new '; - else $data['lastuploaded'] = ''; - - //$data['mtime'] .= $data['hoursleft']; - - - }else{ - $data['mtime']=""; - } - if (isset($data['atime'])){ - if (is_object($data['atime'])) - $data['atime'] =$data['atime']->toDateTime()->format('U'); - $data['atime'] = datefmt_format(getDateTimeFormat(), $data['atime']); - $data['mtime'] = $data['atime']; - } - //format - if (!isset($data['format'])) - $data['format']=""; - //type - if (!isset($data['type'])) - $data['type']="file"; - //expiration - if (isset($data['expiration'])){ - if (!is_object($data['expiration']) && $data['expiration'] == -1){ - $data['expiration'] = "File/folder does not expire"; - }else{ - if (is_object($data['expiration'])) - $data['expiration'] =$data['expiration']->toDateTime()->format('U'); - - $days2expire = intval(( $data['expiration'] - time() ) / (24 * 3600)); - $data['expiration'] = datefmt_format(getDateTimeFormat(), $data['expiration']); - if ($days2expire < 7) - $data['expiration'] =$data['expiration'] ."( in ".$days2expire." days)"; - else - $data['expiration'] =$data['expiration'] . "( in $days2expire days)"; - } - }else{ - $data['expiration'] ="No expiration date"; - } - //size - if (isset($data['files']) && !isset($data['size']) ){ - $data['size'] = calcGSUsedSpaceDir($data['_id']); - } - if (isset($data['size']) && is_numeric($data['size'])) { - $sz = 'BKMGTP'; - $factor = floor((strlen($data['size']) - 1) / 3); - $data['size'] = sprintf("%.2f %s", $data['size'] / pow(1024, $factor),@$sz[$factor]); - }else{ - $data['size']=""; - } - //execution dir - if (isset($data['parentDir'])){ - $data['parentDir'] = getAttr_fromGSFileId($data['parentDir'],'path'); - if (!$data['parentDir']){ - $_SESSION['errorData']['Warning'][]="Accessing data not belonging to your account! Some permission issues may arise"; - } - if ($data['type'] == "file"){ - $parentDir_explode = explode("/",$data['parentDir']); - $executionName = array_pop($parentDir_explode); - }else{ - $path_explode = explode("/",$data['path']); - $executionName = array_pop($path_explode); - } - if($executionName == 'uploads') { +function formatData($data) +{ + //_id id_URL + if (!isset($data['_id'])) + return $data; + $data['_id_URL'] = urlencode($data['_id']); + //mtime atime + if (isset($data['mtime'])) { + if (is_object($data['mtime'])) { + $data['mtime'] = $data['mtime']->toDateTime()->format('U'); + } + $data['mtime'] = datefmt_format(getDateTimeFormat(), $data['mtime']); + + $hoursleft = (time() - (int) $data['mtime']) / 3600; + + //if($hoursleft < 1) $data['lastuploaded'] = ''; + if ($hoursleft < 1) $data['lastuploaded'] = ' new '; + else $data['lastuploaded'] = ''; + + //$data['mtime'] .= $data['hoursleft']; + + + } else { + $data['mtime'] = ""; + } + if (isset($data['atime'])) { + if (is_object($data['atime'])) + $data['atime'] = $data['atime']->toDateTime()->format('U'); + $data['atime'] = datefmt_format(getDateTimeFormat(), $data['atime']); + $data['mtime'] = $data['atime']; + } + //format + if (!isset($data['format'])) + $data['format'] = ""; + //type + if (!isset($data['type'])) + $data['type'] = "file"; + //expiration + if (isset($data['expiration'])) { + if (!is_object($data['expiration']) && $data['expiration'] == -1) { + $data['expiration'] = "File/folder does not expire"; + } else { + if (is_object($data['expiration'])) + $data['expiration'] = $data['expiration']->toDateTime()->format('U'); + + $days2expire = intval(($data['expiration'] - time()) / (24 * 3600)); + $data['expiration'] = datefmt_format(getDateTimeFormat(), $data['expiration']); + if ($days2expire < 7) + $data['expiration'] = $data['expiration'] . "( in " . $days2expire . " days)"; + else + $data['expiration'] = $data['expiration'] . "( in $days2expire days)"; + } + } else { + $data['expiration'] = "No expiration date"; + } + //size + if (isset($data['files']) && !isset($data['size'])) { + $data['size'] = calcGSUsedSpaceDir($data['_id']); + } + if (isset($data['size']) && is_numeric($data['size'])) { + $sz = 'BKMGTP'; + $factor = floor((strlen($data['size']) - 1) / 3); + $data['size'] = sprintf("%.2f %s", $data['size'] / pow(1024, $factor), @$sz[$factor]); + } else { + $data['size'] = ""; + } + //execution dir + if (isset($data['parentDir'])) { + $data['parentDir'] = getAttr_fromGSFileId($data['parentDir'], 'path'); + if (!$data['parentDir']) { + $_SESSION['errorData']['Warning'][] = "Accessing data not belonging to your account! Some permission issues may arise"; + } + if ($data['type'] == "file") { + $parentDir_explode = explode("/", $data['parentDir']); + $executionName = array_pop($parentDir_explode); + } else { + $path_explode = explode("/", $data['path']); + $executionName = array_pop($path_explode); + } + if ($executionName == 'uploads') { $executionName = "0uploads"; $data['longexecutionname'] = 'uploads'; - $data['short_execution'] ="0uploads"; - }else{ + $data['short_execution'] = "0uploads"; + } else { $data['short_execution'] = maxlength(basename($executionName), 15); $data['longexecutionname'] = $executionName; - } - $data['execution'] = $executionName; - - } - //project name - - //if (isset($data['project'])){ - // $p = getProject($data['project']); - // $data['project'] = $p['name']; - //}else{ - $p_code=""; - if (preg_match('/\/(__PROJ[^\/]*)/',$data['path'],$match)){ $p_code = $match[1];} - $p = getProject($p_code); - $data['project'] = $p['name']; - //} - - // description - if (isset($data['description'])){ - if(strlen($data['description']) > 50) $data['description'] = substr($data['description'], 0, 50).'...'; - } - //filename - if (isset($data['pending'])){ - if (!isset($data['files'])){ - $data['filename']=$data['title']; - $data['longfilename']= $data['title']; - #$viewLog_state="enabled"; - #if ($data['pending']=="HOLD" || $data['pending']=="PENDING"){ - # $viewLog_state = 'disabled'; - #}elseif(!is_file($GLOBALS['dataDir']."/".$data['log_file']) && !is_link($GLOBALS['dataDir']."/".$data['log_file'])){ - # $viewLog_state = 'disabled'; - #} - #$data['viewLog'] = "Log file:View"; - #$data['log_file'] = basename($data['log_file']); - }else{ - $data['filename']= maxlength(basename($data['path']), 15); - $data['longfilename']= basename($data['path']); - } - }else{ - $data['filename']= maxlength(basename($data['path']), 15); - $data['longfilename']= basename($data['path']); - } - //file_url - if (isset($data['file_url'])){ - $data['show_file_url']= ""; - } - - // TODO for debug. Temporal. To delete - if ($data['filename']){ - if (!is_url($data['path'])){ - $rfn = $GLOBALS['dataDir']."/".$data['path']; - if (!is_file($rfn) && !is_dir($rfn)){ - $data['filename']="ERROR-".$data['filename']; - } - } - } - if(isset($data['submission_file'])){ - $data['execDetails'] = "Execution details:Analysis parameters"; - }else{ - $data['execDetails'] = ""; - } - if(isset($data['log_file'])){ - if (preg_match('/^\//',$data['log_file'])){ - $data['log_file'] = str_replace($GLOBALS['dataDir']."/","",$data['log_file']); - } - $viewLog_state="enabled"; - if ($data['pending']=="HOLD" || $data['pending']=="PENDING"){ - $viewLog_state = 'disabled'; - }elseif(!is_file($GLOBALS['dataDir']."/".$data['log_file']) && !is_link($GLOBALS['dataDir']."/".$data['log_file'])){ - $viewLog_state = 'disabled'; - } - $data['viewLog'] = "Log file:View"; - }else{ - $data['viewLog'] = ""; } + $data['execution'] = $executionName; + } + //project name + + //if (isset($data['project'])){ + // $p = getProject($data['project']); + // $data['project'] = $p['name']; + //}else{ + $p_code = ""; + if (preg_match('/\/(__PROJ[^\/]*)/', $data['path'], $match)) { + $p_code = $match[1]; + } + $p = getProject($p_code); + $data['project'] = $p['name']; + //} - $data['tools_button'] = 'none'; - - // tools list - if ( isset($data['data_type']) && ($data['data_type'] != "")){ - - $tList = getToolsByDT($data['data_type'], 1); + // description + if (isset($data['description'])) { + if (strlen($data['description']) > 50) $data['description'] = substr($data['description'], 0, 50) . '...'; + } + //filename + if (isset($data['pending'])) { + if (!isset($data['files'])) { + $data['filename'] = $data['title']; + $data['longfilename'] = $data['title']; + #$viewLog_state="enabled"; + #if ($data['pending']=="HOLD" || $data['pending']=="PENDING"){ + # $viewLog_state = 'disabled'; + #}elseif(!is_file($GLOBALS['dataDir']."/".$data['log_file']) && !is_link($GLOBALS['dataDir']."/".$data['log_file'])){ + # $viewLog_state = 'disabled'; + #} + #$data['viewLog'] = "Log file:View"; + #$data['log_file'] = basename($data['log_file']); + } else { + $data['filename'] = maxlength(basename($data['path']), 15); + $data['longfilename'] = basename($data['path']); + } + } else { + $data['filename'] = maxlength(basename($data['path']), 15); + $data['longfilename'] = basename($data['path']); + } + //file_url + if (isset($data['file_url'])) { + $data['show_file_url'] = ""; + } - $data['tools_list'] = ''; + } - $data['renameLink'] = "
  • Rename
    • "; - $data['moveLink'] = "
  • Move
    • "; + //data_type + if (isset($data['data_type']) && $data['data_type']) { + $dt_name = getDataTypeName($data['data_type']); + $data['file_data_type'] = $dt_name; + $data['short_file_data_type'] = maxlength(basename($dt_name), 20); + $data['data_type'] = "Data type:" . $dt_name . ""; + } else { + $data['data_type'] = ""; + } + //notes + if (isset($data['notes']) && strlen($data['notes'])) { + $data['notes'] = "Notes:" . $data['notes'] . ""; + } else { + $data['notes'] = ""; + } + //paired sorted refGenome + if (isset($data['paired']) || isset($data['sorted'])) { + $row = "BAM properties:"; + if (isset($data['paired'])) + $row .= $data['paired']; + if (isset($data['sorted'])) + $row .= " " . $data['sorted']; + $row .= ""; + $data['paired'] = $row; + } else { + $data['paired'] = ""; + } + if (isset($data['refGenome'])) { + $data['refGenome'] = "Assembly:" . $data['refGenome'] . ""; + } else { + $data['refGenome'] = ""; + } + //state and metadataLink + if (isset($data['validated']) && $data['validated']) { + $data['state'] = 'enabled'; + $data['metadataLink'] = "
  • Edit file metadata
    • "; + } else { + $data['state'] = 'disabled'; + $data['metadataLink'] = "
  • Validate file
    • "; + } - //tools list (old school version :) delete - - //visualization - if ( isset($data['format']) ){ + $data['renameLink'] = "
  • Rename
    • "; + $data['moveLink'] = "
  • Move
    • "; - $data['vis_button'] = 'block'; + //tools list (old school version :) delete - $data['vis_button'] = 'none'; + //visualization + if (isset($data['format'])) { - $visualizers = getVisualizers_ListComplete(); - foreach($visualizers as $vis) { - if(in_array($data['format'], $vis["preview"])) { - $data['vis_button'] = 'block'; - switch($vis["_id"]) { + $data['vis_button'] = 'block'; - case "ngl": - $ext = 'pdb'; - if ($pos = strrpos($data['longfilename'], '.')) { - $name = substr($data['longfilename'], 0, $pos); - $ext = substr($data['longfilename'], $pos); - } else { - $name = $data['longfilename']; - } + $data['vis_button'] = 'none'; - $e = ltrim($ext, "."); - $data['PDBView'] = "
  • Preview in 3D
    • "; - break; + $visualizers = getVisualizers_ListComplete(); + foreach ($visualizers as $vis) { + if (in_array($data['format'], $vis["preview"])) { + $data['vis_button'] = 'block'; + switch ($vis["_id"]) { - case "tadkit": - if ($pos = strrpos($data['longfilename'], '.')) { - $name = substr($data['longfilename'], 0, $pos); - $ext = substr($data['longfilename'], $pos); - } else { - $name = $data['longfilename']; - } + case "ngl": + $ext = 'pdb'; + if ($pos = strrpos($data['longfilename'], '.')) { + $name = substr($data['longfilename'], 0, $pos); + $ext = substr($data['longfilename'], $pos); + } else { + $name = $data['longfilename']; + } - $data['PDBView'] = "
  • Preview in 3D
    • "; - break; - - } - } + $e = ltrim($ext, "."); + $data['PDBView'] = "
  • Preview in 3D
    • "; + break; - if(in_array($data['format'], $vis["accepted_file_types"])) { - $data['vis_button'] = 'block'; - switch($vis["_id"]) { - case "ngl": - $data['NGLView'] = "
  • View in NGL
    • "; - break; + case "tadkit": + if ($pos = strrpos($data['longfilename'], '.')) { + $name = substr($data['longfilename'], 0, $pos); + $ext = substr($data['longfilename'], $pos); + } else { + $name = $data['longfilename']; + } - case "jbrowse": - $data['jbrowseLink'] = "
  • View in JBrowse
    • "; - break; + $data['PDBView'] = "
  • Preview in 3D
    • "; + break; + } + } - case "tadkit": - $data['tadkitLink'] = "
  • View in TADkit
    • "; - break; + if (in_array($data['format'], $vis["accepted_file_types"])) { + $data['vis_button'] = 'block'; + switch ($vis["_id"]) { + case "ngl": + $data['NGLView'] = "
  • View in NGL
    • "; + break; - } + case "jbrowse": + $data['jbrowseLink'] = "
  • View in JBrowse
    • "; + break; + case "tadkit": + $data['tadkitLink'] = "
  • View in TADkit
    • "; + break; } } - } - //input_files - if (isset($data['input_files'])){ - $ins =$data['input_files']; - $data['input_files']="Input files:"; - if (count($ins)){ - foreach ($ins as $in){ - $f = getGSFile_fromId($in); - if ($f == 0) { - error_log("File $in not found"); - continue; - } - - $data['input_files'].= "
    "; - $inFolders=explode("/",dirname($f['path'])); - for ($i=count($inFolders)-1;$i>=1;$i--){ - $data['input_files'].= "".$inFolders[$i]."/"; - } - $data['input_files'].= basename($f['path'])."
    "; + } + //input_files + if (isset($data['input_files'])) { + $ins = $data['input_files']; + $data['input_files'] = "Input files:"; + if (count($ins)) { + foreach ($ins as $in) { + $f = getGSFile_fromId($in); + if ($f == 0) { + error_log("File $in not found"); + continue; } - } - $data['input_files'].=""; - } - //rerunLink - if (isset($data['input_files']) && isset($data['tool'])){ - $tool = $GLOBALS['toolsCol']->findOne(array('_id' => $data['tool'])); - if (!empty($tool)){ - $formPath = "tools/".$data['tool']."/input.php"; - $data['rerunLink'] ="
  • Rerun Project
    • "; + + $data['input_files'] .= "
    "; + $inFolders = explode("/", dirname($f['path'])); + for ($i = count($inFolders) - 1; $i >= 1; $i--) { + $data['input_files'] .= "" . $inFolders[$i] . "/"; + } + $data['input_files'] .= basename($f['path']) . "
    "; } } - //viewResultsLink - if (isset($data['tool']) ){ - $tool = $GLOBALS['toolsCol']->findOne(array('_id' => $data['tool'])); - $data['toolname'] = $data['tool']; - if (!empty($tool)){ - if (isset($tool['has_custom_viewer']) && $tool['has_custom_viewer'] === false){ - }else{ - $data['viewResultsLink']=" + $data['input_files'] .= ""; + } + //rerunLink + if (isset($data['input_files']) && isset($data['tool'])) { + $tool = $GLOBALS['toolsCol']->findOne(array('_id' => $data['tool'])); + if (!empty($tool)) { + $formPath = "tools/" . $data['tool'] . "/input.php"; + $data['rerunLink'] = "
  • Rerun Project
    • "; + } + } + //viewResultsLink + if (isset($data['tool'])) { + $tool = $GLOBALS['toolsCol']->findOne(array('_id' => $data['tool'])); + $data['toolname'] = $data['tool']; + if (!empty($tool)) { + if (isset($tool['has_custom_viewer']) && $tool['has_custom_viewer'] === false) { + } else { + $data['viewResultsLink'] = "
    "; - } - } - } - - //analyses tool - if (isset($data['tool'])){ - $tool = $GLOBALS['toolsCol']->findOne(array('_id' => $data['tool'])); - if (!empty($tool)) - $data['tool']="Tool:".$tool['name'] .""; - } - //compressed - $ext = pathinfo($data['path'], PATHINFO_EXTENSION); - $ext = preg_replace('/_\d+$/',"",$ext); - $content_type = ( array_key_exists($ext, mimeTypes()) ? mimeTypes()[$ext] : "application/octet-stream"); - $data['openFunction'] = ($content_type == "text/plain" || $ext=="pdf" || preg_match('/image/',$content_type) || preg_match('/(e|o)\d+/',$ext) || in_array($data['format'],array("FASTQ","FASTA")) ? "openPlainFile" : "downloadFile"); - $data['compressionLink'] = ""; - if (! in_array($data['format'],array("BAM","PNG","JPG") ) ){ - switch (strtolower($ext)) { - case 'tar': - $func = "untar"; - $img = "fa fa-expand"; - $linkTxt= "Uncompress"; - break; - case 'gz': - case 'zip': - $func = "unzip"; - $img = "fa fa-expand"; - $linkTxt= "Uncompress"; - case 'tgz': - $func = "untar"; - $img = "fa fa-expand"; - $linkTxt= "Uncompress"; - break; - case 'bz2': - $func = "bzip2"; - $img = "fa fa-expand"; - $linkTxt= "Uncompress"; - default : - $func = "zip"; - $img = "fa fa-file-zip-o"; - $linkTxt= "Compress"; } - $data['compressionLink'] = "
  • $linkTxt
    • "; - //$data['compressionLink'] = "
  • $linkTxt
    • "; } - return $data; + } + + //analyses tool + if (isset($data['tool'])) { + $tool = $GLOBALS['toolsCol']->findOne(array('_id' => $data['tool'])); + if (!empty($tool)) + $data['tool'] = "Tool:" . $tool['name'] . ""; + } + //compressed + $ext = pathinfo($data['path'], PATHINFO_EXTENSION); + $ext = preg_replace('/_\d+$/', "", $ext); + $content_type = (array_key_exists($ext, mimeTypes()) ? mimeTypes()[$ext] : "application/octet-stream"); + $data['openFunction'] = ($content_type == "text/plain" || $ext == "pdf" || preg_match('/image/', $content_type) || preg_match('/(e|o)\d+/', $ext) || in_array($data['format'], array("FASTQ", "FASTA")) ? "openPlainFile" : "downloadFile"); + $data['compressionLink'] = ""; + if (! in_array($data['format'], array("BAM", "PNG", "JPG"))) { + switch (strtolower($ext)) { + case 'tar': + $func = "untar"; + $img = "fa fa-expand"; + $linkTxt = "Uncompress"; + break; + case 'gz': + case 'zip': + $func = "unzip"; + $img = "fa fa-expand"; + $linkTxt = "Uncompress"; + case 'tgz': + $func = "untar"; + $img = "fa fa-expand"; + $linkTxt = "Uncompress"; + break; + case 'bz2': + $func = "bzip2"; + $img = "fa fa-expand"; + $linkTxt = "Uncompress"; + default: + $func = "zip"; + $img = "fa fa-file-zip-o"; + $linkTxt = "Compress"; + } + $data['compressionLink'] = "
  • $linkTxt
    • "; + //$data['compressionLink'] = "
  • $linkTxt
    • "; + } + return $data; } //update Mongo lastjobs -function updatePendingFiles($sessionId,$singleJob=Array()){ - $SGE_updated = Array(); // jobs to be monitored in next round. Stored in SESSION. Updated by checkPendingJobs.php (called by ajax) +function updatePendingFiles($sessionId, $singleJob = array()) +{ + $SGE_updated = array(); // jobs to be monitored in next round. Stored in SESSION. Updated by checkPendingJobs.php (called by ajax) - // get jobs from mongo[users][lastjobs] - $lastjobs = getUserJobs($sessionId); + // get jobs from mongo[users][lastjobs] + $lastjobs = getUserJobs($sessionId); - if (count($lastjobs)){ + if (count($lastjobs)) { - //classify jobs - foreach ($lastjobs as $job){ + //classify jobs + foreach ($lastjobs as $job) { - if (!isset($job['_id'])){ - continue; - } - $pid = $job['pid']; + if (!isset($job['_id'])) { + continue; + } + $pid = $job['pid']; - //get qstat info - logger("Start processPendingFiles -> getRunningJobInfo $pid. Log= ".$job['log_file']); - $jobProcess = getRunningJobInfo($pid,$job['launcher'],$job['cloudName']); + //get qstat info + logger("Start processPendingFiles -> getRunningJobInfo $pid. Log= " . $job['log_file']); + $jobProcess = getRunningJobInfo($pid, $job['launcher'], $job['cloudName']); - // TODO: PMES will redirect log info to log_file. Now, info extracted from $jobProcess - updateLogFromJobInfo($job['log_file'],$pid,$job['launcher']); + // TODO: PMES will redirect log info to log_file. Now, info extracted from $jobProcess + updateLogFromJobInfo($job['log_file'], $pid, $job['launcher']); - - //job keeps running: maintain original job data - if (count($jobProcess)){ - //keep monitoring - $job['state'] = $jobProcess['state']; - $SGE_updated[$pid]= $job; - - //job not running : edit SGE_updated to register the change - // and consequently reload workspace (checkPendingJobs.php) - }else{ - log_addFinish($pid,"Automatic job update detects job $pid is not running anymore"); - $SGE_updated[$pid]=$job; - $SGE_updated[$pid]['state']="NOT_RUNNING"; - } - } - } - - //update session and save to mongo - saveUserJobs($sessionId,$SGE_updated); - return 1; + + //job keeps running: maintain original job data + if (count($jobProcess)) { + //keep monitoring + $job['state'] = $jobProcess['state']; + $SGE_updated[$pid] = $job; + + //job not running : edit SGE_updated to register the change + // and consequently reload workspace (checkPendingJobs.php) + } else { + log_addFinish($pid, "Automatic job update detects job $pid is not running anymore"); + $SGE_updated[$pid] = $job; + $SGE_updated[$pid]['state'] = "NOT_RUNNING"; + } + } + } + + //update session and save to mongo + saveUserJobs($sessionId, $SGE_updated); + return 1; } -function processPendingFiles($sessionId,$files=array()){ +function processPendingFiles($sessionId, $files = array()) +{ - $SGE_updated = Array(); // jobs to be monitored. Stored in SESSION. Updated by checkPendingJobs.php (called by ajax) - $filesPending= Array(); // files to be listed - $debug=0; + $SGE_updated = array(); // jobs to be monitored. Stored in SESSION. Updated by checkPendingJobs.php (called by ajax) + $filesPending = array(); // files to be listed + $debug = 0; // get files already in mongo - $filesStored = Array(); - if ($files){ - foreach ($files as $k => $v){ - array_push($filesStored,$v['_id']); - } + $filesStored = array(); + if ($files) { + foreach ($files as $k => $v) { + array_push($filesStored, $v['_id']); + } } // get jobs from mongo[users][lastjobs] $lastjobs = getUserJobs($sessionId); - if (!count($lastjobs)){ + if (!count($lastjobs)) { return $filesPending; } if ($debug) - print "

    JOBS DEL USER HAS [".count($lastjobs)."] JOBS
    \n"; + print "

    JOBS DEL USER HAS [" . count($lastjobs) . "] JOBS
    \n"; // classify jobs - foreach ($lastjobs as $job){ - - if (!isset($job['pid'])){ + foreach ($lastjobs as $job) { + + if (!isset($job['pid'])) { continue; } $pid = $job['pid']; if ($debug) - print "
    \nPID = [$pid] TOOL=".$job['toolId']." WORK_DIR=".$job['working_dir']."
    \n"; + print "
    \nPID = [$pid] TOOL=" . $job['toolId'] . " WORK_DIR=" . $job['working_dir'] . "
    \n"; - //get qstat info - logger("Start processPendingFiles -> getRunningJobInfo $pid. Log= ".$job['log_file']); - $jobProcess = getRunningJobInfo($pid,$job['launcher'],$job['cloudName']); + //get qstat info + logger("Start processPendingFiles -> getRunningJobInfo $pid. Log= " . $job['log_file']); + $jobProcess = getRunningJobInfo($pid, $job['launcher'], $job['cloudName']); - // TODO: PMES will redirect log info to log_file. Now, info extracted from $jobProcess - updateLogFromJobInfo($job['log_file'],$pid,$job['launcher']); - - // - //set as running job - // - $title = (isset($job['title'])?$job['title']:"Job ".$job['execution']); - $descrip = getJobDescription($job['description'],$jobProcess,$lastjobs); - - if (count($jobProcess)){ - - if ($debug) - print "RUNNING JOB"; - - //set dummy id - $dummyId = $job['pid']."_dummy"; - - //get dummy parentDir - if ($job['hasExecutionFolder']){ - // show job in execution dir - $parentDir = fromAbsPath_toPath($job['working_dir']); - }else{ - // show job in output_dir (infered from stageout_data) - $parentDir= 0; - if($job['stageout_data']){ - $output_file_1 = $job['stageout_data']['output_files'][0]; - if ($output_file_1 && $output_file_1['file_path']){ - $parentDir = fromAbsPath_toPath(dirname($output_file_1['file_path'])); - } - } - if (!$parentDir) - $parentDir = $_SESSION['User']['id']."/".$_SESSION['User']['activeProject']."/uploads"; - } - - //set dummy file - $fileDummy = Array( - '_id' => $dummyId, - 'pid' => $pid, - 'title' => $title, - 'mtime' => new MongoDB\BSON\UTCDateTime(strtotime($jobProcess['submission_time'])*1000), - 'size' => "", - 'visible' => 1, - 'tool' => $job['toolId'], - 'project' => $job['project'], - 'parentDir'=> getGSFileId_fromPath($parentDir), - 'description'=> $descrip, - 'pending' => $jobProcess['state'], - 'submission_file' => fromAbsPath_toPath($job['submission_file']), - 'log_file' => fromAbsPath_toPath($job['log_file']), - 'stdout_file' => fromAbsPath_toPath($job['stdout_file']), - 'stderr_file' => fromAbsPath_toPath($job['stderr_file']), - 'job_type' => $job['job_type'] - ); - - if ($jobProcess['state']=="RUNNING" && $job['job_type']=="interactive"){ - $fileDummy['pending'] = "ACTIVE SESSION"; - } - - //list job in workspace - $filesPending[$dummyId] = $fileDummy; - - //update job state in mongo - $job['state'] = $jobProcess['state']; - $SGE_updated[$pid]=$job; - - // - // processing job non running anymore - // - }else{ - log_addFinish($pid,"Workspace reload detects job $pid is not running anymore"); - - unset($_SESSION['errorData']); - $job_in_err=0; - - //get tool info - $tool=getTool_fromId($job['toolId'],1); - if (! isset($tool['_id'])){ - $_SESSION['errorData']['Internal'][]="toolId '".$job['toolId']."' received from JobTool not registered"; - $_SESSION['errorData']['Error'][]="Cannot obtain results from '$title' in folder '".basename($job['working_dir'])."'. Job metadata is not valid."; - log_addOutregister($pid,"Failed to register $pid job outfiles. Job metadata has toolId '".$job['toolId']."'"); - $job_in_err=1; - continue; - } - if ($debug){ - print "
    \nBuilding outsput from toolINFO + stageout_file + stageout_data.\n
    STAGEOUT_DATA.
    "; - var_dump($job['stageout_data']); - print "
    \nSTAGEOUT_FILE.
    "; - var_dump($job['stageout_file']); - } - - // build output list merging: stageout_file + stageout_data + tool defintion data - $outs_files = build_outputs_list($tool,$job['stageout_data'],$job['stageout_file']); - if (count($outs_files)==0){ - log_addOutregister($pid,"Failed to register $pid job outfiles. Output file list empty."); - $job_in_err = 1; - } - if ($debug){ - print "
    \nList of output to register merged.\n
    STAGEOUT_DATA.
    "; - var_dump($outs_files); - print "
    \n"; - } - // checking each expected job output - foreach ($outs_files as $out_name => $outs_data){ - if ($debug){ - print "
    --------------------------------------------------------------
    "; - print "
    REGISTERING output_file with KEY NAME = $out_name DATA =
    \n"; - var_dump($outs_data); - } - // evaluate output_file requirement - $out_def = $tool['output_files'][$out_name]; - $is_required = output_is_required($out_def); - $allow_multiple = output_allow_multiple($out_def); - - //check requirement : allow multiple - if ($allow_multiple === false){ - if (count($outs_data)>1){ - $_SESSION['errorData']['Error'][]="Tool definition does not allow multiple instances for '$out_name', but the execution returned ".count($outs_data).". Registering only one of them."; - } - $outs_data = Array($outs_data[0]); - } - - // start - foreach ($outs_data as $out_data){ - if ($debug){ - print "
    START OUTPUT ITEM REGISTRATION FOR THE FOLLOWING OUT_DATA:
    \n"; - var_dump($out_data); - print "<\br>_____________\n"; - var_dump($out_data['file_path']); - print "<\br>_____________\n"; - var_dump($out_data['meta_data']); - print "<\br>_____________\n"; - } - - //check requirement : required - if (!isset($out_data['file_path'])){ - if ($is_required){ - $_SESSION['errorData']['Error'][]="Job output file ($out_name) not created"; - log_addOutregister($pid,"Job output file ($out_name) not created. No 'file_path' found. Job finished without creating 'out_metadata'?"); - $job_in_err=1; - } - if ($debug){ - print "
    file_path NO SET, but not required. Continuing. The merged metadata is:
    \n"; - var_dump($out_data); + // TODO: PMES will redirect log info to log_file. Now, info extracted from $jobProcess + updateLogFromJobInfo($job['log_file'], $pid, $job['launcher']); + + // + //set as running job + // + $title = (isset($job['title']) ? $job['title'] : "Job " . $job['execution']); + $descrip = getJobDescription($job['description'], $jobProcess, $lastjobs); + + if (count($jobProcess)) { + + if ($debug) + print "RUNNING JOB"; + + //set dummy id + $dummyId = $job['pid'] . "_dummy"; + + //get dummy parentDir + if ($job['hasExecutionFolder']) { + // show job in execution dir + $parentDir = fromAbsPath_toPath($job['working_dir']); + } else { + // show job in output_dir (infered from stageout_data) + $parentDir = 0; + if ($job['stageout_data']) { + $output_file_1 = $job['stageout_data']['output_files'][0]; + if ($output_file_1 && $output_file_1['file_path']) { + $parentDir = fromAbsPath_toPath(dirname($output_file_1['file_path'])); + } } - continue; + if (!$parentDir) + $parentDir = $_SESSION['User']['id'] . "/" . $_SESSION['User']['activeProject'] . "/uploads"; } - // resolve virtual path to local absolute path - $rfn = resolvePath_toLocalAbsolutePath($out_data['file_path'], $job); + //set dummy file + $fileDummy = array( + '_id' => $dummyId, + 'pid' => $pid, + 'title' => $title, + 'mtime' => new MongoDB\BSON\UTCDateTime(strtotime($jobProcess['submission_time']) * 1000), + 'size' => "", + 'visible' => 1, + 'tool' => $job['toolId'], + 'project' => $job['project'], + 'parentDir' => getGSFileId_fromPath($parentDir), + 'description' => $descrip, + 'pending' => $jobProcess['state'], + 'submission_file' => fromAbsPath_toPath($job['submission_file']), + 'log_file' => fromAbsPath_toPath($job['log_file']), + 'stdout_file' => fromAbsPath_toPath($job['stdout_file']), + 'stderr_file' => fromAbsPath_toPath($job['stderr_file']), + 'job_type' => $job['job_type'] + ); + + if ($jobProcess['state'] == "RUNNING" && $job['job_type'] == "interactive") { + $fileDummy['pending'] = "ACTIVE SESSION"; + } - $outPath = fromAbsPath_toPath($rfn); - $fileId = getGSFileId_fromPath($outPath); - if ($debug) - print "PID = [$pid] file_path=".$out_data['file_path']." --> fn=$outPath rfn=$rfn . Has Id? $fileId
    \n"; - - - //convert stage out data into MuGFile - - //associated_files and associated_id/_master: convert to fileIds - $metaReferences=array(); - if (isset($out_data['meta_data']['associated_id']) || isset($out_data['meta_data']['associated_master']) ){ - $assoc = (isset($out_data['meta_data']['associated_id'])?$out_data['meta_data']['associated_id']:$out_data['meta_data']['associated_master']); - $assoc_rfn = resolvePath_toLocalAbsolutePath($assoc, $job); - $assoc_fn = fromAbsPath_toPath($assoc_rfn); - $assoc_id = getGSFileId_fromPath($assoc_fn); - if ($assoc_id =="0"){ - $out_data['meta_data']['associated_id']= $assoc; - }else{ - $metaReferences[$assoc_id] = "associated_id"; - $out_data['meta_data']['associated_id']= $assoc_id; - } - if (isset($out_data['meta_data']['associated_master']) ){unset($out_data['meta_data']['associated_master']);} - if ($debug){ - print "THIS META HAS FILE REFERENCES. Saving associated_id='$assoc_id' instead of the original '$assoc'. If no ID. adding to refs:
    \n"; - var_dump($metaReferences); - } + //list job in workspace + $filesPending[$dummyId] = $fileDummy; + + //update job state in mongo + $job['state'] = $jobProcess['state']; + $SGE_updated[$pid] = $job; + + // + // processing job non running anymore + // + } else { + log_addFinish($pid, "Workspace reload detects job $pid is not running anymore"); + + unset($_SESSION['errorData']); + $job_in_err = 0; + + //get tool info + $tool = getTool_fromId($job['toolId'], 1); + if (! isset($tool['_id'])) { + $_SESSION['errorData']['Internal'][] = "toolId '" . $job['toolId'] . "' received from JobTool not registered"; + $_SESSION['errorData']['Error'][] = "Cannot obtain results from '$title' in folder '" . basename($job['working_dir']) . "'. Job metadata is not valid."; + log_addOutregister($pid, "Failed to register $pid job outfiles. Job metadata has toolId '" . $job['toolId'] . "'"); + $job_in_err = 1; + continue; } - if (isset($out_data['meta_data']['associated_files']) ){ - $assocs=array(); - foreach ($out_data['meta_data']['associated_files'] as $assoc){ - $assoc_rfn = resolvePath_toLocalAbsolutePath($assoc, $job); - $assoc_fn = fromAbsPath_toPath($assoc_rfn); - $assoc_id = getGSFileId_fromPath($assoc_fn); - if ($assoc_id == "0"){ - array_push($assocs,$assoc); - }else{ - array_push($assocs,$assoc_id); - $metaReferences[$assoc_id] = "associated_files"; + if ($debug) { + print "
    \nBuilding outsput from toolINFO + stageout_file + stageout_data.\n
    STAGEOUT_DATA.
    "; + var_dump($job['stageout_data']); + print "
    \nSTAGEOUT_FILE.
    "; + var_dump($job['stageout_file']); + } + + // build output list merging: stageout_file + stageout_data + tool defintion data + $outs_files = build_outputs_list($tool, $job['stageout_data'], $job['stageout_file']); + if (count($outs_files) == 0) { + log_addOutregister($pid, "Failed to register $pid job outfiles. Output file list empty."); + $job_in_err = 1; + } + if ($debug) { + print "
    \nList of output to register merged.\n
    STAGEOUT_DATA.
    "; + var_dump($outs_files); + print "
    \n"; + } + // checking each expected job output + foreach ($outs_files as $out_name => $outs_data) { + if ($debug) { + print "
    --------------------------------------------------------------
    "; + print "
    REGISTERING output_file with KEY NAME = $out_name DATA =
    \n"; + var_dump($outs_data); } - if ($debug){ - print "THIS META HAS FILE REFERENCES. Saving associated_files='$assoc_id' instead of the original '$assoc'. If no ID. adding to refs:
    \n"; - var_dump($metaReferences); + // evaluate output_file requirement + $out_def = $tool['output_files'][$out_name]; + $is_required = output_is_required($out_def); + $allow_multiple = output_allow_multiple($out_def); + + //check requirement : allow multiple + if ($allow_multiple === false) { + if (count($outs_data) > 1) { + $_SESSION['errorData']['Error'][] = "Tool definition does not allow multiple instances for '$out_name', but the execution returned " . count($outs_data) . ". Registering only one of them."; + } + $outs_data = array($outs_data[0]); } - } - $out_data['meta_data']['associated_files']= $assocs; - } - //sources : convert to fileIds and rename to sources - if (isset($out_data['sources'])){ - $sources=array(); - foreach ($out_data['sources'] as $source_path){ - $source_rfn = resolvePath_toLocalAbsolutePath($source_path, $job); - $source_fn = fromAbsPath_toPath($source_rfn); - $sourceid = getGSFileId_fromPath($source_fn); - - array_push($sources,$sourceid); - - if ($debug){ - print "SOURCES ORI = $source_path RFN = $source_rfn FN = $source_fn ID = $sourceid
    "; + // start + foreach ($outs_data as $out_data) { + if ($debug) { + print "
    START OUTPUT ITEM REGISTRATION FOR THE FOLLOWING OUT_DATA:
    \n"; + var_dump($out_data); + print "<\br>_____________\n"; + var_dump($out_data['file_path']); + print "<\br>_____________\n"; + var_dump($out_data['meta_data']); + print "<\br>_____________\n"; + } + + //check requirement : required + if (!isset($out_data['file_path'])) { + if ($is_required) { + $_SESSION['errorData']['Error'][] = "Job output file ($out_name) not created"; + log_addOutregister($pid, "Job output file ($out_name) not created. No 'file_path' found. Job finished without creating 'out_metadata'?"); + $job_in_err = 1; + } + if ($debug) { + print "
    file_path NO SET, but not required. Continuing. The merged metadata is:
    \n"; + var_dump($out_data); + } + continue; } - } - $out_data['sources'] = $sources; - } - - - //validate new file data - list($out_validation_score,$out_mug) = validateMugFile($out_data,true); - if ($debug){ - print "
    Out file model validation returned VALIDATION_SCORE= $out_validation_score and MuG file has ".count($out_mug)." fields
    \n"; - } + // resolve virtual path to local absolute path + $rfn = resolvePath_toLocalAbsolutePath($out_data['file_path'], $job); - // job successfully finished and already in mongo. Update medatada - - if ($fileId){ - if ($debug){ - print "Already in mongo. Adding metadata if there is any
    "; - } - logger("JOB $pid FINISHED SUCCESSFULLY"); - log_addOutregister($pid,"Updating only outfile $out_name '$rfn' metadata from job $pid"); - - if ($out_mug){ - //save metadata - list($out_vre,$metadata) = getVREfile_fromFile($out_mug); - if ($out_validation_score == 1 || $out_validation_score == 0){ - $metadata['validated'] = false; + $outPath = fromAbsPath_toPath($rfn); + $fileId = getGSFileId_fromPath($outPath); + if ($debug) + print "PID = [$pid] file_path=" . $out_data['file_path'] . " --> fn=$outPath rfn=$rfn . Has Id? $fileId
    \n"; + + + //convert stage out data into MuGFile + + //associated_files and associated_id/_master: convert to fileIds + $metaReferences = array(); + if (isset($out_data['meta_data']['associated_id']) || isset($out_data['meta_data']['associated_master'])) { + $assoc = (isset($out_data['meta_data']['associated_id']) ? $out_data['meta_data']['associated_id'] : $out_data['meta_data']['associated_master']); + $assoc_rfn = resolvePath_toLocalAbsolutePath($assoc, $job); + $assoc_fn = fromAbsPath_toPath($assoc_rfn); + $assoc_id = getGSFileId_fromPath($assoc_fn); + if ($assoc_id == "0") { + $out_data['meta_data']['associated_id'] = $assoc; + } else { + $metaReferences[$assoc_id] = "associated_id"; + $out_data['meta_data']['associated_id'] = $assoc_id; + } + if (isset($out_data['meta_data']['associated_master'])) { + unset($out_data['meta_data']['associated_master']); + } + if ($debug) { + print "THIS META HAS FILE REFERENCES. Saving associated_id='$assoc_id' instead of the original '$assoc'. If no ID. adding to refs:
    \n"; + var_dump($metaReferences); + } } - if ($debug){ - print "
    VRE METADATA SEND TO addMetadata IS:
    \n"; - var_dump($metadata); + if (isset($out_data['meta_data']['associated_files'])) { + $assocs = array(); + foreach ($out_data['meta_data']['associated_files'] as $assoc) { + $assoc_rfn = resolvePath_toLocalAbsolutePath($assoc, $job); + $assoc_fn = fromAbsPath_toPath($assoc_rfn); + $assoc_id = getGSFileId_fromPath($assoc_fn); + if ($assoc_id == "0") { + array_push($assocs, $assoc); + } else { + array_push($assocs, $assoc_id); + $metaReferences[$assoc_id] = "associated_files"; + } + if ($debug) { + print "THIS META HAS FILE REFERENCES. Saving associated_files='$assoc_id' instead of the original '$assoc'. If no ID. adding to refs:
    \n"; + var_dump($metaReferences); + } + } + $out_data['meta_data']['associated_files'] = $assocs; } - $ok = addMetadataBNS($fileId,$metadata); - if ($ok == "0") - $_SESSION['errorData']['Warning'][] = "Sorry, could update '".basename($rfn)."' metadata."; - } - // job successfully finished but not yet on mongo. Save output + //sources : convert to fileIds and rename to sources + if (isset($out_data['sources'])) { + $sources = array(); + foreach ($out_data['sources'] as $source_path) { + $source_rfn = resolvePath_toLocalAbsolutePath($source_path, $job); + $source_fn = fromAbsPath_toPath($source_rfn); + $sourceid = getGSFileId_fromPath($source_fn); - }elseif ((is_file($rfn)|| is_dir($rfn)) && $out_mug && $out_validation_score!=0) { - if ($debug){ - print "
    JOB $pid FINISHED AND NOT YET IN MONGO ($outPath). Saving!
    \n"; - } + array_push($sources, $sourceid); - //register file and save updated metadata - list($out_vre,$metadata) = getVREfile_fromFile($out_mug); - if ($debug){ - print "
    VRE METADATA SEND TO saveResults IS:
    \n"; - var_dump($metadata); - } - if ($out_validation_score == 1 || $out_validation_score == 0){ - $metadata['validated'] =false; - } - - if ($debug){ - print "
    VRE METADATA SEND TO saveResults IS:
    \n"; - var_dump($metadata); - } - $fileInfo = saveResults($outPath,$metadata,$job); - - //list new metadata in table - if (is_array($fileInfo)){ - $fileId = $fileInfo['_id']; - if(!$metadata['visible'] === false) - $filesPending[$fileId]=$fileInfo; - if ($debug) - print "\n

    SAVED successfully ".$fileInfo['_id']."!
    \n"; - } - if($fileInfo == "0"){ - $_SESSION['errorData']['Error'][]="Job output file (".basename($rfn).") generated, but with wrong metadata. Please, contact us reporting this error [Job id $pid]"; - log_addOutregister($pid,"Failed to register outfile $out_name '$rfn'. File found but not correctly registered"); - $job_in_err = 1; - }else{ - log_addOutregister($pid,"Outfile $out_name '$rfn' sucessfully registered from job $pid"); - logger("JOB $pid FINISHED SUCCESSFULLY"); - } - - // job successfully finished but file metada not valid. Setting error mode - - }elseif ((is_file($rfn)||is_dir($rfn)) && !$out_mug) { - if ($debug){ - print "
    JOB $pid FINISHED BUT INVALID FILE ($outPath). SET ERROR
    "; - print "

    The invalid file is:
    \n"; - var_dump($out_mug); - } - $_SESSION['errorData']['Error'][]="Job output file '$out_name' generated (".basename($rfn)."), but not registered. Missing required metadata."; - log_addOutregister($pid,"Failed to register outfile $out_name '$rfn'. File found but missing required metadata. The invalid file is: ".json_encode($out_mug)); - $job_in_err=1; + if ($debug) { + print "SOURCES ORI = $source_path RFN = $source_rfn FN = $source_fn ID = $sourceid
    "; + } + } + $out_data['sources'] = $sources; + } - }else{ - $_SESSION['errorData']['Error'][]="Job output outfile ($out_name) generated (".basename($rfn).")."; - log_addOutregister($pid,"Failed to register outfile $out_name '$rfn'. File NOT found in disk"); - if ($debug){ - print "
    JOB $pid FINISHED BUT NO EXPECTED OUTFILE '$rfn' FOUND IN DISK. Set ERROR
    "; - } - $job_in_err=1; - } - - // Update metadata of other files referring current fileId (associated files) - if ($job_in_err==0 && count($metaReferences)){ - if ($debug) - print "
    Update metadata of other FILE REFERENCES afecting current fileId
    \n"; - - foreach ($metaReferences as $assoc_id => $assoc_type){ - $file_assoc = getGSFile_fromId($assoc_id,"onlyMetadata"); - if ($assoc_type == "associated_files") - $file_assoc['associated_id'] = $fileId; - if ($assoc_type == "associated_id"){ - $assocs=array(); - foreach ($file_assoc['associated_files'] as $a){ - if (preg_match('/\//',$a)){ - array_push($assocs,$fileId); - }else{ - array_push($assocs,$a); + + //validate new file data + list($out_validation_score, $out_mug) = validateMugFile($out_data, true); + if ($debug) { + print "
    Out file model validation returned VALIDATION_SCORE= $out_validation_score and MuG file has " . count($out_mug) . " fields
    \n"; + } + + + // job successfully finished and already in mongo. Update medatada + + if ($fileId) { + if ($debug) { + print "Already in mongo. Adding metadata if there is any
    "; } - } - $file_assoc['associated_files']=$assocs; - } - $ok = addMetadataBNS($assoc_id,$file_assoc); - if ($ok == "0") - $_SESSION['errorData']['Warning'][] = "Sorry, could not add reference to '".basename($rfn)."' in the metadata of the associated file '$assoc_id'"; - $ff = getGSFile_fromId($assoc_id); - if ($debug){ - print "
    Updating metadata for assoc file $assoc_id done. This is:
    \n"; - var_dump($ff); + logger("JOB $pid FINISHED SUCCESSFULLY"); + log_addOutregister($pid, "Updating only outfile $out_name '$rfn' metadata from job $pid"); + + if ($out_mug) { + //save metadata + list($out_vre, $metadata) = getVREfile_fromFile($out_mug); + if ($out_validation_score == 1 || $out_validation_score == 0) { + $metadata['validated'] = false; + } + if ($debug) { + print "
    VRE METADATA SEND TO addMetadata IS:
    \n"; + var_dump($metadata); + } + $ok = addMetadataBNS($fileId, $metadata); + if ($ok == "0") + $_SESSION['errorData']['Warning'][] = "Sorry, could update '" . basename($rfn) . "' metadata."; + } + + // job successfully finished but not yet on mongo. Save output + + } elseif ((is_file($rfn) || is_dir($rfn)) && $out_mug && $out_validation_score != 0) { + if ($debug) { + print "
    JOB $pid FINISHED AND NOT YET IN MONGO ($outPath). Saving!
    \n"; + } + + //register file and save updated metadata + list($out_vre, $metadata) = getVREfile_fromFile($out_mug); + if ($debug) { + print "
    VRE METADATA SEND TO saveResults IS:
    \n"; + var_dump($metadata); + } + if ($out_validation_score == 1 || $out_validation_score == 0) { + $metadata['validated'] = false; + } + + if ($debug) { + print "
    VRE METADATA SEND TO saveResults IS:
    \n"; + var_dump($metadata); + } + $fileInfo = saveResults($outPath, $metadata, $job); + + //list new metadata in table + if (is_array($fileInfo)) { + $fileId = $fileInfo['_id']; + if (!$metadata['visible'] === false) + $filesPending[$fileId] = $fileInfo; + if ($debug) + print "\n

    SAVED successfully " . $fileInfo['_id'] . "!
    \n"; + } + if ($fileInfo == "0") { + $_SESSION['errorData']['Error'][] = "Job output file (" . basename($rfn) . ") generated, but with wrong metadata. Please, contact us reporting this error [Job id $pid]"; + log_addOutregister($pid, "Failed to register outfile $out_name '$rfn'. File found but not correctly registered"); + $job_in_err = 1; + } else { + log_addOutregister($pid, "Outfile $out_name '$rfn' sucessfully registered from job $pid"); + logger("JOB $pid FINISHED SUCCESSFULLY"); + } + + // job successfully finished but file metada not valid. Setting error mode + + } elseif ((is_file($rfn) || is_dir($rfn)) && !$out_mug) { + if ($debug) { + print "
    JOB $pid FINISHED BUT INVALID FILE ($outPath). SET ERROR
    "; + print "

    The invalid file is:
    \n"; + var_dump($out_mug); + } + $_SESSION['errorData']['Error'][] = "Job output file '$out_name' generated (" . basename($rfn) . "), but not registered. Missing required metadata."; + log_addOutregister($pid, "Failed to register outfile $out_name '$rfn'. File found but missing required metadata. The invalid file is: " . json_encode($out_mug)); + $job_in_err = 1; + } else { + $_SESSION['errorData']['Error'][] = "Job output outfile ($out_name) generated (" . basename($rfn) . ")."; + log_addOutregister($pid, "Failed to register outfile $out_name '$rfn'. File NOT found in disk"); + if ($debug) { + print "
    JOB $pid FINISHED BUT NO EXPECTED OUTFILE '$rfn' FOUND IN DISK. Set ERROR
    "; + } + $job_in_err = 1; + } + + // Update metadata of other files referring current fileId (associated files) + if ($job_in_err == 0 && count($metaReferences)) { + if ($debug) + print "
    Update metadata of other FILE REFERENCES afecting current fileId
    \n"; + + foreach ($metaReferences as $assoc_id => $assoc_type) { + $file_assoc = getGSFile_fromId($assoc_id, "onlyMetadata"); + if ($assoc_type == "associated_files") + $file_assoc['associated_id'] = $fileId; + if ($assoc_type == "associated_id") { + $assocs = array(); + foreach ($file_assoc['associated_files'] as $a) { + if (preg_match('/\//', $a)) { + array_push($assocs, $fileId); + } else { + array_push($assocs, $a); + } + } + $file_assoc['associated_files'] = $assocs; + } + $ok = addMetadataBNS($assoc_id, $file_assoc); + if ($ok == "0") + $_SESSION['errorData']['Warning'][] = "Sorry, could not add reference to '" . basename($rfn) . "' in the metadata of the associated file '$assoc_id'"; + $ff = getGSFile_fromId($assoc_id); + if ($debug) { + print "
    Updating metadata for assoc file $assoc_id done. This is:
    \n"; + var_dump($ff); + } + } + } } - } } - } - } - // OJO Uncomment only for debugging output_files registry - //$SGE_updated[$job['pid']]=$job; - //$job_in_err=0; + // OJO Uncomment only for debugging output_files registry + //$SGE_updated[$job['pid']]=$job; + //$job_in_err=0; - // jobs nor finished nor running: in error OR deleted OR SESSION[sge] not updated + // jobs nor finished nor running: in error OR deleted OR SESSION[sge] not updated - if ($debug) - print "
    IS JOB IN ERR? ($job_in_err)
    \n"; - - - if ($job_in_err){ - log_addOutregister($pid,"Failed to register all job outfiles",FALSE); - logger("JOB $pid FINISHED but with errors"); - $logFileP = $job['log_file']; - $logFile = fromAbsPath_toPath($job['log_file']); - - // force flash disk status - scandir($GLOBALS['dataDir'].$_SESSION['User']['id']."/".$job['project']); - - // job has log - if (is_file($logFileP) ){ - // move and redefine log and SH file if internalTool - if ($job['hasExecutionFolder'] === false ){ - // right now, redifinition done inside saveResults - } - if ($debug) - print "
    JOB IN ERROR $fileId storing LOG $logFile
    "; - - $logId = getGSFileId_fromPath($logFile); - if (!$logId){ - $logMeta['description'] = "Job log file"; - $logMeta['format'] = "ERR"; - $metaDataLog = prepMetadataLog($logMeta,$logFile); - $logInfo = saveResults($logFile,$metaDataLog,$job); - if (is_array($logInfo)) - $filesPending[$logInfo['_id']]=$logInfo; - } - - // job has neither log nor all outfiles - }else{ - if ($debug) - print "
    JOB $pid NO log (".$logFileP.") NO output ($outPath)
    "; - /* + if ($debug) + print "
    IS JOB IN ERR? ($job_in_err)
    \n"; + + + if ($job_in_err) { + log_addOutregister($pid, "Failed to register all job outfiles", FALSE); + logger("JOB $pid FINISHED but with errors"); + $logFileP = $job['log_file']; + $logFile = fromAbsPath_toPath($job['log_file']); + + // force flash disk status + scandir($GLOBALS['dataDir'] . $_SESSION['User']['id'] . "/" . $job['project']); + + // job has log + if (is_file($logFileP)) { + // move and redefine log and SH file if internalTool + if ($job['hasExecutionFolder'] === false) { + // right now, redifinition done inside saveResults + } + if ($debug) + print "
    JOB IN ERROR $fileId storing LOG $logFile
    "; + + $logId = getGSFileId_fromPath($logFile); + if (!$logId) { + $logMeta['description'] = "Job log file"; + $logMeta['format'] = "ERR"; + $metaDataLog = prepMetadataLog($logMeta, $logFile); + $logInfo = saveResults($logFile, $metaDataLog, $job); + if (is_array($logInfo)) + $filesPending[$logInfo['_id']] = $logInfo; + } + + // job has neither log nor all outfiles + } else { + if ($debug) + print "
    JOB $pid NO log (" . $logFileP . ") NO output ($outPath)
    "; + /* $proj = $GLOBALS['dataDir']."/".dirname($outPath); if (is_dir($proj)){ $projContent = glob($proj.'/*.e[0-9]*', GLOB_BRACE); @@ -1584,28 +1595,28 @@ function processPendingFiles($sessionId,$files=array()){ $_SESSION['errorData']['Error'][]="Execution ".$job['title']." '".basename($outPath)."' failed with neither log nor error file."; } */ - } - }else{ - log_addOutregister($pid,"Output files successfully registed",TRUE); - } + } + } else { + log_addOutregister($pid, "Output files successfully registed", TRUE); + } + } + } - } - } - - if ($debug){ - print "

    FINAL FILES PENDING yes? Num=
    \n"; - var_dump(count($filesPending)); - print "

    \n"; - } - - //update session and save to mongo - saveUserJobs($sessionId,$SGE_updated); - return $filesPending; + if ($debug) { + print "

    FINAL FILES PENDING yes? Num=
    \n"; + var_dump(count($filesPending)); + print "

    \n"; + } + + //update session and save to mongo + saveUserJobs($sessionId, $SGE_updated); + return $filesPending; } -function saveResults($filePath,$metaData=array(),$job=array(),$rfn=0,$asRoot=0){ +function saveResults($filePath, $metaData = array(), $job = array(), $rfn = 0, $asRoot = 0) +{ // NOT saving internal or temporal files //if (in_array($ext,$GLOBALS['internalResults']) || preg_match('/^\./',basename($filePath)) ){ @@ -1614,221 +1625,230 @@ function saveResults($filePath,$metaData=array(),$job=array(),$rfn=0,$asRoot=0){ // check given filePath if ($rfn == 0) - $rfn = $GLOBALS['dataDir']."/".$filePath; - if (preg_match('/^\//',$filePath)){ + $rfn = $GLOBALS['dataDir'] . "/" . $filePath; + if (preg_match('/^\//', $filePath)) { $rfn = $filePath; - $filePath = str_replace($GLOBALS['dataDir']."/","",$rfn); + $filePath = str_replace($GLOBALS['dataDir'] . "/", "", $rfn); } - if ((!is_file($rfn) && !is_dir($rfn)) || (is_file($rfn) && !filesize($rfn)) ){ - if (!is_dir($rfn)){ - $_SESSION['errorData']['Error'][]="Execution result '$rfn' does not exist or has size 0. Cannot save it into database"; + if ((!is_file($rfn) && !is_dir($rfn)) || (is_file($rfn) && !filesize($rfn))) { + if (!is_dir($rfn)) { + $_SESSION['errorData']['Error'][] = "Execution result '$rfn' does not exist or has size 0. Cannot save it into database"; return 0; } } # prepare file metaData - $metaData = prepMetadataResult($metaData,$filePath,$job); - - # prepare Parent - $parentPath = dirname($filePath); - $parentId = getGSFileId_fromPath($parentPath,$asRoot); - if (!$parentId){ - if (isset($job['hasExecutionFolder']) && $job['hasExecutionFolder']===false){ - $parentPath = fromAbsPath_toPath($job['output_dir']); - $parentId = getGSFileId_fromPath($parentPath,$asRoot); - } - if (!$parentId){ - $_SESSION['errorData']['Error'][]="Cannot save result '".basename($filePath)."' at '$parentPath'. This parent directory does not exist or is unaccessible"; - return 0; + $metaData = prepMetadataResult($metaData, $filePath, $job); + + # prepare Parent + $parentPath = dirname($filePath); + $parentId = getGSFileId_fromPath($parentPath, $asRoot); + if (!$parentId) { + if (isset($job['hasExecutionFolder']) && $job['hasExecutionFolder'] === false) { + $parentPath = fromAbsPath_toPath($job['output_dir']); + $parentId = getGSFileId_fromPath($parentPath, $asRoot); + } + if (!$parentId) { + $_SESSION['errorData']['Error'][] = "Cannot save result '" . basename($filePath) . "' at '$parentPath'. This parent directory does not exist or is unaccessible"; + return 0; + } } - } #save Data $fileId = createLabel(); - $insert_type = (isset($metaData['type'])? $metaData['type'] : (is_dir($rfn)?"dir":"file") ); - $size = ($insert_type == "dir" ? getDirectorySize($rfn): filesize($rfn)); - $child_files = (isset($metaData['files'])? $metaData['fields'] : (is_dir($rfn)? array(): false ) ); + $insert_type = (isset($metaData['type']) ? $metaData['type'] : (is_dir($rfn) ? "dir" : "file")); + $size = ($insert_type == "dir" ? getDirectorySize($rfn) : filesize($rfn)); + $child_files = (isset($metaData['files']) ? $metaData['fields'] : (is_dir($rfn) ? array() : false)); - $insertData=array( + $insertData = array( '_id' => $fileId, 'type' => $insert_type, 'owner' => $_SESSION['User']['id'], 'size' => $size, 'path' => $filePath, - 'project'=> $job['project'], - 'mtime' => new MongoDB\BSON\UTCDateTime(filemtime($rfn)*1000), + 'project' => $job['project'], + 'mtime' => new MongoDB\BSON\UTCDateTime(filemtime($rfn) * 1000), 'parentDir' => $parentId ); - if ($child_files !== false){ $insertData['files'] = $child_files;} + if ($child_files !== false) { + $insertData['files'] = $child_files; + } #save to MONGO - $fnId = uploadGSFileBNS($filePath, $rfn, $insertData,$metaData, FALSE,$asRoot); + $fnId = uploadGSFileBNS($filePath, $rfn, $insertData, $metaData, FALSE, $asRoot); - if ($fnId){ + if ($fnId) { $insertData['mtime'] = $insertData['mtime']->toDateTime()->format('U'); - return array_merge($insertData,$metaData); - }else{ - $_SESSION['errorData']['mongoDB'][]="Cannot save execution result 'basename($filePath)' into database. Stored only on disk"; + return array_merge($insertData, $metaData); + } else { + $_SESSION['errorData']['mongoDB'][] = "Cannot save execution result 'basename($filePath)' into database. Stored only on disk"; return 0; } } -function build_outputs_list($tool,$stageout_job,$stageout_file){ +function build_outputs_list($tool, $stageout_job, $stageout_file) +{ - $outs_meta= Array(); + $outs_meta = array(); // check tool output_files - if (!isset($tool['output_files']) || count($tool['output_files'])==0){ - $_SESSION['errorData']['Internal'][]="Tool ".$tool['name']." has not list of 'output_files'. Invalid tool registration"; - $_SESSION['errorData']['Error'][]="Cannot obtain results from execution '".dirname($stageout_file)."'"; + if (!isset($tool['output_files']) || count($tool['output_files']) == 0) { + $_SESSION['errorData']['Internal'][] = "Tool " . $tool['name'] . " has not list of 'output_files'. Invalid tool registration"; + $_SESSION['errorData']['Error'][] = "Cannot obtain results from execution '" . dirname($stageout_file) . "'"; return $outs_meta; } // parse stageout file - $stageout_meta = Array(); - if (isset($stageout_file) && is_file($stageout_file) ){ - $content = file_get_contents($stageout_file); + $stageout_meta = array(); + if (isset($stageout_file) && is_file($stageout_file)) { + $content = file_get_contents($stageout_file); $data = json_decode($content, true); - if (count($data) == 0 || count($data['output_files'])==0){ - $_SESSION['errorData']['Warning'][]="Tool stageout file '".basename($stageout_file)."' is empty or bad formatted"; + if (count($data) == 0 || count($data['output_files']) == 0) { + $_SESSION['errorData']['Warning'][] = "Tool stageout file '" . basename($stageout_file) . "' is empty or bad formatted"; } //index by name - foreach ($data['output_files'] as $out){ - if (isset($out['name'])){ + foreach ($data['output_files'] as $out) { + if (isset($out['name'])) { if (!isset($stageout_meta[$out['name']])) - $stageout_meta[$out['name']]=Array(); - array_push($stageout_meta[$out['name']],$out); - }else{ - $_SESSION['errorData']['Warning'][]="Tool stageout file '".basename($stageout_file)."' is bad formatted. Missing 'name' in 'output_files' list"; + $stageout_meta[$out['name']] = array(); + array_push($stageout_meta[$out['name']], $out); + } else { + $_SESSION['errorData']['Warning'][] = "Tool stageout file '" . basename($stageout_file) . "' is bad formatted. Missing 'name' in 'output_files' list"; continue; } } - }else{ - $_SESSION['errorData']['Warning'][]=date("h:i:s").": Tool stageout file '".$stageout_file."' is not found"; + } elseif ($tool['external'] !== false) { + $_SESSION['errorData']['Warning'][] = date("h:i:s") . ": Tool stageout file '" . $stageout_file . "' is not found"; } - print "\n__________FROM FILE________________\n"; - $json_string = json_encode($stageout_meta, JSON_PRETTY_PRINT); + print "\n__________FROM FILE________________\n"; + $json_string = json_encode($stageout_meta, JSON_PRETTY_PRINT); // check stageout data - $stageout_data = Array(); - if ($stageout_job){ - if (!isset($stageout_job['output_files'])){ - $stageout_data=array(); - }else{ - foreach ($stageout_job['output_files'] as $out){ - if (isset($out['name'])){ - if (!isset($stageout_data[$out['name']])) - $stageout_data[$out['name']]=Array(); - array_push($stageout_data[$out['name']],$out); - }else{ - $_SESSION['errorData']['Warning'][]="Tool job has stageout data is bad formatted. Missing 'name' in 'output_files' list"; - continue; + $stageout_data = array(); + if ($stageout_job) { + if (!isset($stageout_job['output_files'])) { + $stageout_data = array(); + } else { + foreach ($stageout_job['output_files'] as $out) { + if (isset($out['name'])) { + if (!isset($stageout_data[$out['name']])) + $stageout_data[$out['name']] = array(); + array_push($stageout_data[$out['name']], $out); + } else { + $_SESSION['errorData']['Warning'][] = "Tool job has stageout data is bad formatted. Missing 'name' in 'output_files' list"; + continue; + } } - } } } - print "\n__________FROM JOB________________\n"; - $json_string = json_encode($stageout_data, JSON_PRETTY_PRINT); + print "\n__________FROM JOB________________\n"; + $json_string = json_encode($stageout_data, JSON_PRETTY_PRINT); // merging stageout file and stageout data - $stageout_meta = array_merge($stageout_data,$stageout_meta); - - print "\n__________FROM FILE + JOB________________\n"; - $json_string = json_encode($stageout_meta, JSON_PRETTY_PRINT); + $stageout_meta = array_merge($stageout_data, $stageout_meta); + + print "\n__________FROM FILE + JOB________________\n"; + $json_string = json_encode($stageout_meta, JSON_PRETTY_PRINT); // merging file data from tool and stageout_file - foreach ($tool['output_files'] as $out_name => $out_data){ - $outs_meta[$out_name] = Array(); - if (!isset($out_data['file'])){ - $out_data['file']=Array(); - print "Tool has no file attribute for output_file '$out_name'"; + foreach ($tool['output_files'] as $out_name => $out_data) { + $outs_meta[$out_name] = array(); + if (!isset($out_data['file'])) { + $out_data['file'] = array(); + print "Tool has no file attribute for output_file '$out_name'"; } - if (!isset($stageout_meta[$out_name])){ - print "Tool stageout file/data has no metadata for output_file '$out_name'."; - array_push($outs_meta[$out_name],$out_data); + if (!isset($stageout_meta[$out_name])) { + print "Tool stageout file/data has no metadata for output_file '$out_name'."; + array_push($outs_meta[$out_name], $out_data); continue; } - foreach ($stageout_meta[$out_name] as $stg_data){ + foreach ($stageout_meta[$out_name] as $stg_data) { //create merged file data if (isset($out_data['file']['input_files'])) unset($out_data['file']['input_files']); if (isset($stg_data['name'])) unset($stg_data['name']); - $file_merged = array_merge_recursive_distinct($out_data['file'],$stg_data); + $file_merged = array_merge_recursive_distinct($out_data['file'], $stg_data); - array_push($outs_meta[$out_name],$file_merged); + array_push($outs_meta[$out_name], $file_merged); } } return $outs_meta; } -function topDir() { +function topDir() +{ return ($_SESSION['curDir'] == $_SESSION['userId']); } -function upDir() { +function upDir() +{ if (!topDir()) $_SESSION['curDir'] = dirname($_SESSION['curDir']); } -function downDir($fn) { +function downDir($fn) +{ $fnData = $GLOBALS['filesCol']->findOne(array('_id' => $fn)); if (! empty($fnData)) { - if (isset($fnData['type']) && $fnData['type'] == "dir"){ - $_SESSION['curDir'] = $fn; - }else{ - $_SESSION['errorData'][error][]="Cannot change directory. $fn is not a directory "; - } + if (isset($fnData['type']) && $fnData['type'] == "dir") { + $_SESSION['curDir'] = $fn; + } else { + $_SESSION['errorData'][error][] = "Cannot change directory. $fn is not a directory "; + } } } // return sum of FS or Mongo directory (in bytes) -function getUsedDiskSpace($userId = '', $source = "fs") { - if (!$userId) { +function getUsedDiskSpace($userId = '', $source = "fs") +{ + if (!$userId) { $userId = $_SESSION['User']['id']; } - if ($source != "fs") { - if (!preg_match('/^\//', $userId)) { - $userId = $GLOBALS['dataDir']."/".$userId; + if ($source != "fs") { + if (!preg_match('/^\//', $userId)) { + $userId = $GLOBALS['dataDir'] . "/" . $userId; } - $data = explode("\t", exec("du -sb $userId")); - return $data[0]; - } + $data = explode("\t", exec("du -sb $userId")); + return $data[0]; + } return calcGSUsedSpace($userId); } // return sum of FS directory (in bytes) -function getDirectorySize($fn) { - if (!$fn) - return 0; - if (!preg_match('/^\//',$fn) ) - $fn = $GLOBALS['dataDir']."/".$fn; +function getDirectorySize($fn) +{ + if (!$fn) + return 0; + if (!preg_match('/^\//', $fn)) + $fn = $GLOBALS['dataDir'] . "/" . $fn; - $data = explode("\t", exec("du -sb $fn")); - return $data[0]; + $data = explode("\t", exec("du -sb $fn")); + return $data[0]; } // return user diskquota from mongo -function getDiskLimit($login = '') { - if (!$login){ +function getDiskLimit($login = '') +{ + if (!$login) { $login = $_SESSION['User']['_id']; } $sp = getUser_diskQuota($login); - if ($sp === false){ + if ($sp === false) { return $GLOBALS['DISKLIMIT']; - }else{ + } else { return $sp; } } @@ -1861,168 +1881,166 @@ function navigation() { } */ -function formatSize($bytes) { +function formatSize($bytes) +{ $types = array('B', 'KB', 'MB', 'GB', 'TB'); - for ($i = 0; $bytes >= 1024 && $i < ( count($types) - 1 ); $bytes /= 1024, $i++); - return( round($bytes, 2) . "" . $types[$i] ); + for ($i = 0; $bytes >= 1024 && $i < (count($types) - 1); $bytes /= 1024, $i++); + return (round($bytes, 2) . "" . $types[$i]); } -function downloadFile( $rfn ){ - $fileInfo = pathinfo($rfn); - $filename = $fileInfo['basename']; - $fileExtension = $fileInfo['extension']; - $fileExtension = preg_replace('/_\d+$/',"",$fileExtension); - $content_type = (array_key_exists($fileExtension, mimeTypes()) ? mimeTypes()[$fileExtension] : "application/octet-stream"); - $size = filesize($rfn); - $offset = 0; - $length = $size; - - if(isset($_SERVER['HTTP_RANGE'])){ - preg_match('/bytes=(\d+)-(\d+)?/', $_SERVER['HTTP_RANGE'], $matches); - $offset = intval($matches[1]); - $length = intval($matches[2]) - $offset; - - $fhandle = fopen($rfn, 'r'); - fseek($fhandle, $offset); // seek to the requested offset, this is 0 if it's not a partial content request - $data = fread($fhandle, $length); - fclose($fhandle); - - header('HTTP/1.1 206 Partial Content'); - header('Content-Range: bytes ' . $offset . '-' . ($offset + $length) . '/' . $size); - } - header("Content-Disposition: attachment;filename=".$filename); - header('Content-Type: '.$content_type); - header("Accept-Ranges: bytes"); - header("Pragma: public"); - header("Expires: -1"); - header("Cache-Control: no-cache"); - header("Cache-Control: public, must-revalidate, post-check=0, pre-check=0"); - header("Content-Length: ".filesize($rfn)); - $chunksize = 8 * (1024 * 1024); //8MB (highest possible fread length) - - if ($size > $chunksize){ - $handle = fopen($rfn,'rb'); - $buffer = ''; - while (!feof($handle) && (connection_status() === CONNECTION_NORMAL)) { - $buffer = fread($handle, $chunksize); - print $buffer; - ob_flush(); - flush(); - } - if(connection_status() !== CONNECTION_NORMAL) { - echo "Connection aborted"; - } - fclose($handle); - }else{ - ob_clean(); +function downloadFile($rfn) +{ + $fileInfo = pathinfo($rfn); + $filename = $fileInfo['basename']; + $fileExtension = $fileInfo['extension']; + $fileExtension = preg_replace('/_\d+$/', "", $fileExtension); + $content_type = (array_key_exists($fileExtension, mimeTypes()) ? mimeTypes()[$fileExtension] : "application/octet-stream"); + $size = filesize($rfn); + $offset = 0; + $length = $size; + + if (isset($_SERVER['HTTP_RANGE'])) { + preg_match('/bytes=(\d+)-(\d+)?/', $_SERVER['HTTP_RANGE'], $matches); + $offset = intval($matches[1]); + $length = intval($matches[2]) - $offset; + + $fhandle = fopen($rfn, 'r'); + fseek($fhandle, $offset); // seek to the requested offset, this is 0 if it's not a partial content request + $data = fread($fhandle, $length); + fclose($fhandle); + + header('HTTP/1.1 206 Partial Content'); + header('Content-Range: bytes ' . $offset . '-' . ($offset + $length) . '/' . $size); + } + header("Content-Disposition: attachment;filename=" . $filename); + header('Content-Type: ' . $content_type); + header("Accept-Ranges: bytes"); + header("Pragma: public"); + header("Expires: -1"); + header("Cache-Control: no-cache"); + header("Cache-Control: public, must-revalidate, post-check=0, pre-check=0"); + header("Content-Length: " . filesize($rfn)); + $chunksize = 8 * (1024 * 1024); //8MB (highest possible fread length) + + if ($size > $chunksize) { + $handle = fopen($rfn, 'rb'); + $buffer = ''; + while (!feof($handle) && (connection_status() === CONNECTION_NORMAL)) { + $buffer = fread($handle, $chunksize); + print $buffer; + ob_flush(); flush(); - readfile($rfn); } - exit(0); + if (connection_status() !== CONNECTION_NORMAL) { + echo "Connection aborted"; + } + fclose($handle); + } else { + ob_clean(); + flush(); + readfile($rfn); + } + exit(0); } -function refresh_token($force=false){ - - if (!$_SESSION['User']['Token']['access_token']){ - ob_clean(); - header('Location: '.$GLOBALS['BASEURL'].'/htmlib/errordb.php?msg=MuG Authentification Session Expired. Login again'); - } - $existingTokenO = new AccessToken($_SESSION['User']['Token']); +function refresh_token($force = false) +{ - //$provider = new MuG_Oauth2Provider\MuG_Oauth2Provider(['redirectUri'=> 'http://' . $_SERVER['SERVER_NAME'] . $_SERVER['PHP_SELF']]); - $provider = new MuG_Oauth2Provider\MuG_Oauth2Provider(['redirectUri'=> $GLOBALS['URL'] . $_SERVER['PHP_SELF']]); - - if ($force || $existingTokenO->hasExpired()) { - try { - $newTokenO = $provider->getAccessToken('refresh_token', [ 'refresh_token' => $existingTokenO->getRefreshToken()]); - if ($newTokenO->getToken()){ - $newToken = json_decode(json_encode($newTokenO),true); - } - } - catch (Exception $e){ - $_SESSION['errorData']['Error'][] = "Cannot validate token from refresh token."; - $_SESSION['errorData']['Error'][] = $e->getMessage(); - return false; + if (!$_SESSION['User']['Token']['access_token']) { + ob_clean(); + header('Location: ' . $GLOBALS['BASEURL'] . '/htmlib/errordb.php?msg=MuG Authentification Session Expired. Login again'); } + $existingTokenO = new AccessToken($_SESSION['User']['Token']); - // save in mongo - $user = $_SESSION['User']; - $user['Token'] = $newToken; - updateUser($user); - - // load new token in session - $_SESSION['User']['Token'] = $newToken; - return true; - - }else{ - $_SESSION['errorData']['Warning'][] = "Access token not expired yet. Force refresh"; - return false; - } + //$provider = new MuG_Oauth2Provider\MuG_Oauth2Provider(['redirectUri'=> 'http://' . $_SERVER['SERVER_NAME'] . $_SERVER['PHP_SELF']]); + $provider = new MuG_Oauth2Provider\MuG_Oauth2Provider(['redirectUri' => $GLOBALS['URL'] . $_SERVER['PHP_SELF']]); -} - - - -function refresh_vault_token($force=false){ + if ($force || $existingTokenO->hasExpired()) { + try { + $newTokenO = $provider->getAccessToken('refresh_token', ['refresh_token' => $existingTokenO->getRefreshToken()]); + if ($newTokenO->getToken()) { + $newToken = json_decode(json_encode($newTokenO), true); + } + } catch (Exception $e) { + $_SESSION['errorData']['Error'][] = "Cannot validate token from refresh token."; + $_SESSION['errorData']['Error'][] = $e->getMessage(); + return false; + } - if (!$_SESSION['User']['Vault']['vaultKey']){ - ob_clean(); - header('Location: '.$GLOBALS['BASEURL'].'/htmlib/errordb.php?msg=Vault Authentification Session Expired. Login again'); - } + // save in mongo + $user = $_SESSION['User']; + $user['Token'] = $newToken; + updateUser($user); - //echo "refresh_vault_token"; - //echo $_SESSION['User']['Vault']['vaultKey']; + // load new token in session + $_SESSION['User']['Token'] = $newToken; + return true; + } else { + $_SESSION['errorData']['Warning'][] = "Access token not expired yet. Force refresh"; + return false; + } +} - $existingToken = $_SESSION['User']['Vault']['vaultKey']; - //$provider = new MuG_Oauth2Provider\MuG_Oauth2Provider(['redirectUri'=> 'http://' . $_SERVER['SERVER_NAME'] . $_SERVER['PHP_SELF']]); - //$provider = new MuG_Oauth2Provider\MuG_Oauth2Provider(['redirectUri'=> $GLOBALS['URL'] . $_SERVER['PHP_SELF']]); - //add them to mongo - $provider = new VaultClient($_SESSION['User']['Vault']['vaultUrl'], $_SESSION['User']['Vault']['vaultToken'], $_SESSION['User']['Token']['access_token'], $_SESSION['User']['Vault']['vaultRole'], $_POST['username']); +function refresh_vault_token($force = false) +{ - $expirationTime = $provider->getTokenExpirationTime($_SESSION['User']['Vault']['vaultUrl'], $existingToken); - //echo "refresh_vault_token_1"; - //echo $expirationTime; - - //expiration time can be or the timestamp or false: - //if ($force || $provider->isTokenExpired($_SESSION['User']['Vault']['vaultUrl'], $existingToken)) { - if ($force || ($expirationTime !== false)) { - try { - //echo "refresh_vault_token_2"; - $newToken = $provider->renewVaultToken($_SESSION['User']['Vault']['vaultUrl'], $existingToken); - //echo "refresh_vault_token_3"; - //echo $newToken; - if ($newToken) { - $newToken = json_decode($newToken,true); - $user = $_SESSION['User']; - $_SESSION['User']['Vault']['vaultKey'] = $newToken; - $tokenTime = $provider->getTokenExpirationTime($_SESSION['User']['Vault']['vaultUrl'], $user['Vault']['vaultKey']); - //echo "refresh_vault_token_4"; - //echo $tokenTime; - if ($tokenTime !== false) { - $user['Vault']['expires_in'] = $tokenTime; - //echo "Updated"; - } - updateUser($user); - $_SESSION['User']['Vault']['vaultKey'] = $newToken; - $_SESSION['User']['Vault']['expires_in'] = $tokenTime; - //updateUser($user); - return true; - } - } - catch (Exception $e){ - $_SESSION['errorData']['Error'][] = "Cannot validate token from refresh Vault token."; - $_SESSION['errorData']['Error'][] = $e->getMessage(); - return false; - } - - }else{ - $_SESSION['errorData']['Warning'][] = "Vault token is expired, you should save or update your credentials to generate a new one."; - return false; - } + if (!$_SESSION['User']['Vault']['vaultKey']) { + ob_clean(); + header('Location: ' . $GLOBALS['BASEURL'] . '/htmlib/errordb.php?msg=Vault Authentification Session Expired. Login again'); + } + //echo "refresh_vault_token"; + //echo $_SESSION['User']['Vault']['vaultKey']; + + $existingToken = $_SESSION['User']['Vault']['vaultKey']; + + //$provider = new MuG_Oauth2Provider\MuG_Oauth2Provider(['redirectUri'=> 'http://' . $_SERVER['SERVER_NAME'] . $_SERVER['PHP_SELF']]); + //$provider = new MuG_Oauth2Provider\MuG_Oauth2Provider(['redirectUri'=> $GLOBALS['URL'] . $_SERVER['PHP_SELF']]); + + //add them to mongo + $provider = new VaultClient($_SESSION['User']['Vault']['vaultUrl'], $_SESSION['User']['Vault']['vaultToken'], $_SESSION['User']['Token']['access_token'], $_SESSION['User']['Vault']['vaultRole'], $_POST['username']); + + $expirationTime = $provider->getTokenExpirationTime($_SESSION['User']['Vault']['vaultUrl'], $existingToken); + //echo "refresh_vault_token_1"; + //echo $expirationTime; + + //expiration time can be or the timestamp or false: + //if ($force || $provider->isTokenExpired($_SESSION['User']['Vault']['vaultUrl'], $existingToken)) { + if ($force || ($expirationTime !== false)) { + try { + //echo "refresh_vault_token_2"; + $newToken = $provider->renewVaultToken($_SESSION['User']['Vault']['vaultUrl'], $existingToken); + //echo "refresh_vault_token_3"; + //echo $newToken; + if ($newToken) { + $newToken = json_decode($newToken, true); + $user = $_SESSION['User']; + $_SESSION['User']['Vault']['vaultKey'] = $newToken; + $tokenTime = $provider->getTokenExpirationTime($_SESSION['User']['Vault']['vaultUrl'], $user['Vault']['vaultKey']); + //echo "refresh_vault_token_4"; + //echo $tokenTime; + if ($tokenTime !== false) { + $user['Vault']['expires_in'] = $tokenTime; + //echo "Updated"; + } + updateUser($user); + $_SESSION['User']['Vault']['vaultKey'] = $newToken; + $_SESSION['User']['Vault']['expires_in'] = $tokenTime; + //updateUser($user); + return true; + } + } catch (Exception $e) { + $_SESSION['errorData']['Error'][] = "Cannot validate token from refresh Vault token."; + $_SESSION['errorData']['Error'][] = $e->getMessage(); + return false; + } + } else { + $_SESSION['errorData']['Warning'][] = "Vault token is expired, you should save or update your credentials to generate a new one."; + return false; + } } @@ -2045,7 +2063,8 @@ function downloadFileSmall( $rfn ){ */ -function mimeTypes() { +function mimeTypes() +{ $mime_types = array( "log" => "text/plain", "txt" => "text/plain", @@ -2054,14 +2073,14 @@ function mimeTypes() { "out" => "text/plain", "csv" => "text/plain", "gff" => "text/plain", - "gff3"=> "text/plain", - "wig"=> "text/plain", - "bed"=> "text/plain", - "json"=> "text/plain", - "bedgraph"=> "text/plain", - "tre"=> "text/plain", - "nxt"=> "text/plain", - "nwt"=> "text/plain", + "gff3" => "text/plain", + "wig" => "text/plain", + "bed" => "text/plain", + "json" => "text/plain", + "bedgraph" => "text/plain", + "tre" => "text/plain", + "nxt" => "text/plain", + "nwt" => "text/plain", //"sh" => "application/x-sh", "sh" => "text/plain", "pdb" => "chemical/x-pdb", @@ -2074,8 +2093,8 @@ function mimeTypes() { "dcd" => "application/octet-stream", "exe" => "application/octet-stream", "gtar" => "application/octet-stream", - "bam"=> "application/octet-stream", - "sam"=> "application/octet-stream", + "bam" => "application/octet-stream", + "sam" => "application/octet-stream", "tar" => "application/x-tar", "gz" => "application/application/x-gzip", "tgz" => "application/application/x-gzip", @@ -2105,7 +2124,8 @@ function mimeTypes() { "doc" => "application/msword", "xls" => "application/vnd.ms-excel", "ppt" => "application/vnd.ms-powerpoint", - "tsv" => "text/tab-separated-values"); + "tsv" => "text/tab-separated-values" + ); return $mime_types; } @@ -2124,11 +2144,12 @@ function check_key_repeats($key, $hash) { } */ -function return_bytes($val) { +function return_bytes($val) +{ $val = trim($val); - $last = strtolower($val[strlen($val)-1]); + $last = strtolower($val[strlen($val) - 1]); $val = intval($val); - switch($last) { + switch ($last) { case 'g': $val *= 1024; case 'm': @@ -2142,227 +2163,230 @@ function return_bytes($val) { // resolve virtual path (relative or absolutes) to local absolute path -function resolvePath_toLocalAbsolutePath($path,$job){ - - $rfn =""; - // file_path is an absolute path - if (preg_match('/^\//',$path)){ - if (preg_match('/^'.preg_quote($job['root_dir_virtual'],'/').'/',$path)){ - //PMES mounts dataDir/user_id as root_dir_virtual - if ($job['launcher'] == "PMES"){ - $rfn = str_replace($job['root_dir_virtual'],$GLOBALS['dataDir'].$_SESSION['User']['id'],$path); - - //SGE finds mounted dataDir as root_dir_virtual - }elseif ($job['launcher'] == "SGE" || $job['launcher'] == "ega_demo" || $job['launcher'] == "docker_SGE"){ - $rfn = str_replace($job['root_dir_mug'],$GLOBALS['dataDir'],$path); - } - // direct from file_path - }else{ - $rfn = $path; - } - - // file_path is relative - }else{ - // file_path is only a file name (file) - if (!preg_match('/\//',$path)){ - //$rfn = $GLOBALS['dataDir'].$_SESSION['User']['id']."/".$_SESSION['User']['activeProject']."/".$job['execution']."/".$path; - $rfn = $job["output_dir"]."/".$path; - - // file_path is relative to user data directory (run/file) - }elseif (preg_match('/^'.$job['execution'].'/',$path)){ - //$rfn = $GLOBALS['dataDir'].$_SESSION['User']['id']."/".$_SESSION['User']['activeProject']."/".$path; - $rfn = dirname($job["output_dir"])."/".$path; - - // file_path is relative to root directory (userid/prj/run/file) - }elseif (preg_match('/^'.$_SESSION['User']['id'].'/',$path)){ - $rfn = $GLOBALS['dataDir']."/".$path; - - // file_path contains $(working_dir) tag - }elseif(preg_match('/(working_dir)/',$path)){ - $rfn = str_replace("$(working_dir)",$job['working_dir']."/",$path); +function resolvePath_toLocalAbsolutePath($path, $job) +{ + + $rfn = ""; + // file_path is an absolute path + if (preg_match('/^\//', $path)) { + if (preg_match('/^' . preg_quote($job['root_dir_virtual'], '/') . '/', $path)) { + //PMES mounts dataDir/user_id as root_dir_virtual + if ($job['launcher'] == "PMES") { + $rfn = str_replace($job['root_dir_virtual'], $GLOBALS['dataDir'] . $_SESSION['User']['id'], $path); + + //SGE finds mounted dataDir as root_dir_virtual + } elseif ($job['launcher'] == "SGE" || $job['launcher'] == "ega_demo" || $job['launcher'] == "docker_SGE") { + $rfn = str_replace($job['root_dir_mug'], $GLOBALS['dataDir'], $path); + } + // direct from file_path + } else { + $rfn = $path; + } + + // file_path is relative + } else { + // file_path is only a file name (file) + if (!preg_match('/\//', $path)) { + //$rfn = $GLOBALS['dataDir'].$_SESSION['User']['id']."/".$_SESSION['User']['activeProject']."/".$job['execution']."/".$path; + $rfn = $job["output_dir"] . "/" . $path; + + // file_path is relative to user data directory (run/file) + } elseif (preg_match('/^' . $job['execution'] . '/', $path)) { + //$rfn = $GLOBALS['dataDir'].$_SESSION['User']['id']."/".$_SESSION['User']['activeProject']."/".$path; + $rfn = dirname($job["output_dir"]) . "/" . $path; + + // file_path is relative to root directory (userid/prj/run/file) + } elseif (preg_match('/^' . $_SESSION['User']['id'] . '/', $path)) { + $rfn = $GLOBALS['dataDir'] . "/" . $path; - // file_path is relative to app working directory (userid/prj/run/file) - }else{ - $rfn = $job['working_dir']."/".$path; + // file_path contains $(working_dir) tag + } elseif (preg_match('/(working_dir)/', $path)) { + $rfn = str_replace("$(working_dir)", $job['working_dir'] . "/", $path); + + // file_path is relative to app working directory (userid/prj/run/file) + } else { + $rfn = $job['working_dir'] . "/" . $path; + } } - } - //clean slashes - $rfn = preg_replace('#/+#','/',$rfn); + //clean slashes + $rfn = preg_replace('#/+#', '/', $rfn); - //return absolute path - return $rfn; + //return absolute path + return $rfn; } -function deleteFiles($fileIds, $force = false) { - if (!is_array($fileIds)) { - $fileIds = [$fileIds]; - } +function deleteFiles($fileIds, $force = false) +{ + if (!is_array($fileIds)) { + $fileIds = [$fileIds]; + } - $result = true; - foreach ($fileIds as $fileId) { - $file = getGSFile_fromId($fileId); + $result = true; + foreach ($fileIds as $fileId) { + $file = getGSFile_fromId($fileId); if (!$file) { - $_SESSION['errorData']['Error'][]="Cannot delete file with id '$fileId'. Entry not found"; + $_SESSION['errorData']['Error'][] = "Cannot delete file with id '$fileId'. Entry not found"; $result = false; continue; } // check file exists $fileLocalPath = $file['path']; - $filePath = $GLOBALS['dataDir']."/$fileLocalPath"; + $filePath = $GLOBALS['dataDir'] . "/$fileLocalPath"; if (!file_exists($filePath) && !$force && $file['data_source'] != "EGA") { $_SESSION['errorData']['Error'][] = "filePath '$filePath' not found"; - $_SESSION['errorData']['Error'][] = "Cannot delete file with id '".basename($fileLocalPath)."'. File not found."; + $_SESSION['errorData']['Error'][] = "Cannot delete file with id '" . basename($fileLocalPath) . "'. File not found."; $result = false; continue; } // delete file from DMP $r = deleteGSFileBNS($fileId); - if ($r == 0){ - $_SESSION['errorData']['Error'][]="Cannot delete file '".basename($fileLocalPath)."'. Cannot delete entry from the repository."; + if ($r == 0) { + $_SESSION['errorData']['Error'][] = "Cannot delete file '" . basename($fileLocalPath) . "'. Cannot delete entry from the repository."; $result = false; continue; } // delete file from disk - if (file_exists($filePath)){ + if (file_exists($filePath)) { unlink($filePath); - if (error_get_last()){ - $_SESSION['errorData']['Error'][]="Errors encountered while deleting file '".basename($fileLocalPath)."'."; - $_SESSION['errorData']['Error'][]=error_get_last()["message"]; - $result = false; - continue; + if (error_get_last()) { + $_SESSION['errorData']['Error'][] = "Errors encountered while deleting file '" . basename($fileLocalPath) . "'."; + $_SESSION['errorData']['Error'][] = error_get_last()["message"]; + $result = false; + continue; } } // if is an associated file, update master file - if (isset($file['associated_id'])){ - $master_id = $file['associated_id']; - $master = getGSFile_fromId($master_id,"onlyMetadata"); - if ($master){ - //print "FILE IS an associated FILE! update $master_id
    "; - if (($k = array_search($fileId, $master['associated_files'])) !== false) { - unset($master['associated_files'][$k]); - $r = addMetadataBNS($master_id,$master); - if ($r == "0" ){ - $_SESSION['errorData']['Error'][] = "File '".basename($fileLocalPath)."' successfully deleted, but cannot update its master file $master_id metadata"; - $result = false; - continue; - } - } - } - - // if has associated files, delete them - }elseif (isset($file['associated_files'])){ - - //print "FILE HAS associated files! deleteing them !
    "; - foreach ($file['associated_files'] as $assoc_id){ - $r = deleteFiles($assoc_id); - if (!$r){ - $_SESSION['errorData']['Warning'][]= "File '".basename($fileLocalPath)."' successfully deleted, but not its associated file ($assoc_id)."; - $result = false; - } - } - } - } - return $result; -} + if (isset($file['associated_id'])) { + $master_id = $file['associated_id']; + $master = getGSFile_fromId($master_id, "onlyMetadata"); + if ($master) { + //print "FILE IS an associated FILE! update $master_id
    "; + if (($k = array_search($fileId, $master['associated_files'])) !== false) { + unset($master['associated_files'][$k]); + $r = addMetadataBNS($master_id, $master); + if ($r == "0") { + $_SESSION['errorData']['Error'][] = "File '" . basename($fileLocalPath) . "' successfully deleted, but cannot update its master file $master_id metadata"; + $result = false; + continue; + } + } + } + // if has associated files, delete them + } elseif (isset($file['associated_files'])) { -function moveFiles($fns,$target_fn){ + //print "FILE HAS associated files! deleteing them !
    "; + foreach ($file['associated_files'] as $assoc_id) { + $r = deleteFiles($assoc_id); + if (!$r) { + $_SESSION['errorData']['Warning'][] = "File '" . basename($fileLocalPath) . "' successfully deleted, but not its associated file ($assoc_id)."; + $result = false; + } + } + } + } + return $result; +} - $result = true; - $multipleFiles = FALSE; - //check target - $target_fn = rtrim($target_fn,"/"); - $targetId = getGSFileId_fromPath($target_fn); - $target_dir = ""; - $target_filename= ""; +function moveFiles($fns, $target_fn) +{ - // is a single fn given, target must be a file - if (!is_array($fns)){ + $result = true; $multipleFiles = FALSE; - $fns = array($fns); - if ($targetId){ - $_SESSION['errorData']['Error'][]="Cannot move file into target path '$target_fn'. File already exists"; - $result = false; - }else{ - $target_dir = rtrim(dirname($target_fn),"/"); - $target_filename = basename($target_fn); - } - - // is array of fn given, target must be a directory - }else{ - $multipleFiles = TRUE; - if (!$targetId || !is_dir($GLOBALS['dataDir']."/$target_fn") ){ - $_SESSION['errorData']['Error'][]="Cannot move multiple files into target directory '".$GLOBALS['dataDir']."/$target_fn'. Target must be un existing directory"; - $result = false; - }else{ - $target_dir = rtrim($target_fn,"/"); - } - } - - // move each fn - if ($result){ - foreach($fns as $fn){ - $file = getGSFile_fromId($fn); - if (!$file){ - $_SESSION['errorData']['Error'][]="Cannot move file with id '$fn'. Entry not found"; - $result = false; - continue; - } - - // check file exists - $file_fn = $file['path']; - $file_rfn = $GLOBALS['dataDir']."/$file_fn"; - - if (!file_exists($file_rfn)){ - $_SESSION['errorData']['Error'][]="Cannot move file named '".basename($file_fn)."'. File not found."; - $result = false; - continue; - } - - // set complete target path - if ($multipleFiles === TRUE){ - $target_filename = basename($file_fn); - } - $target_dir_rfn = $GLOBALS['dataDir'] . "/$target_dir"; - - // move file from DMP - $r = moveGSFileBNS($file_fn,"$target_dir/$target_filename"); - if ($r == "0"){ - $_SESSION['errorData']['Error'][]="Error while moving file '".basename($file_fn)."'"; - $result = false; - continue; - } - - // move file in disk - rename($file_rfn,"$target_dir_rfn/$target_filename"); - if (!is_file("$target_dir_rfn/$target_filename")){ - $_SESSION['errorData']['Error'][]= "Error while writting moved file"; - $result = false; - continue; - } - - // move associated ids - if (isset($file['associated_files'])){ - foreach ($file['associated_files'] as $assoc_id){ - $assoc = getGSFile_fromId($assoc_id); - if ($assoc){ - $r = moveGSFileBNS($assoc['path'],"$target_dir/".basename($assoc_path)); - if ($r == "0"){ - $_SESSION['errorData']['Warning'][]= "File '".basename($file_fn)."' successfully moved, but not its associated file (".basename($assoc['path']).")."; - $result = false; - } - } - } - } - } - } - - return $result; + + //check target + $target_fn = rtrim($target_fn, "/"); + $targetId = getGSFileId_fromPath($target_fn); + $target_dir = ""; + $target_filename = ""; + + // is a single fn given, target must be a file + if (!is_array($fns)) { + $multipleFiles = FALSE; + $fns = array($fns); + if ($targetId) { + $_SESSION['errorData']['Error'][] = "Cannot move file into target path '$target_fn'. File already exists"; + $result = false; + } else { + $target_dir = rtrim(dirname($target_fn), "/"); + $target_filename = basename($target_fn); + } + + // is array of fn given, target must be a directory + } else { + $multipleFiles = TRUE; + if (!$targetId || !is_dir($GLOBALS['dataDir'] . "/$target_fn")) { + $_SESSION['errorData']['Error'][] = "Cannot move multiple files into target directory '" . $GLOBALS['dataDir'] . "/$target_fn'. Target must be un existing directory"; + $result = false; + } else { + $target_dir = rtrim($target_fn, "/"); + } + } + + // move each fn + if ($result) { + foreach ($fns as $fn) { + $file = getGSFile_fromId($fn); + if (!$file) { + $_SESSION['errorData']['Error'][] = "Cannot move file with id '$fn'. Entry not found"; + $result = false; + continue; + } + + // check file exists + $file_fn = $file['path']; + $file_rfn = $GLOBALS['dataDir'] . "/$file_fn"; + + if (!file_exists($file_rfn)) { + $_SESSION['errorData']['Error'][] = "Cannot move file named '" . basename($file_fn) . "'. File not found."; + $result = false; + continue; + } + + // set complete target path + if ($multipleFiles === TRUE) { + $target_filename = basename($file_fn); + } + $target_dir_rfn = $GLOBALS['dataDir'] . "/$target_dir"; + + // move file from DMP + $r = moveGSFileBNS($file_fn, "$target_dir/$target_filename"); + if ($r == "0") { + $_SESSION['errorData']['Error'][] = "Error while moving file '" . basename($file_fn) . "'"; + $result = false; + continue; + } + + // move file in disk + rename($file_rfn, "$target_dir_rfn/$target_filename"); + if (!is_file("$target_dir_rfn/$target_filename")) { + $_SESSION['errorData']['Error'][] = "Error while writting moved file"; + $result = false; + continue; + } + + // move associated ids + if (isset($file['associated_files'])) { + foreach ($file['associated_files'] as $assoc_id) { + $assoc = getGSFile_fromId($assoc_id); + if ($assoc) { + $r = moveGSFileBNS($assoc['path'], "$target_dir/" . basename($assoc_path)); + if ($r == "0") { + $_SESSION['errorData']['Warning'][] = "File '" . basename($file_fn) . "' successfully moved, but not its associated file (" . basename($assoc['path']) . ")."; + $result = false; + } + } + } + } + } + } + + return $result; } diff --git a/sge/shared_scripts/download_from_url.sh b/sge/shared_scripts/download_from_url.sh new file mode 100755 index 00000000..0b55e9cd --- /dev/null +++ b/sge/shared_scripts/download_from_url.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +VERSION=0.1.0 +SUBJECT=wget +USAGE="USAGE: $SUBJECT --url URL --output OUTNAME --working_dir OUTDIR" + +# --- Option processing -------------------------------------------- + + +while [[ $# -gt 1 ]] +do +key="$1" +case $key in + --url) + Url="$2" + shift + ;; + --output) + Outname="$2" + shift + ;; + --working_dir) + Tmpdir="$2" + shift + ;; + --archiver) + Archiver="$2" + shift + ;; + --compressor) + Compressor="$2" + shift + ;; + *) + # unknown option + ;; +esac +shift # past argument or value +done + +# Set defaults +Archiver=${Archiver:-'0'} # options 'TAR' +Compressor=${Compressor:-'0'} # options 'GZIP', 'ZIP', 'BZIP2' +Tmpdir=${Tmpdir:-$(pwd)} + +# Print parameters +echo '- Url='$Url +echo '- Archiver='$Archiver +echo '- Compressor='$Compressor + +# Do things +cd $Tmpdir +if [ $Archiver = 'TAR' ] +then + mkdir $Outname +fi + +if [ $Archiver = '0' ] && [ $Compressor = '0' ]; then + echo "wget $Url -O $Outname" + wget $Url -O $Outname + +elif [ $Archiver = 'TAR' ] && [ $Compressor = 'GZIP' ]; then + echo "wget $Url -O- | tar -xzv -C $Outname/" + wget $Url -O- | tar -xzv -C $Outname/ + +elif [ $Archiver = 'TAR' ] && [ $Compressor = 'BZIP2' ]; then + wget $Url -O- | tar -xjv -C $Outname/ + +elif [ $Compressor = 'GZIP' ]; then + echo "wget $Url -O- | gunzip -c > $Outname" + wget $Url -O- | gunzip -c > $Outname + +elif [ $Compressor = 'BZIP2' ]; then + wget $Url -O- | bunzip2 -c > $Outname + +elif [ $Compressor = 'ZIP' ]; then + wget $Url -O output.zip + unzip output.zip -d $Outname + +else + echo "ERROR: Not downloading file. The requested archiver '$Archiver' and/or compressor '$Compressor' are not supported." + echo "Archiver: '0' for not unbundling the donwloaded folder; 'TAR' for applying 'untar' on the downloaded folder" + echo "Compressor: '0' for not uncompressing the downloaded data; 'GZIP', 'ZIP', 'BZIP2' " +fi \ No newline at end of file