diff --git a/SQL/0000-00-00-schema.sql b/SQL/0000-00-00-schema.sql index 19fb5836987..01dff59e7c4 100644 --- a/SQL/0000-00-00-schema.sql +++ b/SQL/0000-00-00-schema.sql @@ -1515,6 +1515,35 @@ INSERT INTO StatisticsTabs (ModuleName, SubModuleName, Description, OrderNo) VAL ('statistics', 'stats_behavioural', 'Behavioural Statistics', 3), ('statistics', 'stats_MRI', 'Imaging Statistics', 4); + +-- ******************************** +-- statistics +-- ******************************** + + +CREATE TABLE `cached_data_type` ( + `CachedDataTypeID` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + `Name` VARCHAR(255) UNIQUE NOT NULL, + PRIMARY KEY (`CachedDataTypeID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + +INSERT INTO `cached_data_type` (`Name`) SELECT 'projects_disk_space'; + + +CREATE TABLE `cached_data` ( + `CachedDataID` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + `CachedDataTypeID` INT(10) UNSIGNED NOT NULL, + `Value` TEXT NOT NULL, + `LastUpdate` TIMESTAMP NOT NULL + DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`CachedDataID`), + CONSTRAINT `FK_cached_data_type` FOREIGN KEY (`CachedDataTypeID`) + REFERENCES `cached_data_type` (`CachedDataTypeID`) + ON UPDATE CASCADE ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + + -- ******************************** -- server_processes tables -- ******************************** diff --git a/SQL/New_patches/2025_09_11_add_data_cache_table.sql b/SQL/New_patches/2025_09_11_add_data_cache_table.sql new file mode 100644 index 00000000000..aaab6cc3c6d --- /dev/null +++ b/SQL/New_patches/2025_09_11_add_data_cache_table.sql @@ -0,0 +1,21 @@ +-- Create cached_data_type table to track different types of cached data +CREATE TABLE `cached_data_type` ( + `CachedDataTypeID` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + `Name` VARCHAR(255) UNIQUE NOT NULL, + PRIMARY KEY (`CachedDataTypeID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Create cached_data table to track cached data +CREATE TABLE `cached_data` ( + `CachedDataID` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT, + `CachedDataTypeID` INT(10) UNSIGNED NOT NULL, + `Value` TEXT NOT NULL, + `LastUpdate` TIMESTAMP NOT NULL + DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`CachedDataID`), + CONSTRAINT `FK_cached_data_type` FOREIGN KEY (`CachedDataTypeID`) + REFERENCES `cached_data_type` (`CachedDataTypeID`) + ON UPDATE CASCADE ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT INTO `cached_data_type` (`Name`) SELECT 'projects_disk_space'; diff --git a/locale/loris.pot b/locale/loris.pot index 59cbaba57dd..f556066be06 100644 --- a/locale/loris.pot +++ b/locale/loris.pot @@ -473,3 +473,9 @@ msgstr "" msgid "Permission denied" msgstr "" + +msgid "GB" +msgstr "" + +msgid "Month" +msgstr "" diff --git a/modules/statistics/jsx/widgets/helpers/chartBuilder.js b/modules/statistics/jsx/widgets/helpers/chartBuilder.js index b3b51d1f2c3..3b979a5576c 100644 --- a/modules/statistics/jsx/widgets/helpers/chartBuilder.js +++ b/modules/statistics/jsx/widgets/helpers/chartBuilder.js @@ -76,7 +76,7 @@ const formatBarData = (data) => { return processedData; }; -const createPieChart = (columns, id, targetModal, colours) => { +const createPieChart = (columns, id, targetModal, colours, units = null, showPieLabelRatio = true) => { let newChart = c3.generate({ bindto: targetModal ? targetModal : id, data: { @@ -92,13 +92,22 @@ const createPieChart = (columns, id, targetModal, colours) => { pie: { label: { format: function(value, ratio, id) { - return value + "("+Math.round(100*ratio)+"%)"; + if (units) { + value = `${value} ${units}`; + } + if (showPieLabelRatio) { + value = `${value} (${(ratio * 100).toFixed(0)}%)`; + } + return value; } } }, tooltip: { format: { value: function (value, ratio) { + if (units) { + value = `${value} ${units}`; + } return `${value} (${(ratio * 100).toFixed(0)}%)`; }, }, @@ -107,7 +116,7 @@ const createPieChart = (columns, id, targetModal, colours) => { return newChart; } -const createBarChart = (t, labels, columns, id, targetModal, colours, dataType) => { +const createBarChart = (labels, columns, id, targetModal, colours, dataType, yLabel) => { let newChart = c3.generate({ bindto: targetModal ? targetModal : id, data: { @@ -130,11 +139,11 @@ const createBarChart = (t, labels, columns, id, targetModal, colours, dataType) axis: { x: { type: 'category', - categories: labels, + categories: labels, }, y: { label: { - text: t('Candidates registered', { ns: 'statistics'}), + text: yLabel, position: 'inner-top' }, }, @@ -167,6 +176,7 @@ const createLineChart = (data, columns, id, label, targetModal, titlePrefix) => } } } + let newChart = c3.generate({ size: { height: targetModal && 500, @@ -226,10 +236,8 @@ const createLineChart = (data, columns, id, label, targetModal, titlePrefix) => name = nameFormat(d[i].name); value = valueFormat(d[i].value, d[i].ratio, d[i].id, d[i].index); - // Calculate percentage based on grand total of entire dataset let percentage = grandTotal > 0 ? ((d[i].value / grandTotal) * 100).toFixed(1) : 0; - bgcolor = $$.levelColor ? $$.levelColor(d[i].value) : color(d[i].id); text += ""; @@ -313,9 +321,9 @@ const setupCharts = async (t, targetIsModal, chartDetails, totalLabel) => { } let chartObject = null; if (chart.chartType === 'pie') { - chartObject = createPieChart(columns, `#${chartID}`, targetIsModal && '#dashboardModal', colours); + chartObject = createPieChart(columns, `#${chartID}`, targetIsModal && '#dashboardModal', colours, chart.units, chart.showPieLabelRatio); } else if (chart.chartType === 'bar') { - chartObject = createBarChart(t, labels, columns, `#${chartID}`, targetIsModal && '#dashboardModal', colours, chart.dataType); + chartObject = createBarChart(labels, columns, `#${chartID}`, targetIsModal && '#dashboardModal', colours, chart.dataType, chart.yLabel); } else if (chart.chartType === 'line') { chartObject = createLineChart(chartData, columns, `#${chartID}`, chart.label, targetIsModal && '#dashboardModal', chart.titlePrefix); } diff --git a/modules/statistics/jsx/widgets/recruitment.js b/modules/statistics/jsx/widgets/recruitment.js index 77812611baa..77348e35313 100644 --- a/modules/statistics/jsx/widgets/recruitment.js +++ b/modules/statistics/jsx/widgets/recruitment.js @@ -32,6 +32,7 @@ const Recruitment = (props) => { dataType: 'pie', label: 'Age (Years)', options: {pie: 'pie', bar: 'bar'}, + yLabel: t('Candidates registered', {ns: 'statistics'}), legend: 'under', chartObject: null, }, @@ -42,6 +43,7 @@ const Recruitment = (props) => { dataType: 'pie', label: 'Ethnicity', options: {pie: 'pie', bar: 'bar'}, + yLabel: t('Candidates registered', {ns: 'statistics'}), legend: 'under', chartObject: null, }, @@ -55,6 +57,7 @@ const Recruitment = (props) => { label: 'Participants', legend: '', options: {pie: 'pie', bar: 'bar'}, + yLabel: t('Candidates registered', {ns: 'statistics'}), chartObject: null, }, 'siterecruitment_bysex': { @@ -64,6 +67,7 @@ const Recruitment = (props) => { dataType: 'bar', legend: 'under', options: {bar: 'bar', pie: 'pie'}, + yLabel: t('Candidates registered', {ns: 'statistics'}), chartObject: null, }, }, @@ -75,6 +79,7 @@ const Recruitment = (props) => { dataType: 'line', legend: '', options: {line: 'line'}, + yLabel: t('Candidates registered', {ns: 'statistics'}), chartObject: null, }, }, @@ -149,6 +154,7 @@ const Recruitment = (props) => { setChartDetails); }; + // Helper functions to calculate totals for each view const getTotalProjectsCount = () => { return Object.keys(json['recruitment'] || {}) .filter((key) => key !== 'overall').length; diff --git a/modules/statistics/jsx/widgets/studyprogression.js b/modules/statistics/jsx/widgets/studyprogression.js index 70d6962c1bc..32df8255495 100644 --- a/modules/statistics/jsx/widgets/studyprogression.js +++ b/modules/statistics/jsx/widgets/studyprogression.js @@ -45,7 +45,8 @@ const StudyProgression = (props) => { legend: 'under', options: {line: 'line'}, chartObject: null, - titlePrefix: 'Month', + yLabel: t('Candidates registered', {ns: 'statistics'}), + titlePrefix: t('Month', {ns: 'loris'}), }, }, 'total_recruitment': { @@ -58,7 +59,25 @@ const StudyProgression = (props) => { legend: '', options: {line: 'line'}, chartObject: null, - titlePrefix: 'Month', + yLabel: t('Candidates registered', {ns: 'statistics'}), + titlePrefix: t('Month', {ns: 'loris'}), + }, + }, + 'project_sizes': { + 'size_byproject': { + sizing: 11, + title: t('Dataset size breakdown by project', {ns: 'statistics'}), + filters: '', + chartType: 'pie', + dataType: 'pie', + label: t('Size (GB)', {ns: 'statistics'}), + units: t('GB', {ns: 'loris'}), + showPieLabelRatio: false, + legend: '', + options: {pie: 'pie', bar: 'bar'}, + chartObject: null, + yLabel: t('Size (GB)', {ns: 'statistics'}), + titlePrefix: t('Project', {ns: 'loris'}), }, }, }); @@ -173,9 +192,16 @@ const StudyProgression = (props) => { {showChart('total_scans', 'scans_bymonth')} ) : ( -

There have been no scans yet.

+

{t('There have been no scans yet.', {ns: 'statistics'})}

), title: title('Site Scans'), + subtitle: t( + 'Total Scans: {{count}}', + { + ns: 'statistics', + count: json['studyprogression']['total_scans'], + } + ), onToggleFilters: () => setShowFiltersScans((prev) => !prev), }, { @@ -212,11 +238,53 @@ const StudyProgression = (props) => { {showChart('total_recruitment', 'siterecruitment_bymonth')} ) : ( -

There have been no candidates registered yet.

+

+ {t( + 'There have been no candidates registered yet.', + {ns: 'statistics'} + )} +

), title: title('Site Recruitment'), onToggleFilters: () => showFiltersBreakdown((prev) => !prev), }, + { + content: + Object.keys(json['options']['projects']).length > 0 ? ( +
+
+
+ {showFiltersBreakdown && ( + { + updateFilters(formDataObj, 'project_sizes'); + }} + /> + )} + {showChart('project_sizes', 'size_byproject')} +
+ ) : ( +

{t('There is no data yet.', {ns: 'statistics'})}

+ ), + title: title('Project Dataset Sizes'), + subtitle: t( + 'Total Size: {{count}} GB', + { + ns: 'statistics', + count: json['studyprogression']['total_size'] ?? -1, + } + ), + }, ]} /> diff --git a/modules/statistics/locale/statistics.pot b/modules/statistics/locale/statistics.pot index f0ea8b7e9f1..c4587a36300 100644 --- a/modules/statistics/locale/statistics.pot +++ b/modules/statistics/locale/statistics.pot @@ -125,3 +125,31 @@ msgstr "" msgid "Unknown" msgstr "" + +msgid "Dataset size breakdown by project" +msgstr "" + +msgid "Dataset Size" +msgid_plural "Dataset Size" +msgstr[0] "" + +msgid "Size (GB)" +msgstr "" + +msgid "%s GB" +msgstr "" + +msgid "Total Size: {{count}} GB" +msgstr "" + +msgid "Total Scans: {{count}}" +msgstr "" + +msgid "There have been no scans yet." +msgstr "" + +msgid "There have been no candidates registered yet." +msgstr "" + +msgid "There is no data yet." +msgstr "" diff --git a/modules/statistics/php/charts.class.inc b/modules/statistics/php/charts.class.inc index 2800dd359ef..c3ecd746d64 100644 --- a/modules/statistics/php/charts.class.inc +++ b/modules/statistics/php/charts.class.inc @@ -106,6 +106,8 @@ class Charts extends \NDB_Page return $this->_handleSiteLineData($request); case 'agedistribution_line': return $this->_handleAgeDistributionByProject($request); + case 'size_byproject': + return $this->_handleProjectSizeBreakdown(); default: return new \LORIS\Http\Response\JSON\NotFound(); } @@ -835,4 +837,44 @@ class Charts extends \NDB_Page } return $data; } + + /** + * Handle an incoming request for project size breakdown. + * + * @return ResponseInterface + */ + private function _handleProjectSizeBreakdown() + { + $DB = \NDB_Factory::singleton()->database(); + $user = \NDB_Factory::singleton()->user(); + $projects = $user->getProjects(); + + $cachedSizeData = json_decode( + html_entity_decode( + $DB->pselectOne( + "SELECT Value + FROM cached_data + JOIN cached_data_type USING (CachedDataTypeID) + WHERE Name='projects_disk_space'", + [] + ) ?? '' + ), + true + ); + + $projectData = []; + if (!is_null($cachedSizeData)) { + foreach ($projects as $project) { + $projectName = $project->getName(); + if (in_array($projectName, array_keys($cachedSizeData))) { + $projectData[] = [ + 'label' => $projectName, + ...$cachedSizeData[$projectName], + ]; + } + } + } + + return (new \LORIS\Http\Response\JsonResponse($projectData)); + } } diff --git a/modules/statistics/php/module.class.inc b/modules/statistics/php/module.class.inc index ef2b360998b..67942806db7 100644 --- a/modules/statistics/php/module.class.inc +++ b/modules/statistics/php/module.class.inc @@ -71,11 +71,11 @@ class Module extends \Module */ public function getWidgets(string $type, \User $user, array $options) : array { + $factory = \NDB_Factory::singleton(); + $baseURL = $factory->settings()->getBaseURL(); switch ($type) { case 'dashboard': - $factory = \NDB_Factory::singleton(); - $baseURL = $factory->settings()->getBaseURL(); - $widget = new \LORIS\dashboard\Widget( + $widget = new \LORIS\dashboard\Widget( new \LORIS\dashboard\WidgetContent( '', '', @@ -100,6 +100,56 @@ class Module extends \Module $widget->setTemplateVariables(['id' => 'statistics_widgets']); return [$widget]; + case 'study-progression': + $DB = $factory->database(); + $cachedSizeData = json_decode( + html_entity_decode( + $DB->pselectOne( + "SELECT Value + FROM cached_data + JOIN cached_data_type USING (CachedDataTypeID) + WHERE Name='projects_disk_space'", + [] + ) ?? '' + ), + true + ); + + $data = []; + + $projects = $user->getProjects(); + if (!is_null($cachedSizeData)) { + foreach ($projects as $project) { + $projectName = $project->getName(); + if (!in_array($projectName, array_keys($cachedSizeData))) { + continue; + } + + $datasetSize = sprintf( + dgettext('statistics', '%s GB'), + $cachedSizeData[$projectName]['total'] + ); + $data[] = [ + 'ProjectID' => $project->getId(), + 'ProjectName' => $projectName, + 'count' => $datasetSize, + 'url' => "$baseURL/dqt", + ]; + } + } + + return [ + new \LORIS\dashboard\DataWidget( + new \LORIS\GUI\LocalizableString( + "dqt", + "Dataset Size", + "Dataset Size", + ), + $data, + "", + 'rgb(186,225,255)', + ) + ]; } return []; } diff --git a/modules/statistics/php/widgets.class.inc b/modules/statistics/php/widgets.class.inc index 56d307b0cf6..823c9b66e31 100644 --- a/modules/statistics/php/widgets.class.inc +++ b/modules/statistics/php/widgets.class.inc @@ -119,6 +119,21 @@ class Widgets extends \NDB_Page implements ETagCalculator $studyWidgets ); + $totalSizeOfProjectsGB = 0; + $projectsSizes = []; + $cachedSizeData = json_decode( + html_entity_decode( + $db->pselectOne( + "SELECT Value + FROM cached_data + JOIN cached_data_type USING (CachedDataTypeID) + WHERE Name='projects_disk_space'", + [] + ) ?? '' + ), + true + ); + foreach ($projects as $pid) { // Set project recruitment data $projectInfo = $config->getProjectSettings(intval(strval($pid))); @@ -129,9 +144,10 @@ class Widgets extends \NDB_Page implements ETagCalculator 'project ID ' . intval(strval($pid)) ); } + $projectName = $projectInfo['Name']; $recruitment[intval(strval($pid))] = $this->_createProjectProgressBar( strval($pid), - $projectInfo['Name'], + $projectName, $projectInfo['recruitmentTarget'], $this->getTotalRecruitmentByProject( $recruitmentRaw, @@ -174,6 +190,14 @@ class Widgets extends \NDB_Page implements ETagCalculator $visitOptions[$visitLabel] = $visitName; } } + + if (!is_null($cachedSizeData)) { + if (in_array($projectName, array_keys($cachedSizeData))) { + $projectSize = $cachedSizeData[$projectName]['total']; + $projectsSizes[$projectName] = $projectSize; + $totalSizeOfProjectsGB += floatval($projectSize); + } + } } $siteOptions = []; @@ -201,10 +225,13 @@ class Widgets extends \NDB_Page implements ETagCalculator 'total_scans' => $totalScans, 'recruitment' => $recruitment, 'progressionData' => $studyProgressionProjects, + 'total_size' => $totalSizeOfProjectsGB, ]; $values['options'] = $options; $values['recruitmentcohorts'] = $recruitmentCohorts; + $values['size_byproject'] = $projectsSizes; + $this->_cache = new \LORIS\Http\Response\JsonResponse($values); return $this->_cache; @@ -599,7 +626,9 @@ class Widgets extends \NDB_Page implements ETagCalculator && isset($value['ProjectID']) && strval($value['ProjectID']) === $projectID ) { - $title = $widgetConfig['ref']->label()->getN($value['count']); + $title = $widgetConfig['ref']->label()->getN( + intval(preg_split('/\s+/', strval($value['count']))[0]) + ); $projectData[] = [ 'title' => $title, diff --git a/raisinbread/RB_files/RB_cached_data.sql b/raisinbread/RB_files/RB_cached_data.sql new file mode 100644 index 00000000000..cabb8271eed --- /dev/null +++ b/raisinbread/RB_files/RB_cached_data.sql @@ -0,0 +1,6 @@ +SET FOREIGN_KEY_CHECKS=0; +TRUNCATE TABLE `cached_data`; +LOCK TABLES `cached_data` WRITE; +INSERT INTO `cached_data` (`CachedDataID`, `CachedDataTypeID`, `Value`, `LastUpdate`) VALUES (1,1,'{"Pumpernickel":{"total":0.8}}','2025-09-10 11:12:13'); +UNLOCK TABLES; +SET FOREIGN_KEY_CHECKS=1; diff --git a/raisinbread/RB_files/RB_cached_data_type.sql b/raisinbread/RB_files/RB_cached_data_type.sql new file mode 100644 index 00000000000..e1f1ec608fb --- /dev/null +++ b/raisinbread/RB_files/RB_cached_data_type.sql @@ -0,0 +1,6 @@ +SET FOREIGN_KEY_CHECKS=0; +TRUNCATE TABLE `cached_data_type`; +LOCK TABLES `cached_data_type` WRITE; +INSERT INTO `cached_data_type` (`CachedDataTypeID`, `Name`) VALUES (1,'projects_disk_space'); +UNLOCK TABLES; +SET FOREIGN_KEY_CHECKS=1; diff --git a/tools/update_projects_disk_space.php b/tools/update_projects_disk_space.php new file mode 100644 index 00000000000..508f430d34d --- /dev/null +++ b/tools/update_projects_disk_space.php @@ -0,0 +1,95 @@ +database(); +$config = \NDB_Config::singleton(); + +$dataDir = $config->getSetting('dataDirBasepath'); +$projects = Utility::getProjectList(); + +$projectData = []; + +foreach ($projects as $pid => $project) { + $projectDir = $db->pselectOne( + "SELECT DISTINCT( + SUBSTRING(SUBSTRING_INDEX(FilePath, '/', 2), 1) + ) + FROM ( + SELECT FilePath, SessionID + FROM physiological_file pf + UNION + SELECT File, SessionID as FilePath FROM files f + ) file_table + LEFT JOIN session s ON s.ID = file_table.SessionID + WHERE FilePath LIKE 'bids_imports%' + AND s.ProjectID=:PID LIMIT 1;", + ['PID' => $pid] + ); + if (!is_null($projectDir)) { + $fullPath = $dataDir . $projectDir; + $dirSizeGB = round( + getDirSize($fullPath) / pow(10, 9), + 1 + ); + $projectData[$project]['total'] = $dirSizeGB; + } +} + +$cachedDataTypeID = $db->pselectOneInt( + "SELECT `CachedDataTypeID` + FROM `cached_data_type` + WHERE `Name` = 'projects_disk_space'", + [] +); + +$rowExists = $db->pselectOne( + "SELECT Value FROM cached_data + WHERE CachedDataTypeID = :CDTID;", + ['CDTID' => $cachedDataTypeID] +); + +if ($rowExists) { + $db->update( + 'cached_data', + ['Value' => json_encode($projectData)], + ['CachedDataTypeID' => $cachedDataTypeID] + ); +} else { + $db->insert( + 'cached_data', + [ + 'CachedDataTypeID' => $cachedDataTypeID, + 'Value' => json_encode($projectData) + ] + ); +} + +echo "cached_data:projects_disk_space updated\r\n"; + +/** + * Calculate directory size, recursively, skipping .tgz files + * + * @param string $directory Target directory + * + * @return int + */ +function getDirSize(string $directory): int +{ + $size = 0; + $files = glob($directory . '/*'); + foreach ($files as $path) { + if (!str_ends_with($path, '.tgz')) { + is_file($path) && $size += filesize($path); + is_dir($path) && $size += getDirSize($path); + } + } + return $size; +}