From adb49a743984bb201cead00690e534eae3b67122 Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Tue, 21 Oct 2025 08:07:34 -0700 Subject: [PATCH 1/6] Add queue metrics for prometheus --- ProcessMaker/Console/Kernel.php | 2 ++ ProcessMaker/Services/MetricsService.php | 36 ++++++++++++++++++++++-- routes/web.php | 2 ++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/ProcessMaker/Console/Kernel.php b/ProcessMaker/Console/Kernel.php index d5f52d15e1..06b62297d6 100644 --- a/ProcessMaker/Console/Kernel.php +++ b/ProcessMaker/Console/Kernel.php @@ -88,6 +88,8 @@ protected function schedule(Schedule $schedule) $schedule->command('metrics:clear')->cron("*/{$clearInterval} * * * *"); break; } + + $schedule->command('horizon:snapshot')->everyFiveMinutes(); } /** diff --git a/ProcessMaker/Services/MetricsService.php b/ProcessMaker/Services/MetricsService.php index 20ae9b8862..4ae8492bca 100644 --- a/ProcessMaker/Services/MetricsService.php +++ b/ProcessMaker/Services/MetricsService.php @@ -3,6 +3,9 @@ namespace ProcessMaker\Services; use Exception; +use Laravel\Horizon\Contracts\JobRepository; +use Laravel\Horizon\Contracts\MetricsRepository; +use Laravel\Horizon\Contracts\WorkloadRepository; use ProcessMaker\Facades\Metrics; use Prometheus\CollectorRegistry; use Prometheus\Counter; @@ -65,7 +68,7 @@ public function getCollectionRegistry(): CollectorRegistry * @param array $labels The labels of the counter. * @return Counter The registered or retrieved counter. */ - public function counter(string $name, string $help = null, array $labels = []): Counter + public function counter(string $name, string|null $help = null, array $labels = []): Counter { $help = $help ?? $name; @@ -85,7 +88,7 @@ public function counter(string $name, string $help = null, array $labels = []): * @param array $labels The labels of the gauge. * @return Gauge The registered or retrieved gauge. */ - public function gauge(string $name, string $help = null, array $labels = []): Gauge + public function gauge(string $name, string|null $help = null, array $labels = []): Gauge { $help = $help ?? $name; @@ -201,7 +204,9 @@ public function addSystemLabels(array $labels) // Add system labels $labels['app_version'] = $this->getApplicationVersion(); $labels['app_name'] = config('app.name'); - $labels['app_custom_label'] = config('app.prometheus_custom_label'); + if (config('app.custom_label')) { + $labels['app_custom_label'] = config('app.custom_label'); + } return $labels; } @@ -223,4 +228,29 @@ private function getApplicationVersion() return $composer_json_path->version ?? '4.0.0'; } + + /** + * These are collected every time the /metrics route is accessed. + * + * @return void + */ + public function collectRealTimeMetrics(): void + { + $metricsRepository = app(MetricsRepository::class); + $jobsRepository = app(JobRepository::class); + $workloadRepository = app(WorkloadRepository::class); + + $this->gauge('horizon_jobs_per_minute', 'Jobs processed per minute')->set($metricsRepository->jobsProcessedPerMinute()); + $this->gauge('horizon_failed_jobs_per_hour', 'Failed jobs per hour')->set($jobsRepository->countRecentlyFailed()); + + foreach ($workloadRepository->get() as $workload) { + $name = $workload['name']; + foreach (['length', 'wait', 'processes'] as $type) { + $this->gauge( + 'horizon_workload_' . $name . '_' . $type, + 'Workload ' . $name . ' ' . $type + )->set($workload[$type]); + } + } + } } diff --git a/routes/web.php b/routes/web.php index 2039cb5ee7..e3d1a06219 100644 --- a/routes/web.php +++ b/routes/web.php @@ -257,6 +257,8 @@ // Metrics Route Route::get('/metrics', function () { + Metrics::collectRealTimeMetrics(); + return response(Metrics::renderMetrics(), 200, [ 'Content-Type' => 'text/plain; version=0.0.4', ]); From 02310ed04a5cc7f7881d8d8b42ef07f1fcf2accd Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Tue, 21 Oct 2025 08:19:04 -0700 Subject: [PATCH 2/6] Add note --- ProcessMaker/Console/Kernel.php | 1 + 1 file changed, 1 insertion(+) diff --git a/ProcessMaker/Console/Kernel.php b/ProcessMaker/Console/Kernel.php index 06b62297d6..1d6ee38a81 100644 --- a/ProcessMaker/Console/Kernel.php +++ b/ProcessMaker/Console/Kernel.php @@ -89,6 +89,7 @@ protected function schedule(Schedule $schedule) break; } + // 5 minutes is recommended in https://laravel.com/docs/12.x/horizon#metrics $schedule->command('horizon:snapshot')->everyFiveMinutes(); } From ccadccac88b2f15b2f91d2e1744e0e8f5ff05990 Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Tue, 21 Oct 2025 08:22:24 -0700 Subject: [PATCH 3/6] Fix config key --- ProcessMaker/Services/MetricsService.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ProcessMaker/Services/MetricsService.php b/ProcessMaker/Services/MetricsService.php index 4ae8492bca..b2196881be 100644 --- a/ProcessMaker/Services/MetricsService.php +++ b/ProcessMaker/Services/MetricsService.php @@ -204,8 +204,8 @@ public function addSystemLabels(array $labels) // Add system labels $labels['app_version'] = $this->getApplicationVersion(); $labels['app_name'] = config('app.name'); - if (config('app.custom_label')) { - $labels['app_custom_label'] = config('app.custom_label'); + if (config('app.prometheus_custom_label')) { + $labels['app_custom_label'] = config('app.prometheus_custom_label'); } return $labels; From 21808982dcfc681213ec697faa2107b9aff87c9a Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Wed, 22 Oct 2025 10:41:45 -0700 Subject: [PATCH 4/6] Handle cache prefixing in bootstrapper --- ProcessMaker/Multitenancy/TenantBootstrapper.php | 2 ++ config/multitenancy.php | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ProcessMaker/Multitenancy/TenantBootstrapper.php b/ProcessMaker/Multitenancy/TenantBootstrapper.php index 44897c4763..a7c09e7482 100644 --- a/ProcessMaker/Multitenancy/TenantBootstrapper.php +++ b/ProcessMaker/Multitenancy/TenantBootstrapper.php @@ -34,6 +34,7 @@ class TenantBootstrapper 'REDIS_PREFIX', 'CACHE_SETTING_PREFIX', 'SCRIPT_MICROSERVICE_CALLBACK', + 'CACHE_PREFIX', ]; public function bootstrap(Application $app) @@ -86,6 +87,7 @@ private function setTenantEnvironmentVariables($tenantData) $this->set('APP_KEY', $this->decrypt($config['app.key'])); $this->set('DB_DATABASE', $tenantData['database']); $this->set('DB_USERNAME', $tenantData['username'] ?? $this->getOriginalValue('DB_USERNAME')); + $this->set('CACHE_PREFIX', $this->getOriginalValue('CACHE_PREFIX') . 'tenant_' . $tenantData['id'] . ':'); $encryptedPassword = $tenantData['password']; $password = null; diff --git a/config/multitenancy.php b/config/multitenancy.php index b1a5aa5e2f..7c56e74d8f 100644 --- a/config/multitenancy.php +++ b/config/multitenancy.php @@ -34,7 +34,6 @@ */ 'switch_tenant_tasks' => [ SwitchTenant::class, - Spatie\Multitenancy\Tasks\PrefixCacheTask::class, ], /* From 10779c365825af4296ef1b670950667d935990db Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Wed, 22 Oct 2025 10:42:29 -0700 Subject: [PATCH 5/6] Make metrics tenant aware --- .../Exception/MultitenancyAccessedLandlord.php | 10 ++++++++++ ProcessMaker/Services/MetricsService.php | 13 ++++++++++--- routes/web.php | 4 +++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/ProcessMaker/Exception/MultitenancyAccessedLandlord.php b/ProcessMaker/Exception/MultitenancyAccessedLandlord.php index d2bb0654ed..cfeb78fd3b 100644 --- a/ProcessMaker/Exception/MultitenancyAccessedLandlord.php +++ b/ProcessMaker/Exception/MultitenancyAccessedLandlord.php @@ -5,11 +5,21 @@ use Exception; use Illuminate\Http\Request; use Illuminate\Http\Response; +use ProcessMaker\Facades\Metrics; class MultitenancyAccessedLandlord extends Exception { public function render(Request $request): Response { + // If we're trying to access the /metrics route, collect landlord metrics and render them + if ($request->path() === 'metrics') { + Metrics::collectQueueMetrics(); + + return response(Metrics::renderMetrics(), 200, [ + 'Content-Type' => 'text/plain; version=0.0.4', + ]); + } + return response()->view('multitenancy.landlord-landing-page'); } diff --git a/ProcessMaker/Services/MetricsService.php b/ProcessMaker/Services/MetricsService.php index b2196881be..824f12b6cc 100644 --- a/ProcessMaker/Services/MetricsService.php +++ b/ProcessMaker/Services/MetricsService.php @@ -7,12 +7,14 @@ use Laravel\Horizon\Contracts\MetricsRepository; use Laravel\Horizon\Contracts\WorkloadRepository; use ProcessMaker\Facades\Metrics; +use ProcessMaker\Multitenancy\Tenant; use Prometheus\CollectorRegistry; use Prometheus\Counter; use Prometheus\Gauge; use Prometheus\Histogram; use Prometheus\RenderTextFormat; -use Prometheus\Storage\Redis; +use Prometheus\Storage\Redis as PrometheusRedis; +use Redis; use RuntimeException; class MetricsService @@ -42,7 +44,12 @@ public function __construct(private $adapter = null) try { // Set up Redis as the adapter if none is provided if ($adapter === null) { - $adapter = Redis::fromExistingConnection(app('redis')->client()); + $redis = app('redis')->client(); + $adapter = PrometheusRedis::fromExistingConnection($redis); + if (app()->has(Tenant::BOOTSTRAPPED_TENANT)) { + $tenantInfo = app(Tenant::BOOTSTRAPPED_TENANT); + $adapter->setPrefix('tenant_' . $tenantInfo['id'] . ':PROMETHEUS_'); + } } $this->collectionRegistry = new CollectorRegistry($adapter); } catch (Exception $e) { @@ -234,7 +241,7 @@ private function getApplicationVersion() * * @return void */ - public function collectRealTimeMetrics(): void + public function collectQueueMetrics(): void { $metricsRepository = app(MetricsRepository::class); $jobsRepository = app(JobRepository::class); diff --git a/routes/web.php b/routes/web.php index e3d1a06219..30f1cdaa35 100644 --- a/routes/web.php +++ b/routes/web.php @@ -257,7 +257,9 @@ // Metrics Route Route::get('/metrics', function () { - Metrics::collectRealTimeMetrics(); + if (!config('app.multitenancy')) { + Metrics::collectQueueMetrics(); + } return response(Metrics::renderMetrics(), 200, [ 'Content-Type' => 'text/plain; version=0.0.4', From 0f0dd737b7b740cb4ccdeadd9b48873922c6aefe Mon Sep 17 00:00:00 2001 From: Nolan Ehrstrom Date: Thu, 23 Oct 2025 07:50:43 -0700 Subject: [PATCH 6/6] Fix schedule:run cron error --- ProcessMaker/Jobs/ErrorHandling.php | 2 +- ProcessMaker/Multitenancy/SwitchTenant.php | 2 +- .../Multitenancy/TenantBootstrapper.php | 18 ++++-------------- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/ProcessMaker/Jobs/ErrorHandling.php b/ProcessMaker/Jobs/ErrorHandling.php index ea0049a742..db880d3767 100644 --- a/ProcessMaker/Jobs/ErrorHandling.php +++ b/ProcessMaker/Jobs/ErrorHandling.php @@ -213,7 +213,7 @@ public static function convertResponseToException($result) if (str_starts_with($result['message'], 'Command exceeded timeout of')) { throw new ScriptTimeoutException($result['message']); } - throw new ScriptException($result['message']); + throw new ScriptException(json_encode($result, JSON_PRETTY_PRINT)); } } } diff --git a/ProcessMaker/Multitenancy/SwitchTenant.php b/ProcessMaker/Multitenancy/SwitchTenant.php index 1902a0b8e5..c549e54418 100644 --- a/ProcessMaker/Multitenancy/SwitchTenant.php +++ b/ProcessMaker/Multitenancy/SwitchTenant.php @@ -70,7 +70,7 @@ private function overrideConfigs(Application $app, IsTenant $tenant) if (!isset($tenant->config['app.docker_host_url'])) { // There is no specific override in the tenant's config so set it to the app url - $newConfig['app.docker_host_url'] = config('app.url'); + $newConfig['app.docker_host_url'] = config('app.docker_host_url', config('app.url')); } // Set config from the entry in the tenants table diff --git a/ProcessMaker/Multitenancy/TenantBootstrapper.php b/ProcessMaker/Multitenancy/TenantBootstrapper.php index a7c09e7482..2ed8a9b07d 100644 --- a/ProcessMaker/Multitenancy/TenantBootstrapper.php +++ b/ProcessMaker/Multitenancy/TenantBootstrapper.php @@ -2,6 +2,7 @@ namespace ProcessMaker\Multitenancy; +use Dotenv\Dotenv; use Illuminate\Encryption\Encrypter; use Illuminate\Http\Request; use Illuminate\Support\Env; @@ -44,8 +45,6 @@ public function bootstrap(Application $app) } $this->app = $app; - self::saveLandlordValues($app); - $tenantData = null; // Try to find tenant by ID first if TENANT env var is set @@ -101,21 +100,12 @@ private function setTenantEnvironmentVariables($tenantData) $this->set('LOG_PATH', $this->app->storagePath('logs/processmaker.log')); } - public static function saveLandlordValues($app) + private function getOriginalValue($key) { - if ($app->has('landlordValues')) { - self::$landlordValues = $app->make('landlordValues'); - - return; + if (self::$landlordValues === []) { + self::$landlordValues = Dotenv::parse(file_get_contents(base_path('.env'))); } - foreach (self::$landlordKeysToSave as $key) { - self::$landlordValues[$key] = $_SERVER[$key] ?? ''; - } - } - - private function getOriginalValue($key) - { if (!isset(self::$landlordValues[$key])) { return ''; }