diff --git a/ProcessMaker/Console/Kernel.php b/ProcessMaker/Console/Kernel.php index d5f52d15e1..1d6ee38a81 100644 --- a/ProcessMaker/Console/Kernel.php +++ b/ProcessMaker/Console/Kernel.php @@ -88,6 +88,9 @@ protected function schedule(Schedule $schedule) $schedule->command('metrics:clear')->cron("*/{$clearInterval} * * * *"); break; } + + // 5 minutes is recommended in https://laravel.com/docs/12.x/horizon#metrics + $schedule->command('horizon:snapshot')->everyFiveMinutes(); } /** diff --git a/ProcessMaker/Exception/MultitenancyAccessedLandlord.php b/ProcessMaker/Exception/MultitenancyAccessedLandlord.php index d2bb0654ed..cfeb78fd3b 100644 --- a/ProcessMaker/Exception/MultitenancyAccessedLandlord.php +++ b/ProcessMaker/Exception/MultitenancyAccessedLandlord.php @@ -5,11 +5,21 @@ use Exception; use Illuminate\Http\Request; use Illuminate\Http\Response; +use ProcessMaker\Facades\Metrics; class MultitenancyAccessedLandlord extends Exception { public function render(Request $request): Response { + // If we're trying to access the /metrics route, collect landlord metrics and render them + if ($request->path() === 'metrics') { + Metrics::collectQueueMetrics(); + + return response(Metrics::renderMetrics(), 200, [ + 'Content-Type' => 'text/plain; version=0.0.4', + ]); + } + return response()->view('multitenancy.landlord-landing-page'); } diff --git a/ProcessMaker/Jobs/ErrorHandling.php b/ProcessMaker/Jobs/ErrorHandling.php index ea0049a742..db880d3767 100644 --- a/ProcessMaker/Jobs/ErrorHandling.php +++ b/ProcessMaker/Jobs/ErrorHandling.php @@ -213,7 +213,7 @@ public static function convertResponseToException($result) if (str_starts_with($result['message'], 'Command exceeded timeout of')) { throw new ScriptTimeoutException($result['message']); } - throw new ScriptException($result['message']); + throw new ScriptException(json_encode($result, JSON_PRETTY_PRINT)); } } } diff --git a/ProcessMaker/Multitenancy/SwitchTenant.php b/ProcessMaker/Multitenancy/SwitchTenant.php index 1902a0b8e5..c549e54418 100644 --- a/ProcessMaker/Multitenancy/SwitchTenant.php +++ b/ProcessMaker/Multitenancy/SwitchTenant.php @@ -70,7 +70,7 @@ private function overrideConfigs(Application $app, IsTenant $tenant) if (!isset($tenant->config['app.docker_host_url'])) { // There is no specific override in the tenant's config so set it to the app url - $newConfig['app.docker_host_url'] = config('app.url'); + $newConfig['app.docker_host_url'] = config('app.docker_host_url', config('app.url')); } // Set config from the entry in the tenants table diff --git a/ProcessMaker/Multitenancy/TenantBootstrapper.php b/ProcessMaker/Multitenancy/TenantBootstrapper.php index 44897c4763..2ed8a9b07d 100644 --- a/ProcessMaker/Multitenancy/TenantBootstrapper.php +++ b/ProcessMaker/Multitenancy/TenantBootstrapper.php @@ -2,6 +2,7 @@ namespace ProcessMaker\Multitenancy; +use Dotenv\Dotenv; use Illuminate\Encryption\Encrypter; use Illuminate\Http\Request; use Illuminate\Support\Env; @@ -34,6 +35,7 @@ class TenantBootstrapper 'REDIS_PREFIX', 'CACHE_SETTING_PREFIX', 'SCRIPT_MICROSERVICE_CALLBACK', + 'CACHE_PREFIX', ]; public function bootstrap(Application $app) @@ -43,8 +45,6 @@ public function bootstrap(Application $app) } $this->app = $app; - self::saveLandlordValues($app); - $tenantData = null; // Try to find tenant by ID first if TENANT env var is set @@ -86,6 +86,7 @@ private function setTenantEnvironmentVariables($tenantData) $this->set('APP_KEY', $this->decrypt($config['app.key'])); $this->set('DB_DATABASE', $tenantData['database']); $this->set('DB_USERNAME', $tenantData['username'] ?? $this->getOriginalValue('DB_USERNAME')); + $this->set('CACHE_PREFIX', $this->getOriginalValue('CACHE_PREFIX') . 'tenant_' . $tenantData['id'] . ':'); $encryptedPassword = $tenantData['password']; $password = null; @@ -99,21 +100,12 @@ private function setTenantEnvironmentVariables($tenantData) $this->set('LOG_PATH', $this->app->storagePath('logs/processmaker.log')); } - public static function saveLandlordValues($app) + private function getOriginalValue($key) { - if ($app->has('landlordValues')) { - self::$landlordValues = $app->make('landlordValues'); - - return; + if (self::$landlordValues === []) { + self::$landlordValues = Dotenv::parse(file_get_contents(base_path('.env'))); } - foreach (self::$landlordKeysToSave as $key) { - self::$landlordValues[$key] = $_SERVER[$key] ?? ''; - } - } - - private function getOriginalValue($key) - { if (!isset(self::$landlordValues[$key])) { return ''; } diff --git a/ProcessMaker/Services/MetricsService.php b/ProcessMaker/Services/MetricsService.php index 20ae9b8862..824f12b6cc 100644 --- a/ProcessMaker/Services/MetricsService.php +++ b/ProcessMaker/Services/MetricsService.php @@ -3,13 +3,18 @@ namespace ProcessMaker\Services; use Exception; +use Laravel\Horizon\Contracts\JobRepository; +use Laravel\Horizon\Contracts\MetricsRepository; +use Laravel\Horizon\Contracts\WorkloadRepository; use ProcessMaker\Facades\Metrics; +use ProcessMaker\Multitenancy\Tenant; use Prometheus\CollectorRegistry; use Prometheus\Counter; use Prometheus\Gauge; use Prometheus\Histogram; use Prometheus\RenderTextFormat; -use Prometheus\Storage\Redis; +use Prometheus\Storage\Redis as PrometheusRedis; +use Redis; use RuntimeException; class MetricsService @@ -39,7 +44,12 @@ public function __construct(private $adapter = null) try { // Set up Redis as the adapter if none is provided if ($adapter === null) { - $adapter = Redis::fromExistingConnection(app('redis')->client()); + $redis = app('redis')->client(); + $adapter = PrometheusRedis::fromExistingConnection($redis); + if (app()->has(Tenant::BOOTSTRAPPED_TENANT)) { + $tenantInfo = app(Tenant::BOOTSTRAPPED_TENANT); + $adapter->setPrefix('tenant_' . $tenantInfo['id'] . ':PROMETHEUS_'); + } } $this->collectionRegistry = new CollectorRegistry($adapter); } catch (Exception $e) { @@ -65,7 +75,7 @@ public function getCollectionRegistry(): CollectorRegistry * @param array $labels The labels of the counter. * @return Counter The registered or retrieved counter. */ - public function counter(string $name, string $help = null, array $labels = []): Counter + public function counter(string $name, string|null $help = null, array $labels = []): Counter { $help = $help ?? $name; @@ -85,7 +95,7 @@ public function counter(string $name, string $help = null, array $labels = []): * @param array $labels The labels of the gauge. * @return Gauge The registered or retrieved gauge. */ - public function gauge(string $name, string $help = null, array $labels = []): Gauge + public function gauge(string $name, string|null $help = null, array $labels = []): Gauge { $help = $help ?? $name; @@ -201,7 +211,9 @@ public function addSystemLabels(array $labels) // Add system labels $labels['app_version'] = $this->getApplicationVersion(); $labels['app_name'] = config('app.name'); - $labels['app_custom_label'] = config('app.prometheus_custom_label'); + if (config('app.prometheus_custom_label')) { + $labels['app_custom_label'] = config('app.prometheus_custom_label'); + } return $labels; } @@ -223,4 +235,29 @@ private function getApplicationVersion() return $composer_json_path->version ?? '4.0.0'; } + + /** + * These are collected every time the /metrics route is accessed. + * + * @return void + */ + public function collectQueueMetrics(): void + { + $metricsRepository = app(MetricsRepository::class); + $jobsRepository = app(JobRepository::class); + $workloadRepository = app(WorkloadRepository::class); + + $this->gauge('horizon_jobs_per_minute', 'Jobs processed per minute')->set($metricsRepository->jobsProcessedPerMinute()); + $this->gauge('horizon_failed_jobs_per_hour', 'Failed jobs per hour')->set($jobsRepository->countRecentlyFailed()); + + foreach ($workloadRepository->get() as $workload) { + $name = $workload['name']; + foreach (['length', 'wait', 'processes'] as $type) { + $this->gauge( + 'horizon_workload_' . $name . '_' . $type, + 'Workload ' . $name . ' ' . $type + )->set($workload[$type]); + } + } + } } diff --git a/config/multitenancy.php b/config/multitenancy.php index b1a5aa5e2f..7c56e74d8f 100644 --- a/config/multitenancy.php +++ b/config/multitenancy.php @@ -34,7 +34,6 @@ */ 'switch_tenant_tasks' => [ SwitchTenant::class, - Spatie\Multitenancy\Tasks\PrefixCacheTask::class, ], /* diff --git a/routes/web.php b/routes/web.php index 2039cb5ee7..30f1cdaa35 100644 --- a/routes/web.php +++ b/routes/web.php @@ -257,6 +257,10 @@ // Metrics Route Route::get('/metrics', function () { + if (!config('app.multitenancy')) { + Metrics::collectQueueMetrics(); + } + return response(Metrics::renderMetrics(), 200, [ 'Content-Type' => 'text/plain; version=0.0.4', ]);