Add queue metrics for prometheus

ProcessMaker/Console/Kernel.php

@@ -88,6 +88,9 @@ protected function schedule(Schedule $schedule)
                 $schedule->command('metrics:clear')->cron("*/{$clearInterval} * * * *");
                 break;
         }
+
+        // 5 minutes is recommended in https://laravel.com/docs/12.x/horizon#metrics
+        $schedule->command('horizon:snapshot')->everyFiveMinutes();
     }
 
     /**

ProcessMaker/Exception/MultitenancyAccessedLandlord.php

@@ -5,11 +5,21 @@
 use Exception;
 use Illuminate\Http\Request;
 use Illuminate\Http\Response;
+use ProcessMaker\Facades\Metrics;
 
 class MultitenancyAccessedLandlord extends Exception
 {
     public function render(Request $request): Response
     {
+        // If we're trying to access the /metrics route, collect landlord metrics and render them
+        if ($request->path() === 'metrics') {
+            Metrics::collectQueueMetrics();
+
+            return response(Metrics::renderMetrics(), 200, [
+                'Content-Type' => 'text/plain; version=0.0.4',
+            ]);
+        }
+
         return response()->view('multitenancy.landlord-landing-page');
     }
 

ProcessMaker/Jobs/ErrorHandling.php

@@ -213,7 +213,7 @@ public static function convertResponseToException($result)
             if (str_starts_with($result['message'], 'Command exceeded timeout of')) {
                 throw new ScriptTimeoutException($result['message']);
             }
-            throw new ScriptException($result['message']);
+            throw new ScriptException(json_encode($result, JSON_PRETTY_PRINT));
         }
     }
 }

ProcessMaker/Multitenancy/SwitchTenant.php

@@ -70,7 +70,7 @@ private function overrideConfigs(Application $app, IsTenant $tenant)
 
         if (!isset($tenant->config['app.docker_host_url'])) {
             // There is no specific override in the tenant's config so set it to the app url
-            $newConfig['app.docker_host_url'] = config('app.url');
+            $newConfig['app.docker_host_url'] = config('app.docker_host_url', config('app.url'));
         }
 
         // Set config from the entry in the tenants table

ProcessMaker/Multitenancy/TenantBootstrapper.php

@@ -2,6 +2,7 @@
 
 namespace ProcessMaker\Multitenancy;
 
+use Dotenv\Dotenv;
 use Illuminate\Encryption\Encrypter;
 use Illuminate\Http\Request;
 use Illuminate\Support\Env;
@@ -34,6 +35,7 @@ class TenantBootstrapper
         'REDIS_PREFIX',
         'CACHE_SETTING_PREFIX',
         'SCRIPT_MICROSERVICE_CALLBACK',
+        'CACHE_PREFIX',
     ];
 
     public function bootstrap(Application $app)
@@ -43,8 +45,6 @@ public function bootstrap(Application $app)
         }
         $this->app = $app;
 
-        self::saveLandlordValues($app);
-
         $tenantData = null;
 
         // Try to find tenant by ID first if TENANT env var is set
@@ -86,6 +86,7 @@ private function setTenantEnvironmentVariables($tenantData)
         $this->set('APP_KEY', $this->decrypt($config['app.key']));
         $this->set('DB_DATABASE', $tenantData['database']);
         $this->set('DB_USERNAME', $tenantData['username'] ?? $this->getOriginalValue('DB_USERNAME'));
+        $this->set('CACHE_PREFIX', $this->getOriginalValue('CACHE_PREFIX') . 'tenant_' . $tenantData['id'] . ':');
 
         $encryptedPassword = $tenantData['password'];
         $password = null;
@@ -99,21 +100,12 @@ private function setTenantEnvironmentVariables($tenantData)
         $this->set('LOG_PATH', $this->app->storagePath('logs/processmaker.log'));
     }
 
-    public static function saveLandlordValues($app)
+    private function getOriginalValue($key)
     {
-        if ($app->has('landlordValues')) {
-            self::$landlordValues = $app->make('landlordValues');
-
-            return;
+        if (self::$landlordValues === []) {
+            self::$landlordValues = Dotenv::parse(file_get_contents(base_path('.env')));
         }
 
-        foreach (self::$landlordKeysToSave as $key) {
-            self::$landlordValues[$key] = $_SERVER[$key] ?? '';
-        }
-    }
-
-    private function getOriginalValue($key)
-    {
         if (!isset(self::$landlordValues[$key])) {
             return '';
         }

ProcessMaker/Services/MetricsService.php

@@ -3,13 +3,18 @@
 namespace ProcessMaker\Services;
 
 use Exception;
+use Laravel\Horizon\Contracts\JobRepository;
+use Laravel\Horizon\Contracts\MetricsRepository;
+use Laravel\Horizon\Contracts\WorkloadRepository;
 use ProcessMaker\Facades\Metrics;
+use ProcessMaker\Multitenancy\Tenant;
 use Prometheus\CollectorRegistry;
 use Prometheus\Counter;
 use Prometheus\Gauge;
 use Prometheus\Histogram;
 use Prometheus\RenderTextFormat;
-use Prometheus\Storage\Redis;
+use Prometheus\Storage\Redis as PrometheusRedis;
+use Redis;
 use RuntimeException;
 
 class MetricsService
@@ -39,7 +44,12 @@ public function __construct(private $adapter = null)
         try {
             // Set up Redis as the adapter if none is provided
             if ($adapter === null) {
-                $adapter = Redis::fromExistingConnection(app('redis')->client());
+                $redis = app('redis')->client();
+                $adapter = PrometheusRedis::fromExistingConnection($redis);
+                if (app()->has(Tenant::BOOTSTRAPPED_TENANT)) {
+                    $tenantInfo = app(Tenant::BOOTSTRAPPED_TENANT);
+                    $adapter->setPrefix('tenant_' . $tenantInfo['id'] . ':PROMETHEUS_');
+                }
             }
             $this->collectionRegistry = new CollectorRegistry($adapter);
         } catch (Exception $e) {
@@ -65,7 +75,7 @@ public function getCollectionRegistry(): CollectorRegistry
      * @param array $labels The labels of the counter.
      * @return Counter The registered or retrieved counter.
      */
-    public function counter(string $name, string $help = null, array $labels = []): Counter
+    public function counter(string $name, string|null $help = null, array $labels = []): Counter
     {
         $help = $help ?? $name;
 
@@ -85,7 +95,7 @@ public function counter(string $name, string $help = null, array $labels = []):
      * @param array $labels The labels of the gauge.
      * @return Gauge The registered or retrieved gauge.
      */
-    public function gauge(string $name, string $help = null, array $labels = []): Gauge
+    public function gauge(string $name, string|null $help = null, array $labels = []): Gauge
     {
         $help = $help ?? $name;
 
@@ -201,7 +211,9 @@ public function addSystemLabels(array $labels)
         // Add system labels
         $labels['app_version'] = $this->getApplicationVersion();
         $labels['app_name'] = config('app.name');
-        $labels['app_custom_label'] = config('app.prometheus_custom_label');
+        if (config('app.prometheus_custom_label')) {
+            $labels['app_custom_label'] = config('app.prometheus_custom_label');
+        }
 
         return $labels;
     }
@@ -223,4 +235,29 @@ private function getApplicationVersion()
 
         return $composer_json_path->version ?? '4.0.0';
     }
+
+    /**
+     * These are collected every time the /metrics route is accessed.
+     *
+     * @return void
+     */
+    public function collectQueueMetrics(): void
+    {
+        $metricsRepository = app(MetricsRepository::class);
+        $jobsRepository = app(JobRepository::class);
+        $workloadRepository = app(WorkloadRepository::class);
+
+        $this->gauge('horizon_jobs_per_minute', 'Jobs processed per minute')->set($metricsRepository->jobsProcessedPerMinute());
+        $this->gauge('horizon_failed_jobs_per_hour', 'Failed jobs per hour')->set($jobsRepository->countRecentlyFailed());
+
+        foreach ($workloadRepository->get() as $workload) {
+            $name = $workload['name'];
+            foreach (['length', 'wait', 'processes'] as $type) {
+                $this->gauge(
+                    'horizon_workload_' . $name . '_' . $type,
+                    'Workload ' . $name . ' ' . $type
+                )->set($workload[$type]);
+            }
+        }
+    }
 }

config/multitenancy.php

@@ -34,7 +34,6 @@
      */
     'switch_tenant_tasks' => [
         SwitchTenant::class,
-        Spatie\Multitenancy\Tasks\PrefixCacheTask::class,
     ],
 
     /*

routes/web.php

@@ -257,6 +257,10 @@
 
 // Metrics Route
 Route::get('/metrics', function () {
+    if (!config('app.multitenancy')) {
+        Metrics::collectQueueMetrics();
+    }
+
     return response(Metrics::renderMetrics(), 200, [
         'Content-Type' => 'text/plain; version=0.0.4',
     ]);