diff --git a/ProcessMaker/Http/Controllers/Admin/QueuesController.php b/ProcessMaker/Http/Controllers/Admin/QueuesController.php index b156f05b7c..0dc78b5369 100644 --- a/ProcessMaker/Http/Controllers/Admin/QueuesController.php +++ b/ProcessMaker/Http/Controllers/Admin/QueuesController.php @@ -5,18 +5,28 @@ use Illuminate\Auth\Access\AuthorizationException; use ProcessMaker\Events\QueueManagementAccessed; use ProcessMaker\Http\Controllers\Controller; +use ProcessMaker\Providers\TenantQueueServiceProvider; class QueuesController extends Controller { public function index() { - if (auth()->user()->is_administrator) { - // Register the Event - QueueManagementAccessed::dispatch(); + if (!auth()->user()->is_administrator) { + throw new AuthorizationException(); + } - return view('admin.queues.index'); + if (config('app.multitenancy')) { + if (!TenantQueueServiceProvider::allowAllTenats()) { + // Its multitenancy and they don't have access to all tenants so + // redirect to the tenant-filtered queue management page. + // Otherwise, show the horizon queue manager. + return redirect()->route('tenant-queue.index'); + } } - throw new AuthorizationException(); + // Register the Event + QueueManagementAccessed::dispatch(); + + return view('admin.queues.index'); } } diff --git a/ProcessMaker/Http/Controllers/Admin/TenantQueueController.php b/ProcessMaker/Http/Controllers/Admin/TenantQueueController.php index 5635224eda..008a3495fd 100644 --- a/ProcessMaker/Http/Controllers/Admin/TenantQueueController.php +++ b/ProcessMaker/Http/Controllers/Admin/TenantQueueController.php @@ -20,13 +20,20 @@ class TenantQueueController extends Controller public function __construct() { // Check if tenant job tracking is enabled - $enabled = config('queue.tenant_tracking_enabled', false); + $enabled = TenantQueueServiceProvider::enabled(); if (!$enabled) { if (!app()->runningInConsole()) { abort(404, 'Tenant queue tracking is disabled'); } } + + // If the route binding has a tenant id, check if the user is allowed to access the tenant queue + if ($id = (int) request()->route('tenantId')) { + if (!TenantQueueServiceProvider::allowAllTenats() && $id !== app('currentTenant')?->id) { + throw new AuthorizationException(); + } + } } /** @@ -52,6 +59,12 @@ public function getTenants(): JsonResponse $tenantsWithJobs = TenantQueueServiceProvider::getTenantsWithJobs(); + if (!TenantQueueServiceProvider::allowAllTenats()) { + $tenantsWithJobs = array_filter($tenantsWithJobs, function ($tenantData) { + return (int) $tenantData['id'] === app('currentTenant')?->id; + }); + } + // Enrich with tenant information $tenants = []; foreach ($tenantsWithJobs as $tenantData) { @@ -118,6 +131,12 @@ public function getOverallStats(): JsonResponse $tenantsWithJobs = TenantQueueServiceProvider::getTenantsWithJobs(); + if (!TenantQueueServiceProvider::allowAllTenats()) { + $tenantsWithJobs = array_filter($tenantsWithJobs, function ($tenantData) { + return (int) $tenantData['id'] === app('currentTenant')?->id; + }); + } + $overallStats = [ 'total_tenants' => count($tenantsWithJobs), 'total_jobs' => 0, diff --git a/ProcessMaker/Multitenancy/MakeQueueTenantAwareAction.php b/ProcessMaker/Multitenancy/MakeQueueTenantAwareAction.php index bfb3ff10a5..25118798fd 100644 --- a/ProcessMaker/Multitenancy/MakeQueueTenantAwareAction.php +++ b/ProcessMaker/Multitenancy/MakeQueueTenantAwareAction.php @@ -2,26 +2,16 @@ namespace ProcessMaker\Multitenancy; -use Illuminate\Queue\Events\JobProcessing; -use Illuminate\Queue\Events\JobRetryRequested; -use Illuminate\Support\Facades\Context; use Spatie\Multitenancy\Actions\MakeQueueTenantAwareAction as BaseMakeQueueTenantAwareAction; class MakeQueueTenantAwareAction extends BaseMakeQueueTenantAwareAction { - /* - * We need to override this method because spatie will throw an error if the tenant is not found. - * However, we want to support non-multitenant instances. If the tenant is not found, - * run the job without a tenant. - */ - protected function bindOrForgetCurrentTenant(JobProcessing|JobRetryRequested $event): void + /** + * We're handling tenant aware queues manually, however, we still need to implement this because for some + * reason the Spatie package calls it in Multitenancy::start(), weather it's a configured action or not. + */ + public function execute() : void { - $tenantId = Context::get($this->currentTenantContextKey()); - if (!$tenantId) { - // No need to do anything. Let the job run without a tenant. - return; - } - - parent::bindOrForgetCurrentTenant($event); + // Do nothing } } diff --git a/ProcessMaker/Multitenancy/TenantAwareDispatcher.php b/ProcessMaker/Multitenancy/TenantAwareDispatcher.php deleted file mode 100644 index 62a20787c6..0000000000 --- a/ProcessMaker/Multitenancy/TenantAwareDispatcher.php +++ /dev/null @@ -1,31 +0,0 @@ -queueResolver); // we need to pass the queueResolver - $this->tenantId = $tenantId; - } - - public function dispatchToQueue($command) - { - $queue = $command->queue; - - // We need to set the default queue here - // because prepending the tenant id means - // it will no longer be empty. - if (empty($queue)) { - $queue = 'default'; - } - $command->queue = 'tenant-' . $this->tenantId . '-' . $queue; - - return parent::dispatchToQueue($command); - } -} diff --git a/ProcessMaker/Multitenancy/TenantBootstrapper.php b/ProcessMaker/Multitenancy/TenantBootstrapper.php index 8e874a8da4..44897c4763 100644 --- a/ProcessMaker/Multitenancy/TenantBootstrapper.php +++ b/ProcessMaker/Multitenancy/TenantBootstrapper.php @@ -17,20 +17,33 @@ */ class TenantBootstrapper { + private static $landlordValues = []; + private $encrypter = null; private $pdo = null; - private $originalValues = null; + private $app = null; + + public static $landlordKeysToSave = [ + 'APP_URL', + 'APP_KEY', + 'LOG_PATH', + 'DB_USERNAME', + 'DB_PASSWORD', + 'REDIS_PREFIX', + 'CACHE_SETTING_PREFIX', + 'SCRIPT_MICROSERVICE_CALLBACK', + ]; public function bootstrap(Application $app) { if (!$this->env('MULTITENANCY')) { return; } + $this->app = $app; - // We need to save the original values for running horizon - $this->saveOriginalValues(); + self::saveLandlordValues($app); $tenantData = null; @@ -49,26 +62,26 @@ public function bootstrap(Application $app) return; } - $this->setTenantEnvironmentVariables($app, $tenantData); + + // Set storage path + $app->useStoragePath($app->basePath('storage/tenant_' . $tenantData['id'])); + + $this->setTenantEnvironmentVariables($tenantData); // Use tenant's translation files. Doing this here so it's available in cached filesystems.php $app->useLangPath(resource_path('lang/tenant_' . $tenantData['id'])); - $tenantData['original_values'] = $this->getOriginalValue(); + $tenantData['original_values'] = self::$landlordValues; Tenant::setBootstrappedTenant($app, $tenantData); } - private function setTenantEnvironmentVariables($app, $tenantData) + private function setTenantEnvironmentVariables($tenantData) { // Additional configs are set in SwitchTenant.php - $tenantId = $tenantData['id']; $config = json_decode($tenantData['config'], true); - $this->set('APP_CONFIG_CACHE', $app->basePath('storage/tenant_' . $tenantId . '/config.php')); - // Do not override packages cache path for now. Wait until the License service is updated. - // $this->set('APP_PACKAGES_CACHE', $app->basePath('storage/tenant_' . $tenantId . '/packages.php')); - $this->set('LARAVEL_STORAGE_PATH', $app->basePath('storage/tenant_' . $tenantId)); + $this->set('APP_CONFIG_CACHE', $this->app->storagePath('config.php')); $this->set('APP_URL', $config['app.url']); $this->set('APP_KEY', $this->decrypt($config['app.key'])); $this->set('DB_DATABASE', $tenantData['database']); @@ -83,51 +96,48 @@ private function setTenantEnvironmentVariables($app, $tenantData) } $this->set('DB_PASSWORD', $password); - $this->set('REDIS_PREFIX', $this->getOriginalValue('REDIS_PREFIX') . 'tenant-' . $tenantId . ':'); - $this->set('LOG_PATH', $app->basePath('storage/tenant_' . $tenantId . '/logs/processmaker.log')); + $this->set('LOG_PATH', $this->app->storagePath('logs/processmaker.log')); } - private function saveOriginalValues() + public static function saveLandlordValues($app) { - if ($this->env('ORIGINAL_VALUES')) { + if ($app->has('landlordValues')) { + self::$landlordValues = $app->make('landlordValues'); + return; } - $toSave = [ - 'APP_URL', - 'APP_KEY', - 'DB_USERNAME', - 'DB_PASSWORD', - 'REDIS_PREFIX', - 'CACHE_SETTING_PREFIX', - 'SCRIPT_MICROSERVICE_CALLBACK', - ]; - $values = []; - foreach ($toSave as $key) { - $values[$key] = $this->env($key); + + foreach (self::$landlordKeysToSave as $key) { + self::$landlordValues[$key] = $_SERVER[$key] ?? ''; } - $this->set('ORIGINAL_VALUES', serialize($values)); } - private function getOriginalValue($key = null) + private function getOriginalValue($key) { - if (!$this->originalValues) { - $this->originalValues = unserialize($this->env('ORIGINAL_VALUES')); - } - if (!$key) { - return $this->originalValues; + if (!isset(self::$landlordValues[$key])) { + return ''; } - return $this->originalValues[$key]; + return self::$landlordValues[$key]; } private function env($key, $default = null) { - return Env::get($key, $default); + $value = $_SERVER[$key] ?? $default; + if ($value === 'true') { + $value = true; + } elseif ($value === 'false') { + $value = false; + } + + return $value; } private function set($key, $value) { - Env::getRepository()->set($key, $value); + // Env::getRepository() is immutable but will use values from $_SERVER and $_ENV + $_SERVER[$key] = $value; + $_ENV[$key] = $value; } private function decrypt($value) diff --git a/ProcessMaker/Providers/ProcessMakerServiceProvider.php b/ProcessMaker/Providers/ProcessMakerServiceProvider.php index fd93a0b222..571eab6629 100644 --- a/ProcessMaker/Providers/ProcessMakerServiceProvider.php +++ b/ProcessMaker/Providers/ProcessMakerServiceProvider.php @@ -9,9 +9,13 @@ use Illuminate\Foundation\Support\Providers\EventServiceProvider as ServiceProvider; use Illuminate\Notifications\Events\BroadcastNotificationCreated; use Illuminate\Notifications\Events\NotificationSent; +use Illuminate\Queue\Events\JobAttempted; +use Illuminate\Queue\Events\JobProcessing; +use Illuminate\Queue\Events\JobRetryRequested; use Illuminate\Support\Arr; use Illuminate\Support\Env; use Illuminate\Support\Facades; +use Illuminate\Support\Facades\Context; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Route; @@ -37,6 +41,7 @@ use ProcessMaker\Managers\ScreenCompiledManager; use ProcessMaker\Models; use ProcessMaker\Multitenancy\Tenant; +use ProcessMaker\Multitenancy\TenantBootstrapper; use ProcessMaker\Observers; use ProcessMaker\PolicyExtension; use ProcessMaker\Repositories\SettingsConfigRepository; @@ -62,6 +67,9 @@ class ProcessMakerServiceProvider extends ServiceProvider // Track the query time for each request private static $queryTime = 0; + // Track the landlord values for multitenancy + private static $landlordValues = null; + public function boot(): void { // Track the start time for service providers boot @@ -228,11 +236,78 @@ public function register(): void $this->app->instance('tenant-resolved', false); } + /** + * In multitenancy, we need to bootstrap a new app with the tenant id set. + * This is because queue workers are long-running processes that are not + * tenant aware. + */ + private static function bootstrapTenantApp(JobProcessing|JobRetryRequested $event): void + { + Context::hydrate($event->job->payload()['illuminate:log:context'] ?? null); + $tenantId = Context::get(config('multitenancy.current_tenant_context_key')); + if ($tenantId) { + if (!method_exists($event->job, 'getRedisQueue')) { + // Not a redis job + return; + } + + // Save the landlord's config values so we can reset them later + if (self::$landlordValues === null) { + foreach (TenantBootstrapper::$landlordKeysToSave as $key) { + self::$landlordValues[$key] = $_SERVER[$key] ?? ''; + } + } + + // Create a new tenant app instance + $_SERVER['TENANT'] = $tenantId; + $_ENV['TENANT'] = $tenantId; + $tenantApp = require app()->bootstrapPath('app.php'); + $tenantApp->instance('landlordValues', self::$landlordValues); + $tenantApp->make(\Illuminate\Contracts\Console\Kernel::class)->bootstrap(); + + // Change the job's app service container to the tenant app + $event->job->getRedisQueue()->setContainer($tenantApp); + } + } + + private static function resetTenantApp($event): void + { + if (!method_exists($event->job, 'getRedisQueue')) { + // Not a redis job + return; + } + + unset($_SERVER['TENANT']); + unset($_ENV['TENANT']); + + if (!self::$landlordValues) { + return; + } + + // Restore the original values since the tenant boostrapper modified them + foreach (self::$landlordValues as $key => $value) { + $_SERVER[$key] = $value; + $_ENV[$key] = $value; + } + } + /** * Register app-level events. */ protected static function registerEvents(): void { + Facades\Event::listen(JobProcessing::class, function (JobProcessing $event) { + self::bootstrapTenantApp($event); + }); + + Facades\Event::listen(JobRetryRequested::class, function (JobRetryRequested $event) { + self::bootstrapTenantApp($event); + }); + + Facades\Event::listen(JobAttempted::class, function (JobAttempted $event) { + self::resetTenantApp($event); + }); + // Listen to the events for our core screen // types and add our javascript Facades\Event::listen(ScreenBuilderStarting::class, function ($event) { diff --git a/ProcessMaker/Providers/TenantQueueServiceProvider.php b/ProcessMaker/Providers/TenantQueueServiceProvider.php index 60bd89c45b..544650b22f 100644 --- a/ProcessMaker/Providers/TenantQueueServiceProvider.php +++ b/ProcessMaker/Providers/TenantQueueServiceProvider.php @@ -6,10 +6,14 @@ use Illuminate\Queue\Events\JobFailed; use Illuminate\Queue\Events\JobProcessed; use Illuminate\Queue\Events\JobProcessing; +use Illuminate\Queue\Events\JobQueued; +use Illuminate\Queue\Events\JobQueueing; +use Illuminate\Support\Facades\Event; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Queue; use Illuminate\Support\Facades\Redis; use Illuminate\Support\ServiceProvider; +use Laravel\Horizon\Events\JobPushed; class TenantQueueServiceProvider extends ServiceProvider { @@ -27,9 +31,7 @@ public function register(): void public function boot(): void { // Only register queue event listeners if tenant job tracking is enabled - $enabled = config('queue.tenant_tracking_enabled', false); - - if ($enabled) { + if (self::enabled()) { $this->registerQueueEventListeners(); } } @@ -39,6 +41,10 @@ public function boot(): void */ protected function registerQueueEventListeners(): void { + Event::listen(JobQueued::class, function (JobQueued $event) { + $this->trackJobByTenant($event, 'pushed', $event->id, $event->queue ?? 'default'); + }); + // Job pending Queue::before(function (JobProcessing $event) { $this->trackJobByTenant($event->job, 'pending'); @@ -63,7 +69,7 @@ protected function registerQueueEventListeners(): void /** * Track job by tenant in Redis. */ - protected function trackJobByTenant($job, string $status): void + protected function trackJobByTenant($job, string $status, $jobId = null, $queue = null): void { try { $tenantId = $this->extractTenantIdFromJob($job); @@ -72,7 +78,7 @@ protected function trackJobByTenant($job, string $status): void return; } - $jobId = $job->getJobId(); + $jobId = $jobId ?? $job->getJobId(); if (!$jobId) { return; } @@ -82,14 +88,14 @@ protected function trackJobByTenant($job, string $status): void $jobData = [ 'id' => $jobId, 'name' => $this->getJobName($job), - 'queue' => $job->getQueue() ?? 'default', + 'queue' => ($queue ?? $job->getQueue()) ?? 'default', 'status' => $status, - 'created_at' => $payload['pushedAt'], + 'created_at' => $status === 'pushed' ? str_replace(',', '.', microtime(true)) : $payload['pushedAt'], 'completed_at' => $status === 'completed' ? str_replace(',', '.', microtime(true)) : null, 'failed_at' => $status === 'failed' ? str_replace(',', '.', microtime(true)) : null, 'updated_at' => str_replace(',', '.', microtime(true)), 'tenant_id' => $tenantId, - 'attempts' => $job->attempts(), + 'attempts' => $status === 'pushed' ? 0 : $job->attempts(), 'payload' => json_encode($payload['data']), ]; @@ -97,6 +103,10 @@ protected function trackJobByTenant($job, string $status): void $jobData['queued_at'] = str_replace(',', '.', microtime(true)); } + if ($status === 'pushed') { + $jobData['pushed_at'] = str_replace(',', '.', microtime(true)); + } + // Check if job already exists $tenantKey = "tenant_jobs:{$tenantId}:{$jobId}"; $existingJobData = Redis::get($tenantKey); @@ -230,7 +240,7 @@ protected function updateTenantJobCounters(string $tenantId, string $status, str */ protected function handleStatusTransition(string $tenantId, string $jobId, string $newStatus): void { - $statuses = ['pending', 'completed', 'failed', 'exception']; + $statuses = ['pushed', 'pending', 'completed', 'failed', 'exception']; foreach ($statuses as $status) { if ($status === $newStatus) { @@ -306,7 +316,7 @@ public static function getTenantJobs(string $tenantId, string $status = null, in $jobIds = Redis::lrange($listKey, 0, $limit - 1); } else { // Get jobs from all statuses - $statuses = ['pending', 'completed', 'failed', 'exception']; + $statuses = ['pushed', 'pending', 'completed', 'failed', 'exception']; $jobIds = []; foreach ($statuses as $status) { @@ -345,6 +355,7 @@ public static function getTenantJobStats(string $tenantId): array return [ 'total' => (int) ($counters['total'] ?? 0), + 'pushed' => (int) ($counters['pushed'] ?? 0), 'pending' => (int) ($counters['pending'] ?? 0), 'completed' => (int) ($counters['completed'] ?? 0), 'failed' => (int) ($counters['failed'] ?? 0), @@ -362,7 +373,7 @@ public static function getTenantsWithJobs(): array $tenants = []; foreach ($keys as $key) { - $tenantId = str_replace('tenant_job_counters:', '', $key); + $tenantId = preg_replace('/^.*:/', '', $key); $stats = self::getTenantJobStats($tenantId); if ($stats['total'] > 0) { @@ -375,4 +386,23 @@ public static function getTenantsWithJobs(): array return $tenants; } + + public static function enabled(): bool + { + if (!config('app.multitenancy')) { + return false; + } + + return !config('queue.disable_tenant_tracking'); + } + + public static function allowAllTenats(): bool + { + if (!config('app.multitenancy')) { + // Only restricted in multitenant environments + return true; + } + + return config('queue.ui_allow_all_tenants'); + } } diff --git a/config/queue.php b/config/queue.php index 74a0c5d7e1..c5029a5baa 100644 --- a/config/queue.php +++ b/config/queue.php @@ -96,11 +96,23 @@ |-------------------------------------------------------------------------- | | These options configure the behavior of tenant-specific job tracking. + | Job tracking is enabled by default when multitenancy is enabled. | When enabled, jobs will be tracked per tenant in Redis for monitoring | and analytics purposes. | + | Set this to true to disable job tracking for all tenants. */ - 'tenant_tracking_enabled' => env('QUEUE_TENANT_TRACKING_ENABLED', false), + 'disable_tenant_tracking' => env('QUEUE_DISABLE_TENANT_TRACKING', false), + /* + |-------------------------------------------------------------------------- + | Restrict Queue UI to Tenant + |-------------------------------------------------------------------------- + | + | Allow viewing queue info for all tenants in a multitenant environment. + | Only enable for debugging! + | + */ + 'ui_allow_all_tenants' => env('QUEUE_UI_ALLOW_ALL_TENANTS', false), ]; diff --git a/resources/js/admin/tenant-queues/JobDetails.vue b/resources/js/admin/tenant-queues/JobDetails.vue index 66bd143c11..5c33f1ad0a 100644 --- a/resources/js/admin/tenant-queues/JobDetails.vue +++ b/resources/js/admin/tenant-queues/JobDetails.vue @@ -104,6 +104,13 @@ {{ job.attempts }} +
+ Pushed At: +
+
+ {{ formatTimestamp(job.pushed_at) }} +
+
Queued At:
diff --git a/resources/js/admin/tenant-queues/TenantJobs.vue b/resources/js/admin/tenant-queues/TenantJobs.vue index c96a807474..67c7d09fd1 100644 --- a/resources/js/admin/tenant-queues/TenantJobs.vue +++ b/resources/js/admin/tenant-queues/TenantJobs.vue @@ -6,7 +6,7 @@ - -