EPG Mapping error

[susnick]

Here is message as in log. If you require something else let me know. Version 0.5.8

SQLSTATE[HY000]: General error: 5 database is locked (Connection: sqlite, SQL: insert into "failed_jobs" ("uuid", "connection", "queue", "payload", "exception", "failed_at") values (44d8ce06-6a24-4d10-87c0-35d270fff3a4, redis, default, {"backoff":null,"uuid":"44d8ce06-6a24-4d10-87c0-35d270fff3a4","timeout":900,"tags":{},"id":"44d8ce06-6a24-4d10-87c0-35d270fff3a4","job":"Illuminate\\Queue\\CallQueuedHandler@call","pushedAt":"1746705621.3812","maxExceptions":null,"maxTries":null,"data":{"command":"O:33:\"App\\Jobs\\MapPlaylistChannelsToEpg\":7:{s:19:\"\u0000*\u0000similaritySearch\";O:36:\"App\\Services\\SimilaritySearchService\":4:{s:56:\"\u0000App\\Services\\SimilaritySearchService\u0000bestFuzzyThreshold\";i:40;s:57:\"\u0000App\\Services\\SimilaritySearchService\u0000upperFuzzyThreshold\";i:70;s:55:\"\u0000App\\Services\\SimilaritySearchService\u0000embedSimThreshold\";d:0.65;s:47:\"\u0000App\\Services\\SimilaritySearchService\u0000stopWords\";a:21:{i:0;s:2:\"tv\";i:1;s:7:\"channel\";i:2;s:7:\"network\";i:3;s:10:\"television\";i:4;s:4:\"east\";i:5;s:4:\"west\";i:6;s:2:\"hd\";i:7;s:3:\"uhd\";i:8;s:2:\"us\";i:9;s:3:\"usa\";i:10;s:3:\"not\";i:11;s:4:\"24\/7\";i:12;s:5:\"1080p\";i:13;s:4:\"720p\";i:14;s:4:\"540p\";i:15;s:4:\"480p\";i:16;s:6:\"arabic\";i:17;s:6:\"latino\";i:18;s:4:\"film\";i:19;s:5:\"movie\";i:20;s:6:\"movies\";}}s:3:\"epg\";i:1;s:8:\"playlist\";i:4;s:8:\"channels\";N;s:5:\"force\";b:0;s:9:\"recurring\";b:0;s:8:\"epgMapId\";i:4;}","commandName":"App\\Jobs\\MapPlaylistChannelsToEpg"},"failOnTimeout":false,"silenced":false,"retryUntil":null,"type":"job","displayName":"App\\Jobs\\MapPlaylistChannelsToEpg","attempts":2}, Illuminate\Queue\TimeoutExceededException: App\Jobs\MapPlaylistChannelsToEpg has timed out. in /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/TimeoutExceededException.php:15 Stack trace: #0 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(805): Illuminate\Queue\TimeoutExceededException::forJob() #1 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(219): Illuminate\Queue\Worker->timeoutExceededException() #2 [internal function]: Illuminate\Queue\Worker->{closure:Illuminate\Queue\Worker::registerTimeoutHandler():216}() #3 /var/www/html/vendor/laravel/framework/src/Illuminate/Support/Pluralizer.php(94): ucfirst() #4 /var/www/html/vendor/laravel/framework/src/Illuminate/Support/Pluralizer.php(54): Illuminate\Support\Pluralizer::matchCase() #5 /var/www/html/vendor/laravel/framework/src/Illuminate/Support/Str.php(963): Illuminate\Support\Pluralizer::plural() #6 /var/www/html/vendor/laravel/framework/src/Illuminate/Support/Str.php(979): Illuminate\Support\Str::plural() #7 /var/www/html/vendor/laravel/framework/src/Illuminate/Database/Eloquent/Model.php(1892): Illuminate\Support\Str::pluralStudly() #8 /var/www/html/vendor/laravel/framework/src/Illuminate/Database/Eloquent/Model.php(629): Illuminate\Database\Eloquent\Model->getTable() #9 /var/www/html/vendor/laravel/framework/src/Illuminate/Database/Eloquent/Builder.php(1655): Illuminate\Database\Eloquent\Model->newInstance() #10 /var/www/html/vendor/laravel/framework/src/Illuminate/Database/Eloquent/Builder.php(966): Illuminate\Database\Eloquent\Builder->newModelInstance() #11 /var/www/html/vendor/laravel/framework/src/Illuminate/Collections/LazyCollection.php(805): Illuminate\Database\Eloquent\Builder->{closure:Illuminate\Database\Eloquent\Builder::cursor():963}() #12 /var/www/html/vendor/laravel/framework/src/Illuminate/Collections/LazyCollection.php(449): Illuminate\Support\LazyCollection->{closure:Illuminate\Support\LazyCollection::map():803}() #13 /var/www/html/app/Services/SimilaritySearchService.php(84): Illuminate\Support\LazyCollection->{closure:Illuminate\Support\LazyCollection::filter():448}() #14 /var/www/html/app/Jobs/MapPlaylistChannelsToEpg.php(133): App\Services\SimilaritySearchService->findMatchingEpgChannel() #15 [internal function]: App\Jobs\MapPlaylistChannelsToEpg->{closure:App\Jobs\MapPlaylistChannelsToEpg::handle():116}() #16 /var/www/html/vendor/laravel/framework/src/Illuminate/Collections/LazyCollection.php(1392): Generator->next() #17 /var/www/html/vendor/laravel/framework/src/Illuminate/Collections/Traits/EnumeratesValues.php(260): Illuminate\Support\LazyCollection->{closure:Illuminate\Support\LazyCollection::chunk():1382}() #18 /var/www/html/app/Jobs/MapPlaylistChannelsToEpg.php(149): Illuminate\Support\LazyCollection->each() #19 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(36): App\Jobs\MapPlaylistChannelsToEpg->handle() #20 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/Util.php(43): Illuminate\Container\BoundMethod::{closure:Illuminate\Container\BoundMethod::call():35}() #21 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(95): Illuminate\Container\Util::unwrapIfClosure() #22 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(35): Illuminate\Container\BoundMethod::callBoundMethod() #23 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/Container.php(696): Illuminate\Container\BoundMethod::call() #24 /var/www/html/vendor/laravel/framework/src/Illuminate/Bus/Dispatcher.php(126): Illuminate\Container\Container->call() #25 /var/www/html/vendor/laravel/framework/src/Illuminate/Pipeline/Pipeline.php(170): Illuminate\Bus\Dispatcher->{closure:Illuminate\Bus\Dispatcher::dispatchNow():123}() #26 /var/www/html/vendor/laravel/framework/src/Illuminate/Pipeline/Pipeline.php(127): Illuminate\Pipeline\Pipeline->{closure:Illuminate\Pipeline\Pipeline::prepareDestination():168}() #27 /var/www/html/vendor/laravel/framework/src/Illuminate/Bus/Dispatcher.php(130): Illuminate\Pipeline\Pipeline->then() #28 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/CallQueuedHandler.php(126): Illuminate\Bus\Dispatcher->dispatchNow() #29 /var/www/html/vendor/laravel/framework/src/Illuminate/Pipeline/Pipeline.php(170): Illuminate\Queue\CallQueuedHandler->{closure:Illuminate\Queue\CallQueuedHandler::dispatchThroughMiddleware():121}() #30 /var/www/html/vendor/laravel/framework/src/Illuminate/Pipeline/Pipeline.php(127): Illuminate\Pipeline\Pipeline->{closure:Illuminate\Pipeline\Pipeline::prepareDestination():168}() #31 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/CallQueuedHandler.php(121): Illuminate\Pipeline\Pipeline->then() #32 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/CallQueuedHandler.php(69): Illuminate\Queue\CallQueuedHandler->dispatchThroughMiddleware() #33 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Jobs/Job.php(102): Illuminate\Queue\CallQueuedHandler->call() #34 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(442): Illuminate\Queue\Jobs\Job->fire() #35 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(392): Illuminate\Queue\Worker->process() #36 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(178): Illuminate\Queue\Worker->runJob() #37 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Console/WorkCommand.php(149): Illuminate\Queue\Worker->daemon() #38 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Console/WorkCommand.php(132): Illuminate\Queue\Console\WorkCommand->runWorker() #39 /var/www/html/vendor/laravel/horizon/src/Console/WorkCommand.php(51): Illuminate\Queue\Console\WorkCommand->handle() #40 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(36): Laravel\Horizon\Console\WorkCommand->handle() #41 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/Util.php(43): Illuminate\Container\BoundMethod::{closure:Illuminate\Container\BoundMethod::call():35}() #42 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(95): Illuminate\Container\Util::unwrapIfClosure() #43 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(35): Illuminate\Container\BoundMethod::callBoundMethod() #44 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/Container.php(696): Illuminate\Container\BoundMethod::call() #45 /var/www/html/vendor/laravel/framework/src/Illuminate/Console/Command.php(213): Illuminate\Container\Container->call() #46 /var/www/html/vendor/symfony/console/Command/Command.php(279): Illuminate\Console\Command->execute() #47 /var/www/html/vendor/laravel/framework/src/Illuminate/Console/Command.php(182): Symfony\Component\Console\Command\Command->run() #48 /var/www/html/vendor/symfony/console/Application.php(1094): Illuminate\Console\Command->run() #49 /var/www/html/vendor/symfony/console/Application.php(342): Symfony\Component\Console\Application->doRunCommand() #50 /var/www/html/vendor/symfony/console/Application.php(193): Symfony\Component\Console\Application->doRun() #51 /var/www/html/vendor/laravel/framework/src/Illuminate/Foundation/Console/Kernel.php(198): Symfony\Component\Console\Application->run() #52 /var/www/html/vendor/laravel/framework/src/Illuminate/Foundation/Application.php(1235): Illuminate\Foundation\Console\Kernel->handle() #53 /var/www/html/artisan(13): Illuminate\Foundation\Application->handleCommand() #54 {main}, 2025-05-08 12:45:41))

[sparkison]

I see a couple things going on: first, it looks like the DB is locked, which you might be able to fix with the latest dev release. Regardless, it looks like it's timing out. Currently the timeout is 15min for matching, which should be more than enough time.

for the Playlist and EPG you're mapping, whats the channel count for each? And are they from your provider, or is the EPG a non-provider EPG (e.g. playlist and epg are from different sources)?

I can increase the timeout to start, but not sure how much it will take. When it timed out, what percentage was the mapping?

[susnick]

The EPG has 2167 channels. The playlist with the error has 12936 channels. The channels are from provider, they just started offering epg data, but it is horrible. I use IPTVBoss for the EPG data. As for how far along it was, im not sure. The page shows 100% with status "Failed". Will the progress stop where it failed?

EDIT:
Looks like it gets to 2%. Its been sitting there for 15 minutes multiple times. It also looks like updating playlists happens during this time, so maybe those lock the database. Cant wait for you to get mysql working, thats my goto database.

[bod09]

Hey, I'm also having this same issue and it's also stuck at 2%, EPG is from my provider.

Error processing.txt

[hardikdabhi]

I am facing same issue. EPG mapping is stuck at 2%. 194534 channels and 11757 EPG from same provider. I moved to dev branch with Postgres.

[Freekers]

My EPG mapping is also stuck at 2% during processing. I've tried resetting the queues, but it remains stuck. Reprocessing is not possible — the only available option is to delete it.

The log files do not show any errors or warnings, but the Queue Manager displays the following:

#0   /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(805):  Illuminate\Queue\TimeoutExceededException::forJob()
#1   /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(219):  Illuminate\Queue\Worker->timeoutExceededException()
#2 [internal function]: Illuminate\Queue\Worker->{closure:Illuminate\Queue\Worker::registerTimeoutHandler():216}()
#3 /var/www/html/vendor/laravel/framework/src/Illuminate/Database/Eloquent/Concerns/HasAttributes.php(1626): array_merge()
#4   /var/www/html/vendor/laravel/framework/src/Illuminate/Database/Eloquent/Concerns/HasAttributes.php(1611):  Illuminate\Database\Eloquent\Model->getCasts()
#5   /var/www/html/vendor/laravel/framework/src/Illuminate/Database/Eloquent/Concerns/HasAttributes.php(2232):  Illuminate\Database\Eloquent\Model->hasCast()
#6   /var/www/html/vendor/laravel/framework/src/Illuminate/Database/Eloquent/Concerns/HasAttributes.php(520):  Illuminate\Database\Eloquent\Model->transformModelValue()
#7   /var/www/html/vendor/laravel/framework/src/Illuminate/Database/Eloquent/Concerns/HasAttributes.php(474):  Illuminate\Database\Eloquent\Model->getAttributeValue()
#8   /var/www/html/vendor/laravel/framework/src/Illuminate/Database/Eloquent/Model.php(2260):  Illuminate\Database\Eloquent\Model->getAttribute()
#9 /var/www/html/app/Services/SimilaritySearchService.php(87): Illuminate\Database\Eloquent\Model->__get()
#10 /var/www/html/app/Jobs/MapPlaylistChannelsToEpg.php(136): App\Services\SimilaritySearchService->findMatchingEpgChannel()
#11 [internal function]: App\Jobs\MapPlaylistChannelsToEpg->{closure:App\Jobs\MapPlaylistChannelsToEpg::handle():116}()
#12 /var/www/html/vendor/laravel/framework/src/Illuminate/Collections/LazyCollection.php(1392): Generator->next()
#13   /var/www/html/vendor/laravel/framework/src/Illuminate/Collections/Traits/EnumeratesValues.php(260):   Illuminate\Support\LazyCollection->{closure:Illuminate\Support\LazyCollection::chunk():1382}()
#14 /var/www/html/app/Jobs/MapPlaylistChannelsToEpg.php(152): Illuminate\Support\LazyCollection->each()
#15   /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(36):  App\Jobs\MapPlaylistChannelsToEpg->handle()
#16   /var/www/html/vendor/laravel/framework/src/Illuminate/Container/Util.php(43):   Illuminate\Container\BoundMethod::{closure:Illuminate\Container\BoundMethod::call():35}()
#17   /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(95):  Illuminate\Container\Util::unwrapIfClosure()
#18   /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(35):  Illuminate\Container\BoundMethod::callBoundMethod()
#19 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/Container.php(696): Illuminate\Container\BoundMethod::call()
#20 /var/www/html/vendor/laravel/framework/src/Illuminate/Bus/Dispatcher.php(126): Illuminate\Container\Container->call()
#21   /var/www/html/vendor/laravel/framework/src/Illuminate/Pipeline/Pipeline.php(170):   Illuminate\Bus\Dispatcher->{closure:Illuminate\Bus\Dispatcher::dispatchNow():123}()
#22   /var/www/html/vendor/laravel/framework/src/Illuminate/Pipeline/Pipeline.php(127):   Illuminate\Pipeline\Pipeline->{closure:Illuminate\Pipeline\Pipeline::prepareDestination():168}()
#23 /var/www/html/vendor/laravel/framework/src/Illuminate/Bus/Dispatcher.php(130): Illuminate\Pipeline\Pipeline->then()
#24   /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/CallQueuedHandler.php(126):  Illuminate\Bus\Dispatcher->dispatchNow()
#25   /var/www/html/vendor/laravel/framework/src/Illuminate/Pipeline/Pipeline.php(170):   Illuminate\Queue\CallQueuedHandler->{closure:Illuminate\Queue\CallQueuedHandler::dispatchThroughMiddleware():121}()
#26   /var/www/html/vendor/laravel/framework/src/Illuminate/Pipeline/Pipeline.php(127):   Illuminate\Pipeline\Pipeline->{closure:Illuminate\Pipeline\Pipeline::prepareDestination():168}()
#27   /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/CallQueuedHandler.php(121):  Illuminate\Pipeline\Pipeline->then()
#28   /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/CallQueuedHandler.php(69):  Illuminate\Queue\CallQueuedHandler->dispatchThroughMiddleware()
#29 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Jobs/Job.php(102): Illuminate\Queue\CallQueuedHandler->call()
#30 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(442): Illuminate\Queue\Jobs\Job->fire()
#31 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(392): Illuminate\Queue\Worker->process()
#32 /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(178): Illuminate\Queue\Worker->runJob()
#33   /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Console/WorkCommand.php(149):  Illuminate\Queue\Worker->daemon()
#34   /var/www/html/vendor/laravel/framework/src/Illuminate/Queue/Console/WorkCommand.php(132):  Illuminate\Queue\Console\WorkCommand->runWorker()
#35 /var/www/html/vendor/laravel/horizon/src/Console/WorkCommand.php(51): Illuminate\Queue\Console\WorkCommand->handle()
#36   /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(36):  Laravel\Horizon\Console\WorkCommand->handle()
#37   /var/www/html/vendor/laravel/framework/src/Illuminate/Container/Util.php(43):   Illuminate\Container\BoundMethod::{closure:Illuminate\Container\BoundMethod::call():35}()
#38   /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(95):  Illuminate\Container\Util::unwrapIfClosure()
#39   /var/www/html/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(35):  Illuminate\Container\BoundMethod::callBoundMethod()
#40 /var/www/html/vendor/laravel/framework/src/Illuminate/Container/Container.php(696): Illuminate\Container\BoundMethod::call()
#41 /var/www/html/vendor/laravel/framework/src/Illuminate/Console/Command.php(213): Illuminate\Container\Container->call()
#42 /var/www/html/vendor/symfony/console/Command/Command.php(279): Illuminate\Console\Command->execute()
#43   /var/www/html/vendor/laravel/framework/src/Illuminate/Console/Command.php(182):  Symfony\Component\Console\Command\Command->run()
#44 /var/www/html/vendor/symfony/console/Application.php(1094): Illuminate\Console\Command->run()
#45 /var/www/html/vendor/symfony/console/Application.php(342): Symfony\Component\Console\Application->doRunCommand()
#46 /var/www/html/vendor/symfony/console/Application.php(193): Symfony\Component\Console\Application->doRun()
#47   /var/www/html/vendor/laravel/framework/src/Illuminate/Foundation/Console/Kernel.php(198):  Symfony\Component\Console\Application->run()
#48   /var/www/html/vendor/laravel/framework/src/Illuminate/Foundation/Application.php(1235):  Illuminate\Foundation\Console\Kernel->handle()
#49 /var/www/html/artisan(13): Illuminate\Foundation\Application->handleCommand()
#50 {main}

I'm currently running the dev version in combination with the built-in PostgreSQL database.

[sparkison]

It's looking like timeout, the mapping is just taking too long. I'll see if I can split things up so processing goes more quickly.