From dce3b47372e526533a8752a40bee77854bbef39d Mon Sep 17 00:00:00 2001 From: Brett Spaulding Date: Tue, 13 Aug 2024 14:38:01 -0400 Subject: [PATCH] [IMP] Extra handling for missing link case --- .../Console/Commands/ProcessArtistQueue.php | 6 ++--- .../app/Http/Controllers/ApiController.php | 22 ++++++++++--------- php/src/app/Models/AlbumQueue.php | 14 ++++++++---- php/src/app/Models/ArtistQueue.php | 13 ++++++++++- php/src/app/Models/WebScraper.php | 20 ++++++++++++++--- php/src/public/js/app.js | 2 +- .../views/modals/modal-catalog.blade.php | 2 +- 7 files changed, 56 insertions(+), 23 deletions(-) diff --git a/php/src/app/Console/Commands/ProcessArtistQueue.php b/php/src/app/Console/Commands/ProcessArtistQueue.php index 2eb6ef1..f765ab7 100644 --- a/php/src/app/Console/Commands/ProcessArtistQueue.php +++ b/php/src/app/Console/Commands/ProcessArtistQueue.php @@ -28,10 +28,10 @@ class ProcessArtistQueue extends Command public function handle() { // This queue will prompt the scraping of all artist albums, mark done when complete - $artists = ArtistQueue::where('state', 'pending')->get(); - $bar = new ProgressBar($this->output, count($artists)); + $artist_queue = ArtistQueue::where('state', 'pending')->get(); + $bar = new ProgressBar($this->output, count($artist_queue)); $bar->start(); - foreach ($artists as $artist) { + foreach ($artist_queue as $artist) { $artist->state = 'in_progress'; $artist->save(); $artist->process_artist(); diff --git a/php/src/app/Http/Controllers/ApiController.php b/php/src/app/Http/Controllers/ApiController.php index 4c04b72..a86fdae 100644 --- a/php/src/app/Http/Controllers/ApiController.php +++ b/php/src/app/Http/Controllers/ApiController.php @@ -2,7 +2,7 @@ namespace App\Http\Controllers; -use Illuminate\Support\Facades\Artisan; +use App\Jobs\RunArtistQueue; use App\Models\AlbumQueue; use App\Models\Artist; use App\Models\WebDriver; @@ -38,14 +38,16 @@ class ApiController extends Controller foreach ($album_queue as $queue) { $album = $queue->album; $artist = $album->artist; - $response[] = [ - 'name' => $album->name, - 'artist_id' => $artist->toArray(), - 'url_remote' => $album->url_remote, - 'thumbnail' => $album->thumbnail, - 'image' => $album->image, - 'state' => $queue->state, - ]; + if ($album && $artist) { + $response[] = [ + 'name' => $album->name, + 'artist_id' => $artist->toArray(), + 'url_remote' => $album->url_remote, + 'thumbnail' => $album->thumbnail, + 'image' => $album->image, + 'state' => $queue->state, + ]; + } } return json_encode($response); } @@ -59,7 +61,7 @@ class ApiController extends Controller { \Log::info('==========================='); \Log::info('Queue running for Artists..'); - Artisan::queue('app:process-artist-queue'); + ArtistQueue::run_queue(); } public function search_artist(string $artist) diff --git a/php/src/app/Models/AlbumQueue.php b/php/src/app/Models/AlbumQueue.php index e972dc3..13e211e 100644 --- a/php/src/app/Models/AlbumQueue.php +++ b/php/src/app/Models/AlbumQueue.php @@ -9,18 +9,24 @@ class AlbumQueue extends Model { use HasFactory; - public function enqueue($album_id): bool + public function enqueue($album): bool { $result = false; - $album_queued = AlbumQueue::where('album_id', $album_id->id)->first(); - if (is_null($album_queued) && $album_id->state === 'pending') { - $this->album_id = $album_id->id; + $album_queued = AlbumQueue::where('album_id', $album->id)->first(); + if (is_null($album_queued) && $album->state === 'pending') { + $this->album_id = $album->id; $this->save(); $result = true; } return $result; } + public static function addQueue($album_id): bool + { + $queue = new AlbumQueue(); + $queue->enqueue($album_id); + } + public function album() { return $this->belongsTo(Album::class); diff --git a/php/src/app/Models/ArtistQueue.php b/php/src/app/Models/ArtistQueue.php index ca3f7cb..c74d46a 100644 --- a/php/src/app/Models/ArtistQueue.php +++ b/php/src/app/Models/ArtistQueue.php @@ -51,5 +51,16 @@ class ArtistQueue extends Model } } - + public static function run_queue() + { + // This queue will prompt the scraping of all artist albums, mark done when complete + $artist_queue = ArtistQueue::where('state', 'pending')->get(); + foreach ($artist_queue as $queue) { + $queue->state = 'in_progress'; + $queue->save(); + $queue->process_artist(); + $queue->state = 'done'; + $queue->save(); + } + } } diff --git a/php/src/app/Models/WebScraper.php b/php/src/app/Models/WebScraper.php index 5db760f..c03d794 100644 --- a/php/src/app/Models/WebScraper.php +++ b/php/src/app/Models/WebScraper.php @@ -5,6 +5,7 @@ namespace App\Models; use App\Utils\ImageUrl; use Facebook\WebDriver\Remote\RemoteWebDriver; use Facebook\WebDriver\WebDriverBy; +use Facebook\WebDriver\WebDriverAction; use Facebook\WebDriver\WebDriverExpectedCondition; class WebScraper @@ -120,8 +121,7 @@ class WebScraper 'image' => $imageFileUrl, ]; $album_id = Album::findOrCreateByName($artist, $albumTitle, $data); - $album_queue = new AlbumQueue(); - $album_queue->enqueue($album_id); + AlbumQueue::addQueue($album_id); } /** @@ -153,13 +153,27 @@ class WebScraper } } else { \Log::info('Could not locate Albums button'); - $ytRows = $driver->findElements(WebDriverBy::cssSelector('ytmusic-carousel-shelf-renderer')); foreach ($ytRows as $ytRow) { $contentGroup = $ytRow->findElements(WebDriverBy::cssSelector('#content-group')); foreach ($contentGroup as $group) { $groupName = $group->getText(); if ($groupName == 'Albums') { + + // Sometimes we don't have the option to click the albums button to filter + // Yet, the albums are in a carousel and the images won't load unless they are in view + $caroselNextButton = $driver->findElements(WebDriverBy::cssSelector('#next-items-button')); + if ($caroselNextButton) { + // Youtube is smart enough to block this without an action + for ($i = 0; $i <= 3; $i++) { + if ($caroselNextButton[0]->isEnabled()) { + $action = $driver->action(); + $action->moveToElement($caroselNextButton[0])->click()->perform(); + sleep(1); + } + } + } + $itemsContainer = $ytRow->findElements(WebDriverBy::cssSelector('#items')); foreach ($itemsContainer as $item) { $albumContainers = $item->findElements(WebDriverBy::cssSelector('ytmusic-two-row-item-renderer')); diff --git a/php/src/public/js/app.js b/php/src/public/js/app.js index 37909ff..30e0663 100644 --- a/php/src/public/js/app.js +++ b/php/src/public/js/app.js @@ -64,7 +64,7 @@ function artist_queue_toggle(element) { url: `/api/queue/artist/${self.data('artist_id')}`, success: () => { proc_notification('success', 'Queued Download', `Artist ${artist_name} Queued for Download!`); - ArtistTable.ajax.reload(); + // ArtistTable.ajax.reload(); }, error: (response) => { console.log(response); diff --git a/php/src/resources/views/modals/modal-catalog.blade.php b/php/src/resources/views/modals/modal-catalog.blade.php index 87b6769..fb94ff9 100644 --- a/php/src/resources/views/modals/modal-catalog.blade.php +++ b/php/src/resources/views/modals/modal-catalog.blade.php @@ -13,7 +13,7 @@
- +