[FIX] php: Handling for albums section if no link

php/src/app/Http/Controllers/ApiController.php

@@ -2,6 +2,7 @@
 
 namespace App\Http\Controllers;
 
+use Illuminate\Support\Facades\Artisan;
 use App\Models\AlbumQueue;
 use App\Models\Artist;
 use App\Models\WebDriver;
@@ -56,6 +57,8 @@ class ApiController extends Controller
 
     public function queue_artist_run()
     {
+        \Log::info('===========================');
+        \Log::info('Queue running for Artists..');
         Artisan::queue('app:process-artist-queue');
     }
 
@@ -79,15 +82,18 @@ class ApiController extends Controller
 
     public function queue_waiting()
     {
+        \Log::info('===========================');
+        \Log::info('Queue running for Albums..');
+        $data = array('queue' => false);
         $queue = AlbumQueue::where('state', 'pending')->first();
+        if (!is_null($queue)) {
             $album = $queue->album;
             $artist = $album->artist;
-
-        \Log::info('======================');
-        \Log::info('Queue running for album: ' . $album->name);
             $queue->state = 'in_progress';
             $queue->save();
             $data = array('queue' => $queue->toArray(), 'album' => $album->toArray(), 'artist' => $artist->toArray());
+
+        }
         return json_encode($data);
     }
 

php/src/app/Models/ArtistQueue.php

@@ -24,12 +24,17 @@ class ArtistQueue extends Model
         return $result;
     }
 
+    public function artist()
+    {
+        return $this->belongsTo(Artist::class);
+    }
+
 
     public function process_artist()
     {
         // Scrape the artist page for image, and album data (image, url, name)
         $driver = WebDriver::setUp();
-        $artist_id = Artist::where('id', $this->artist_id)->get()->first();
+        $artist_id = $this->artist;
         if ($artist_id->count() > 0) {
             try {
                 $album_count = WebScraper::scrapeAlbums($driver, $artist_id);
@@ -38,6 +43,7 @@ class ArtistQueue extends Model
             } catch (Exception $e) {
                 \Log::warning('Failed to scrape albums: ' . $e->getMessage());
             } finally {
+                $artist_id->change_state('done');
                 $driver->quit();
             }
         } else {

php/src/app/Models/WebScraper.php

@@ -101,6 +101,29 @@ class WebScraper
         return $response;
     }
 
+    public static function processAlbums($albumContainer, $artist)
+    {
+        $albumLink = $albumContainer->findElement(WebDriverBy::cssSelector('a'));
+        $albumHref = $albumLink->getAttribute('href');
+        $albumTitle = $albumLink->getAttribute('title');
+        $albumThumbnail = $albumLink->findElement(WebDriverBy::cssSelector('img'))->getAttribute('src');
+
+        // Resize image and save to file, provide path to data
+        $imageUrl = ImageUrl::modifyGoogleImageUrl($albumThumbnail);
+        $imageFileUrl = ImageUrl::save_img_url($imageUrl, 'album');
+
+        $data = [
+            'name' => $albumTitle,
+            'artist_id' => $artist->id,
+            'thumbnail' => $albumThumbnail,
+            'url_remote' => $albumHref,
+            'image' => $imageFileUrl,
+        ];
+        $album_id = Album::findOrCreateByName($artist, $albumTitle, $data);
+        $album_queue = new AlbumQueue();
+        $album_queue->enqueue($album_id);
+    }
+
     /**
      * Scrape the album data from given artist page, create new album records and queue those records for download
      *
@@ -112,9 +135,11 @@ class WebScraper
         $driver->get($url);
         $response = 0;
         try {
-            $albumBtn = $driver->findElement(WebDriverBy::xpath('//a[text()="Albums"]'));
+            \Log::info('Looking for Albums button..');
+            $albumBtn = $driver->findElements(WebDriverBy::xpath('//a[text()="Albums"]'));
             if ($albumBtn) {
-                $albumBtn->click();
+                \Log::info('Clicking on located Albums button..');
+                $albumBtn[0]->click();
                 sleep(3);
                 $itemsContainer = $driver->findElements(WebDriverBy::cssSelector('#items'));
                 foreach ($itemsContainer as $item) {
@@ -122,29 +147,33 @@ class WebScraper
                     if ($albumContainers) {
                         foreach ($albumContainers as $albumContainer) {
                             $response += 1;
-                            $albumLink = $albumContainer->findElement(WebDriverBy::cssSelector('a'));
+                            WebScraper::processAlbums($albumContainer, $artist_id);
-                            $albumHref = $albumLink->getAttribute('href');
+                        }
-                            $albumTitle = $albumLink->getAttribute('title');
+                    }
-                            $albumThumbnail = $albumLink->findElement(WebDriverBy::cssSelector('img'))->getAttribute('src');
+                }
-
+            } else {
-                            // Resize image and save to file, provide path to data
+                \Log::info('Could not locate Albums button');
-                            $imageUrl = ImageUrl::modifyGoogleImageUrl($albumThumbnail);
-                            $imageFileUrl = ImageUrl::save_img_url($imageUrl, 'album');
-
-                            $data = [
-                                'name' => $albumTitle,
-                                'artist_id' => $artist_id->id,
-                                'thumbnail' => $albumThumbnail,
-                                'url_remote' => $albumHref,
-                                'image' => $imageFileUrl,
-                            ];
-                            $album_id = Album::findOrCreateByName($artist_id, $albumTitle, $data);
 
-                            $album_queue = new AlbumQueue();
+                $ytRows = $driver->findElements(WebDriverBy::cssSelector('ytmusic-carousel-shelf-renderer'));
-                            $album_queue->enqueue($album_id);
+                foreach ($ytRows as $ytRow) {
+                    $contentGroup = $ytRow->findElements(WebDriverBy::cssSelector('#content-group'));
+                    foreach ($contentGroup as $group) {
+                        $groupName = $group->getText();
+                        if ($groupName == 'Albums') {
+                            $itemsContainer = $ytRow->findElements(WebDriverBy::cssSelector('#items'));
+                            foreach ($itemsContainer as $item) {
+                                $albumContainers = $item->findElements(WebDriverBy::cssSelector('ytmusic-two-row-item-renderer'));
+                                if ($albumContainers) {
+                                    foreach ($albumContainers as $albumContainer) {
+                                        WebScraper::processAlbums($albumContainer, $artist_id);
+                                    }
+                                }
                             }
                         }
                     }
+
+                }
+
             }
         } catch (\Exception $e) {
             \Log::warning('Failed to scrape albums: ---------');

php/src/public/js/app.js

@@ -160,9 +160,10 @@ $(document).ready(function () {
             {
                 data: 'id', orderable: false, render: (data, type, row) => {
                     let stateDiable = row.state === 'in_progress' ? 'disabled' : '';
-                    let stateClass = row.state === 'in_progress' ? '' : 'btn-primary';
+                    let stateClass = row.state === 'done' ? 'btn-success' : 'btn-primary';
                     let artist_name = row.name;
-                    return `<button class="btn ${stateClass}" style="float: right;" data-artist_name="${artist_name}" data-artist_id="${data}" onclick="artist_queue_toggle(this)" ${stateDiable}><i class="las la-cloud-download-alt"></i> Download</button>`
+                    let button_icon = row.state === 'done' ? '<i class="las la-redo-alt"></i>' : '<i class="las la-cloud-download-alt"></i>';
+                    return `<button class="btn ${stateClass}" style="float: right;" data-artist_name="${artist_name}" data-artist_id="${data}" onclick="artist_queue_toggle(this)" ${stateDiable}>${button_icon} Download</button>`
                 }
             }
         ],

php/src/routes/console.php

@@ -7,4 +7,3 @@ Artisan::command('inspire', function () {
     $this->comment(Inspiring::quote());
 })->purpose('Display an inspiring quote')->hourly();
 
-Artisan::command('app:process-artist-queue')->everyMinute();

python/app.py

@@ -13,25 +13,28 @@ app = Flask(__name__)
 redis = Redis(host='redis', port=6379)
 
 
-# def process_artist_queue():
+def process_artist_queue():
-#     requests.get('http://nginx/api/queue/artists/run')
+    print('Running Artist Queue process..')
-#     return
+    print('---')
+    requests.get('http://nginx/api/queue/artists/run')
+    return
 
 def process_album_queue():
     print('Running Album Queue Process..')
     print('---')
     response = requests.get('http://nginx/api/album/queue')
     data = response.json()
-    artist = data.get('artist')
+    artist = data.get('artist', False)
-    album = data.get('album')
+    album = data.get('album', False)
     queue = data.get('queue')
-    if artist and album and queue:
+    if not queue == False and artist and album:
         result = download_album(album, artist)
         requests.post('http://nginx/api/album/queue/update/%s' % queue.get('id'), json=result)
     return
 
 cron = BackgroundScheduler({'apscheduler.job_defaults.max_instances': 1}, daemon=True)
 cron.add_job(process_album_queue, 'interval', minutes=1)
+cron.add_job(process_artist_queue, 'interval', minutes=1)
 cron.start()
 
 if __name__ == "__main__":