[IMP] Functionally working

docker-compose.yml

@@ -34,7 +34,6 @@ services:
             - "9000:9000"
         volumes:
             - ./php/src:/var/www/html:delegated
-            - ./shared:/var/
         networks:
             - laravel
 
@@ -49,9 +48,8 @@ services:
         ports:
             - "5000:5000"
         volumes:
-            - ./python:/code:Z
+            - ./python:/python:Z
             - ./php/src:/var/www/html:delegated
-            - ./shared:/var/
         depends_on:
             - redis
         networks:
@@ -143,36 +141,36 @@ services:
             - SE_EVENT_BUS_HOST=selenium-hub
             - SE_EVENT_BUS_PUBLISH_PORT=4442
             - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
-            - SE_NODE_MAX_SESSIONS=5
+            - SE_NODE_MAX_SESSIONS=15
-            - SE_NODE_MAX_SESSION=5
+            - SE_NODE_MAX_SESSION=15
 
-    edge:
+#    edge:
-        image: selenium/node-edge:nightly
+#        image: selenium/node-edge:nightly
-        shm_size: 8gb
+#        shm_size: 8gb
-        networks:
+#        networks:
-            - laravel
+#            - laravel
-        depends_on:
+#        depends_on:
-            - selenium-hub
+#            - selenium-hub
-        environment:
+#        environment:
-            - SE_EVENT_BUS_HOST=selenium-hub
+#            - SE_EVENT_BUS_HOST=selenium-hub
-            - SE_EVENT_BUS_PUBLISH_PORT=4442
+#            - SE_EVENT_BUS_PUBLISH_PORT=4442
-            - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
+#            - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
-            - SE_NODE_MAX_SESSIONS=5
+#            - SE_NODE_MAX_SESSIONS=5
-            - SE_NODE_MAX_SESSION=5
+#            - SE_NODE_MAX_SESSION=5
-
+#
-    firefox:
+#    firefox:
-        image: selenium/node-firefox:nightly
+#        image: selenium/node-firefox:nightly
-        shm_size: 8gb
+#        shm_size: 8gb
-        networks:
+#        networks:
-            - laravel
+#            - laravel
-        depends_on:
+#        depends_on:
-            - selenium-hub
+#            - selenium-hub
-        environment:
+#        environment:
-            - SE_EVENT_BUS_HOST=selenium-hub
+#            - SE_EVENT_BUS_HOST=selenium-hub
-            - SE_EVENT_BUS_PUBLISH_PORT=4442
+#            - SE_EVENT_BUS_PUBLISH_PORT=4442
-            - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
+#            - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
-            - SE_NODE_MAX_SESSIONS=5
+#            - SE_NODE_MAX_SESSIONS=5
-            - SE_NODE_MAX_SESSION=5
+#            - SE_NODE_MAX_SESSION=5
 
     selenium-hub:
         image: selenium/hub:latest

php/src/app/Http/Controllers/ApiController.php

@@ -2,6 +2,7 @@
 
 namespace App\Http\Controllers;
 
+use App\Models\AlbumQueue;
 use App\Models\Artist;
 use App\Models\WebDriver;
 use App\Models\WebScraper;
@@ -29,11 +30,35 @@ class ApiController extends Controller
         return $response;
     }
 
+    public function get_album_queue()
+    {
+        $album_queue = AlbumQueue::where('state', '!=', 'done')->get();
+        $response = array();
+        foreach ($album_queue as $queue) {
+            $album = $queue->album;
+            $artist = $album->artist;
+            $response[] = [
+                'name' => $album->name,
+                'artist_id' => $artist->toArray(),
+                'url_remote' => $album->url_remote,
+                'thumbnail' => $album->thumbnail,
+                'image' => $album->image,
+                'state' => $queue->state,
+            ];
+        }
+        return json_encode($response);
+    }
+
     public function queue_artist($id, ArtistQueue $artistQueue): bool
     {
         return $artistQueue->enqueue($id);
     }
 
+    public function queue_artist_run()
+    {
+        Artisan::queue('app:process-artist-queue');
+    }
+
     public function search_artist(string $artist)
     {
         $url = 'https://music.youtube.com/search?q=' . str_replace(' ', '+', $artist);
@@ -52,4 +77,48 @@ class ApiController extends Controller
         return response()->json($response);
     }
 
+    public function queue_waiting()
+    {
+        $queue = AlbumQueue::where('state', 'pending')->first();
+        $album = $queue->album;
+        $artist = $album->artist;
+
+        \Log::info('======================');
+        \Log::info('Queue running for album: ' . $album->name);
+        $queue->state = 'in_progress';
+        $queue->save();
+        $data = array('queue' => $queue->toArray(), 'album' => $album->toArray(), 'artist' => $artist->toArray());
+        return json_encode($data);
+    }
+
+    public function queue_update(Request $request, $id)
+    {
+        $queue = AlbumQueue::where('id', $id)->first();
+        $album = $queue->album;
+        $artist = $album->artist;
+
+        if ($queue->exists()) {
+
+            if (isset($request['album']) || isset($request['artist'])) {
+                $album_local_url = $request['album']['url_local'] ?? '';
+                $artist_local_url = $request['artist']['url_local'] ?? '';
+
+                if ($album_local_url || $artist_local_url) {
+                    if ($artist_local_url && is_string($artist_local_url)) {
+                        $artist->url_local = $artist_local_url;
+                        $artist->save();
+                    }
+                    if ($album_local_url && is_string($album_local_url)) {
+                        $album->url_local = $album_local_url;
+                        $album->save();
+                    }
+                    $queue->state = 'done';
+                    $queue->save();
+                }
+
+            }
+        }
+
+    }
+
 }

php/src/app/Models/Album.php

@@ -4,6 +4,7 @@ namespace App\Models;
 
 use Illuminate\Database\Eloquent\Factories\HasFactory;
 use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
 use Mockery\Exception;
 
 class Album extends Model
@@ -13,7 +14,7 @@ class Album extends Model
     public function change_state(string $state)
     {
         $available_states = array("pending", "in_progress", "done");
-        if (!in_array($state, $available_states)){
+        if (!in_array($state, $available_states)) {
             throw new Exception('Invalid state');
         }
         $this->state = $state;
@@ -51,4 +52,9 @@ class Album extends Model
         return $album;
     }
 
+    public function artist(): BelongsTo
+    {
+        return $this->belongsTo(Artist::class);
+    }
+
 }

php/src/app/Models/AlbumQueue.php

@@ -21,8 +21,9 @@ class AlbumQueue extends Model
         return $result;
     }
 
-    public function process_album()
+    public function album()
     {
-        // Either python pings to process the queue or laravel will send the data to python for processing
+        return $this->belongsTo(Album::class);
     }
+
 }

php/src/config/app.php

@@ -39,7 +39,7 @@ return [
     |
     */
 
-    'debug' => (bool) env('APP_DEBUG', false),
+    'debug' => (bool)env('APP_DEBUG', false),
 
     /*
     |--------------------------------------------------------------------------

php/src/public/js/app.js

@@ -3,6 +3,15 @@ const appModal = $('#modalDownloadQueue');
 const loader = $("#loader-wrapper");
 let ArtistTable = {}; // Initialized for ajax reload
 
+function requestQueue() {
+    $.ajax({
+        url: '/api/queue/albums',
+        success: (response) => {
+            Alpine.store('app').Queue = JSON.parse(response);
+        }
+    })
+}
+
 function template_artist_result(element) {
     return `
         <div class="card w-100 p-2 mb-2">
@@ -120,15 +129,10 @@ function bind_action_buttons() {
 document.addEventListener('alpine:init', () => {
     console.log('Alpine:init');
     Alpine.store('app', {
-        init() {
-            // TODO: Poll for artists and queue
-            this.Queue = [];
-        },
-
         Queue: [],          // Rendered in the 'Queue' modal
-
     });
-
+    requestQueue();
+    setInterval(requestQueue, 5000);
     $("#loader-wrapper").fadeOut(900);
 
 });

php/src/resources/views/components/download-queue.blade.php

@@ -11,13 +11,13 @@
                             <div class="dl_queue_img">
 
                                 <!-- Downloading Spinner -->
-                                <template x-if="album.state === 'progress'">
+                                <template x-if="album.state === 'in_progress'">
                                     <div class="icn-downloading">
                                         <i class="la la-6x la-compact-disc icn-spinner text-white"></i>
                                     </div>
                                 </template>
                                 <!-- Album Art -->
-                                <img :src="album.image" class="img-fluid rounded-start"
+                                <img :src="album.thumbnail" class="img-fluid rounded-start"
                                      :alt="album.name" style="width: 100%; height: 100%; min-height: 180px;">
                             </div>
 
@@ -25,13 +25,13 @@
                         <div class="col-md-9 vinyl-card">
                             <div class="card-body">
                                 <a :href="album.url_remote" target="_blank">
-                                    <h3 class="card-title">@{{ album.name }}</h3>
+                                    <h3 class="card-title" x-text="album.name"></h3>
                                 </a>
                                 <h5 class="card-text"
                                     style="font-weight: bold; font-family: serif;"><span x-text="album.artist_id.name"/>
                                 </h5>
 
-                                <template x-if="album.state === 'progress'">
+                                <template x-if="album.state === 'in_progress'">
                                     <p>Downloading...</p>
                                 </template>
 

php/src/resources/views/modals/modal-settings.blade.php

@@ -10,7 +10,7 @@
                         aria-label="Close"></button>
             </div>
             <div class="modal-body">
-                <p>No settings yet, check back later. A configurable output path is on the todo list!</p>
+                <p>No settings yet, check back later.</p>
             </div>
             <div class="modal-footer">
                 <button type="button" class="btn btn-primary" data-bs-dismiss="modal">Save</button>

php/src/routes/console.php

@@ -6,3 +6,5 @@ use Illuminate\Support\Facades\Artisan;
 Artisan::command('inspire', function () {
     $this->comment(Inspiring::quote());
 })->purpose('Display an inspiring quote')->hourly();
+
+Artisan::command('app:process-artist-queue')->everyMinute();

php/src/routes/web.php

@@ -1,16 +1,24 @@
 <?php
 
+use \Illuminate\Foundation\Http\Middleware\VerifyCsrfToken;
+use App\Http\Middleware\DisableCsrf;
 use App\Http\Controllers\ApiController;
 use Illuminate\Support\Facades\Route;
 
 Route::get('/', function () {
     return view('pages.main');
 });
-
+//  User submitted input to scrape for artist data
 Route::get('/artist/{artist}', [ApiController::class, 'search_artist'])->name('api.search.artist');
-
+//  Return list of all artists in database
-// Get all artists
 Route::get('api/artists/', [ApiController::class, 'get_artists'])->name('api.artist');
-
+//  Add artist to queue
-// Queue single artist
 Route::get('api/queue/artist/{id}', [ApiController::class, 'queue_artist'])->name('api.artist.queue');
+//  Returns a single album that is ready for download
+Route::get('api/album/queue', [ApiController::class, 'queue_waiting'])->name('api.queue.waiting');
+// Update the queue from handler
+Route::post('api/album/queue/update/{id}', [ApiController::class, 'queue_update'])->name('api.queue.update')->withoutMiddleware(VerifyCsrfToken::class);
+// Prompt the artist queue remotely
+Route::get('api/queue/artists/run', [ApiController::class, 'queue_artist_run'])->name('api.queue.run');
+// Client side queue data
+Route::get('/api/queue/albums', [ApiController::class, 'get_album_queue'])->name('api.queue.albums');

php/src/storage/app/.gitignore

@@ -1,3 +1,4 @@
 *
 !public/
+!music/
 !.gitignore

php/src/storage/app/music/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

python/app.py

@@ -1,58 +1,39 @@
 import json
 from apscheduler.schedulers.background import BackgroundScheduler
-from flask import Flask, render_template
+from flask import Flask
 from redis import Redis
 from utils.download import download_album
-from utils.processor import process_download
+import requests
+
+import logging
+_logger = logging.getLogger(__name__)
 
 
 app = Flask(__name__)
 redis = Redis(host='redis', port=6379)
 
 
-def process_downloads():
+# def process_artist_queue():
-    print('Processing Downloads..')
+#     requests.get('http://nginx/api/queue/artists/run')
-    pending_downloads = Album.search([('downloaded', '=', False), ('downloading', '=', False)])
+#     return
-    if pending_downloads:
+
-        ready_album = pending_downloads[:1]
+def process_album_queue():
-        if ready_album:
+    print('Running Album Queue Process..')
-            album = ready_album[0]
+    print('---')
-            print('...................')
+    response = requests.get('http://nginx/api/album/queue')
-            print('Downloading Album..')
+    data = response.json()
-            print(album)
+    artist = data.get('artist')
-            Album.write(album['id'], {'downloading': True})
+    album = data.get('album')
-            download_album(album)
+    queue = data.get('queue')
-
+    if artist and album and queue:
-
+        result = download_album(album, artist)
-cron = BackgroundScheduler({'apscheduler.job_defaults.max_instances': 2}, daemon=True)
+        requests.post('http://nginx/api/album/queue/update/%s' % queue.get('id'), json=result)
-cron.add_job(process_downloads, 'interval', minutes=1)
+    return
+
+cron = BackgroundScheduler({'apscheduler.job_defaults.max_instances': 1}, daemon=True)
+cron.add_job(process_album_queue, 'interval', minutes=1)
 cron.start()
 
-
-@app.route('/api/v1/process/album')
-def get_artist(path):
-    """
-    Process for the requested Album
-    :param path: The Artist to get files for
-    :return: a status
-    """
-    if path:
-        res = process_download(path)
-    else:
-        res = {'status': 501, 'message': 'Could not process download..'}
-
-    return res
-
-
-@app.route('/api/v1/get/queue')
-def get_queue():
-    album_ids = Album.search([('downloaded', '=', False)])
-    data = {}
-    if album_ids:
-        data.update({'album_ids': album_ids})
-    return render_template('download_queue.html', **data)
-
-
 if __name__ == "__main__":
     print('Starting App...')
     app.run(host="0.0.0.0", debug=True)

python/dockerfiles/python.dockerfile

@@ -2,8 +2,8 @@ FROM debian:bullseye-slim
 RUN apt update && apt upgrade -y
 RUN apt install ffmpeg -y
 RUN apt install curl python3-pip -y
-ADD . /code
+ADD . /python
-WORKDIR /code
+WORKDIR /python
 
 RUN pip3 install -r requirements.txt
 ENV FLASK_APP=app

python/utils/download.py

@@ -1,37 +1,34 @@
+import shutil
 import os
-from database import Model
-from const import *
-from .yt_dlp_logger import *
-import wget
 import yt_dlp
+from .yt_dlp_logger import YtDlpLogger, yt_dlp_log_hook
 
-Album = Model('album')
+MEDIA_FOLDER = '/var/www/html/storage/app/music'
 
 
-def download_album(album):
+def download_album(album, artist):
     """
-    Take a list of albums and process the downloads
+    Take an artist and album dict and use yt-dlp to download the album to the local filestore.
     :param album: Dict of album data
-    :return:
+    :return: dict of save data
     """
-    artist = album.get('artist')
+    response = {
-    artist_path = MEDIA_FOLDER + '/%s' % artist
+        'artist': {},
+        'album': {},
+    }
+    artist_path = MEDIA_FOLDER + '/%s' % artist.get('name')
     if not os.path.exists(artist_path):
+        # Make artist folder and copy the artist image, add
         os.mkdir(artist_path)
+        shutil.copy2(artist.get('image'), artist_path + '/artist.jpg')
+        response['artist'] = {'url_local': artist_path}
 
-    print('---')
     # Create album folder
-    album_title = album.get('album')
+    album_title = album.get('name')
     album_path = artist_path + '/%s' % album_title
     if not os.path.exists(album_path):
         os.mkdir(album_path)
-
+        shutil.copy2(album.get('image'), album_path + '/album.jpg')
-    # Save album cover
-    if album.get('cover'):
-        try:
-            download_file(album.get('cover'), album_path)
-        except Exception as e:
-            print("Warning: %s" % e)
 
     # Download album
     ydl_opts = {
@@ -45,18 +42,13 @@ def download_album(album):
             'preferredcodec': 'mp3',
         }]
     }
+    download_url = 'https://music.youtube.com/' + album.get('url_remote')
 
     with yt_dlp.YoutubeDL(ydl_opts) as ydl:
         try:
-            error_code = ydl.download('https://youtube.com' + album.get('link'))
+            ydl.download(download_url)
         except Exception as e:
-            print('!!!!!!!!!')
+            print('yt-dlp download failed: =========')
             print(e)
-        finally:
+    response['album'] = {'url_local': album_path}
-            Album.unlink(album['id'])
+    return response
-
-
-def download_file(url, output):
-    filename = wget.download(url, out=output)
-    os.rename(filename, output + '/album.jpg')
-    return filename