[IMP] Functionally working

refactor_total
Brett Spaulding 1 year ago
parent 842e440bd2
commit bb0b800fa2

@ -34,7 +34,6 @@ services:
- "9000:9000"
volumes:
- ./php/src:/var/www/html:delegated
- ./shared:/var/
networks:
- laravel
@ -49,9 +48,8 @@ services:
ports:
- "5000:5000"
volumes:
- ./python:/code:Z
- ./python:/python:Z
- ./php/src:/var/www/html:delegated
- ./shared:/var/
depends_on:
- redis
networks:
@ -143,36 +141,36 @@ services:
- SE_EVENT_BUS_HOST=selenium-hub
- SE_EVENT_BUS_PUBLISH_PORT=4442
- SE_EVENT_BUS_SUBSCRIBE_PORT=4443
- SE_NODE_MAX_SESSIONS=5
- SE_NODE_MAX_SESSION=5
- SE_NODE_MAX_SESSIONS=15
- SE_NODE_MAX_SESSION=15
edge:
image: selenium/node-edge:nightly
shm_size: 8gb
networks:
- laravel
depends_on:
- selenium-hub
environment:
- SE_EVENT_BUS_HOST=selenium-hub
- SE_EVENT_BUS_PUBLISH_PORT=4442
- SE_EVENT_BUS_SUBSCRIBE_PORT=4443
- SE_NODE_MAX_SESSIONS=5
- SE_NODE_MAX_SESSION=5
firefox:
image: selenium/node-firefox:nightly
shm_size: 8gb
networks:
- laravel
depends_on:
- selenium-hub
environment:
- SE_EVENT_BUS_HOST=selenium-hub
- SE_EVENT_BUS_PUBLISH_PORT=4442
- SE_EVENT_BUS_SUBSCRIBE_PORT=4443
- SE_NODE_MAX_SESSIONS=5
- SE_NODE_MAX_SESSION=5
# edge:
# image: selenium/node-edge:nightly
# shm_size: 8gb
# networks:
# - laravel
# depends_on:
# - selenium-hub
# environment:
# - SE_EVENT_BUS_HOST=selenium-hub
# - SE_EVENT_BUS_PUBLISH_PORT=4442
# - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
# - SE_NODE_MAX_SESSIONS=5
# - SE_NODE_MAX_SESSION=5
#
# firefox:
# image: selenium/node-firefox:nightly
# shm_size: 8gb
# networks:
# - laravel
# depends_on:
# - selenium-hub
# environment:
# - SE_EVENT_BUS_HOST=selenium-hub
# - SE_EVENT_BUS_PUBLISH_PORT=4442
# - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
# - SE_NODE_MAX_SESSIONS=5
# - SE_NODE_MAX_SESSION=5
selenium-hub:
image: selenium/hub:latest

@ -2,6 +2,7 @@
namespace App\Http\Controllers;
use App\Models\AlbumQueue;
use App\Models\Artist;
use App\Models\WebDriver;
use App\Models\WebScraper;
@ -29,11 +30,35 @@ class ApiController extends Controller
return $response;
}
public function get_album_queue()
{
$album_queue = AlbumQueue::where('state', '!=', 'done')->get();
$response = array();
foreach ($album_queue as $queue) {
$album = $queue->album;
$artist = $album->artist;
$response[] = [
'name' => $album->name,
'artist_id' => $artist->toArray(),
'url_remote' => $album->url_remote,
'thumbnail' => $album->thumbnail,
'image' => $album->image,
'state' => $queue->state,
];
}
return json_encode($response);
}
public function queue_artist($id, ArtistQueue $artistQueue): bool
{
return $artistQueue->enqueue($id);
}
public function queue_artist_run()
{
Artisan::queue('app:process-artist-queue');
}
public function search_artist(string $artist)
{
$url = 'https://music.youtube.com/search?q=' . str_replace(' ', '+', $artist);
@ -52,4 +77,48 @@ class ApiController extends Controller
return response()->json($response);
}
public function queue_waiting()
{
$queue = AlbumQueue::where('state', 'pending')->first();
$album = $queue->album;
$artist = $album->artist;
\Log::info('======================');
\Log::info('Queue running for album: ' . $album->name);
$queue->state = 'in_progress';
$queue->save();
$data = array('queue' => $queue->toArray(), 'album' => $album->toArray(), 'artist' => $artist->toArray());
return json_encode($data);
}
public function queue_update(Request $request, $id)
{
$queue = AlbumQueue::where('id', $id)->first();
$album = $queue->album;
$artist = $album->artist;
if ($queue->exists()) {
if (isset($request['album']) || isset($request['artist'])) {
$album_local_url = $request['album']['url_local'] ?? '';
$artist_local_url = $request['artist']['url_local'] ?? '';
if ($album_local_url || $artist_local_url) {
if ($artist_local_url && is_string($artist_local_url)) {
$artist->url_local = $artist_local_url;
$artist->save();
}
if ($album_local_url && is_string($album_local_url)) {
$album->url_local = $album_local_url;
$album->save();
}
$queue->state = 'done';
$queue->save();
}
}
}
}
}

@ -4,6 +4,7 @@ namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Mockery\Exception;
class Album extends Model
@ -13,7 +14,7 @@ class Album extends Model
public function change_state(string $state)
{
$available_states = array("pending", "in_progress", "done");
if (!in_array($state, $available_states)){
if (!in_array($state, $available_states)) {
throw new Exception('Invalid state');
}
$this->state = $state;
@ -51,4 +52,9 @@ class Album extends Model
return $album;
}
public function artist(): BelongsTo
{
return $this->belongsTo(Artist::class);
}
}

@ -21,8 +21,9 @@ class AlbumQueue extends Model
return $result;
}
public function process_album()
public function album()
{
// Either python pings to process the queue or laravel will send the data to python for processing
return $this->belongsTo(Album::class);
}
}

@ -39,7 +39,7 @@ return [
|
*/
'debug' => (bool) env('APP_DEBUG', false),
'debug' => (bool)env('APP_DEBUG', false),
/*
|--------------------------------------------------------------------------

@ -3,6 +3,15 @@ const appModal = $('#modalDownloadQueue');
const loader = $("#loader-wrapper");
let ArtistTable = {}; // Initialized for ajax reload
function requestQueue() {
$.ajax({
url: '/api/queue/albums',
success: (response) => {
Alpine.store('app').Queue = JSON.parse(response);
}
})
}
function template_artist_result(element) {
return `
<div class="card w-100 p-2 mb-2">
@ -120,15 +129,10 @@ function bind_action_buttons() {
document.addEventListener('alpine:init', () => {
console.log('Alpine:init');
Alpine.store('app', {
init() {
// TODO: Poll for artists and queue
this.Queue = [];
},
Queue: [], // Rendered in the 'Queue' modal
});
requestQueue();
setInterval(requestQueue, 5000);
$("#loader-wrapper").fadeOut(900);
});

@ -11,13 +11,13 @@
<div class="dl_queue_img">
<!-- Downloading Spinner -->
<template x-if="album.state === 'progress'">
<template x-if="album.state === 'in_progress'">
<div class="icn-downloading">
<i class="la la-6x la-compact-disc icn-spinner text-white"></i>
</div>
</template>
<!-- Album Art -->
<img :src="album.image" class="img-fluid rounded-start"
<img :src="album.thumbnail" class="img-fluid rounded-start"
:alt="album.name" style="width: 100%; height: 100%; min-height: 180px;">
</div>
@ -25,13 +25,13 @@
<div class="col-md-9 vinyl-card">
<div class="card-body">
<a :href="album.url_remote" target="_blank">
<h3 class="card-title">@{{ album.name }}</h3>
<h3 class="card-title" x-text="album.name"></h3>
</a>
<h5 class="card-text"
style="font-weight: bold; font-family: serif;"><span x-text="album.artist_id.name"/>
</h5>
<template x-if="album.state === 'progress'">
<template x-if="album.state === 'in_progress'">
<p>Downloading...</p>
</template>

@ -10,7 +10,7 @@
aria-label="Close"></button>
</div>
<div class="modal-body">
<p>No settings yet, check back later. A configurable output path is on the todo list!</p>
<p>No settings yet, check back later.</p>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-primary" data-bs-dismiss="modal">Save</button>

@ -6,3 +6,5 @@ use Illuminate\Support\Facades\Artisan;
Artisan::command('inspire', function () {
$this->comment(Inspiring::quote());
})->purpose('Display an inspiring quote')->hourly();
Artisan::command('app:process-artist-queue')->everyMinute();

@ -1,16 +1,24 @@
<?php
use \Illuminate\Foundation\Http\Middleware\VerifyCsrfToken;
use App\Http\Middleware\DisableCsrf;
use App\Http\Controllers\ApiController;
use Illuminate\Support\Facades\Route;
Route::get('/', function () {
return view('pages.main');
});
// User submitted input to scrape for artist data
Route::get('/artist/{artist}', [ApiController::class, 'search_artist'])->name('api.search.artist');
// Get all artists
// Return list of all artists in database
Route::get('api/artists/', [ApiController::class, 'get_artists'])->name('api.artist');
// Queue single artist
// Add artist to queue
Route::get('api/queue/artist/{id}', [ApiController::class, 'queue_artist'])->name('api.artist.queue');
// Returns a single album that is ready for download
Route::get('api/album/queue', [ApiController::class, 'queue_waiting'])->name('api.queue.waiting');
// Update the queue from handler
Route::post('api/album/queue/update/{id}', [ApiController::class, 'queue_update'])->name('api.queue.update')->withoutMiddleware(VerifyCsrfToken::class);
// Prompt the artist queue remotely
Route::get('api/queue/artists/run', [ApiController::class, 'queue_artist_run'])->name('api.queue.run');
// Client side queue data
Route::get('/api/queue/albums', [ApiController::class, 'get_album_queue'])->name('api.queue.albums');

@ -1,3 +1,4 @@
*
!public/
!music/
!.gitignore

@ -0,0 +1,2 @@
*
!.gitignore

@ -1,58 +1,39 @@
import json
from apscheduler.schedulers.background import BackgroundScheduler
from flask import Flask, render_template
from flask import Flask
from redis import Redis
from utils.download import download_album
from utils.processor import process_download
import requests
import logging
_logger = logging.getLogger(__name__)
app = Flask(__name__)
redis = Redis(host='redis', port=6379)
def process_downloads():
print('Processing Downloads..')
pending_downloads = Album.search([('downloaded', '=', False), ('downloading', '=', False)])
if pending_downloads:
ready_album = pending_downloads[:1]
if ready_album:
album = ready_album[0]
print('...................')
print('Downloading Album..')
print(album)
Album.write(album['id'], {'downloading': True})
download_album(album)
cron = BackgroundScheduler({'apscheduler.job_defaults.max_instances': 2}, daemon=True)
cron.add_job(process_downloads, 'interval', minutes=1)
# def process_artist_queue():
# requests.get('http://nginx/api/queue/artists/run')
# return
def process_album_queue():
print('Running Album Queue Process..')
print('---')
response = requests.get('http://nginx/api/album/queue')
data = response.json()
artist = data.get('artist')
album = data.get('album')
queue = data.get('queue')
if artist and album and queue:
result = download_album(album, artist)
requests.post('http://nginx/api/album/queue/update/%s' % queue.get('id'), json=result)
return
cron = BackgroundScheduler({'apscheduler.job_defaults.max_instances': 1}, daemon=True)
cron.add_job(process_album_queue, 'interval', minutes=1)
cron.start()
@app.route('/api/v1/process/album')
def get_artist(path):
"""
Process for the requested Album
:param path: The Artist to get files for
:return: a status
"""
if path:
res = process_download(path)
else:
res = {'status': 501, 'message': 'Could not process download..'}
return res
@app.route('/api/v1/get/queue')
def get_queue():
album_ids = Album.search([('downloaded', '=', False)])
data = {}
if album_ids:
data.update({'album_ids': album_ids})
return render_template('download_queue.html', **data)
if __name__ == "__main__":
print('Starting App...')
app.run(host="0.0.0.0", debug=True)

@ -2,8 +2,8 @@ FROM debian:bullseye-slim
RUN apt update && apt upgrade -y
RUN apt install ffmpeg -y
RUN apt install curl python3-pip -y
ADD . /code
WORKDIR /code
ADD . /python
WORKDIR /python
RUN pip3 install -r requirements.txt
ENV FLASK_APP=app

@ -1,37 +1,34 @@
import shutil
import os
from database import Model
from const import *
from .yt_dlp_logger import *
import wget
import yt_dlp
from .yt_dlp_logger import YtDlpLogger, yt_dlp_log_hook
Album = Model('album')
MEDIA_FOLDER = '/var/www/html/storage/app/music'
def download_album(album):
def download_album(album, artist):
"""
Take a list of albums and process the downloads
Take an artist and album dict and use yt-dlp to download the album to the local filestore.
:param album: Dict of album data
:return:
:return: dict of save data
"""
artist = album.get('artist')
artist_path = MEDIA_FOLDER + '/%s' % artist
response = {
'artist': {},
'album': {},
}
artist_path = MEDIA_FOLDER + '/%s' % artist.get('name')
if not os.path.exists(artist_path):
# Make artist folder and copy the artist image, add
os.mkdir(artist_path)
shutil.copy2(artist.get('image'), artist_path + '/artist.jpg')
response['artist'] = {'url_local': artist_path}
print('---')
# Create album folder
album_title = album.get('album')
album_title = album.get('name')
album_path = artist_path + '/%s' % album_title
if not os.path.exists(album_path):
os.mkdir(album_path)
# Save album cover
if album.get('cover'):
try:
download_file(album.get('cover'), album_path)
except Exception as e:
print("Warning: %s" % e)
shutil.copy2(album.get('image'), album_path + '/album.jpg')
# Download album
ydl_opts = {
@ -45,18 +42,13 @@ def download_album(album):
'preferredcodec': 'mp3',
}]
}
download_url = 'https://music.youtube.com/' + album.get('url_remote')
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
try:
error_code = ydl.download('https://youtube.com' + album.get('link'))
ydl.download(download_url)
except Exception as e:
print('!!!!!!!!!')
print('yt-dlp download failed: =========')
print(e)
finally:
Album.unlink(album['id'])
def download_file(url, output):
filename = wget.download(url, out=output)
os.rename(filename, output + '/album.jpg')
return filename
response['album'] = {'url_local': album_path}
return response

Loading…
Cancel
Save