brett.spaulding
/
getDiscography
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[FIX] php: Handling for albums section if no link

Browse Source
main
Brett Spaulding 1 year ago
parent 1807deed67
commit ec39ea48be
6 changed files with 83 additions and 39 deletions
Show Stats Download Patch File Download Diff File

12
php/src/app/Http/Controllers/ApiController.php
Unescape View File

@ -2,6 +2,7 @@
namespace App\Http\Controllers;
use Illuminate\Support\Facades\Artisan;
use App\Models\AlbumQueue;
use App\Models\Artist;
use App\Models\WebDriver;
@ -56,6 +57,8 @@ class ApiController extends Controller
public function queue_artist_run()
{
\Log::info('===========================');
\Log::info('Queue running for Artists..');
Artisan::queue('app:process-artist-queue');
}
@ -79,15 +82,18 @@ class ApiController extends Controller
public function queue_waiting()
{
\Log::info('===========================');
\Log::info('Queue running for Albums..');
$data = array('queue' => false);
$queue = AlbumQueue::where('state', 'pending')->first();
if (!is_null($queue)) {
$album = $queue->album;
$artist = $album->artist;
\Log::info('======================');
\Log::info('Queue running for album: ' . $album->name);
$queue->state = 'in_progress';
$queue->save();
$data = array('queue' => $queue->toArray(), 'album' => $album->toArray(), 'artist' => $artist->toArray());
}
return json_encode($data);
}

8
php/src/app/Models/ArtistQueue.php
Unescape View File

@ -24,12 +24,17 @@ class ArtistQueue extends Model
return $result;
}
public function artist()
{
return $this->belongsTo(Artist::class);
}
public function process_artist()
{
// Scrape the artist page for image, and album data (image, url, name)
$driver = WebDriver::setUp();
$artist_id = Artist::where('id', $this->artist_id)->get()->first();
$artist_id = $this->artist;
if ($artist_id->count() > 0) {
try {
$album_count = WebScraper::scrapeAlbums($driver, $artist_id);
@ -38,6 +43,7 @@ class ArtistQueue extends Model
} catch (Exception $e) {
\Log::warning('Failed to scrape albums: ' . $e->getMessage());
} finally {
$artist_id->change_state('done');
$driver->quit();
}
} else {

71
php/src/app/Models/WebScraper.php
Unescape View File

@ -101,6 +101,29 @@ class WebScraper
return $response;
}
public static function processAlbums($albumContainer, $artist)
{
$albumLink = $albumContainer->findElement(WebDriverBy::cssSelector('a'));
$albumHref = $albumLink->getAttribute('href');
$albumTitle = $albumLink->getAttribute('title');
$albumThumbnail = $albumLink->findElement(WebDriverBy::cssSelector('img'))->getAttribute('src');
// Resize image and save to file, provide path to data
$imageUrl = ImageUrl::modifyGoogleImageUrl($albumThumbnail);
$imageFileUrl = ImageUrl::save_img_url($imageUrl, 'album');
$data = [
'name' => $albumTitle,
'artist_id' => $artist->id,
'thumbnail' => $albumThumbnail,
'url_remote' => $albumHref,
'image' => $imageFileUrl,
];
$album_id = Album::findOrCreateByName($artist, $albumTitle, $data);
$album_queue = new AlbumQueue();
$album_queue->enqueue($album_id);
}
/**
* Scrape the album data from given artist page, create new album records and queue those records for download
*
@ -112,9 +135,11 @@ class WebScraper
$driver->get($url);
$response = 0;
try {
$albumBtn = $driver->findElement(WebDriverBy::xpath('//a[text()="Albums"]'));
\Log::info('Looking for Albums button..');
$albumBtn = $driver->findElements(WebDriverBy::xpath('//a[text()="Albums"]'));
if ($albumBtn) {
$albumBtn->click();
\Log::info('Clicking on located Albums button..');
$albumBtn[0]->click();
sleep(3);
$itemsContainer = $driver->findElements(WebDriverBy::cssSelector('#items'));
foreach ($itemsContainer as $item) {
@ -122,29 +147,33 @@ class WebScraper
if ($albumContainers) {
foreach ($albumContainers as $albumContainer) {
$response += 1;
$albumLink = $albumContainer->findElement(WebDriverBy::cssSelector('a'));
$albumHref = $albumLink->getAttribute('href');
$albumTitle = $albumLink->getAttribute('title');
$albumThumbnail = $albumLink->findElement(WebDriverBy::cssSelector('img'))->getAttribute('src');
// Resize image and save to file, provide path to data
$imageUrl = ImageUrl::modifyGoogleImageUrl($albumThumbnail);
$imageFileUrl = ImageUrl::save_img_url($imageUrl, 'album');
$data = [
'name' => $albumTitle,
'artist_id' => $artist_id->id,
'thumbnail' => $albumThumbnail,
'url_remote' => $albumHref,
'image' => $imageFileUrl,
];
$album_id = Album::findOrCreateByName($artist_id, $albumTitle, $data);
WebScraper::processAlbums($albumContainer, $artist_id);
}
}
}
} else {
\Log::info('Could not locate Albums button');
$album_queue = new AlbumQueue();
$album_queue->enqueue($album_id);
$ytRows = $driver->findElements(WebDriverBy::cssSelector('ytmusic-carousel-shelf-renderer'));
foreach ($ytRows as $ytRow) {
$contentGroup = $ytRow->findElements(WebDriverBy::cssSelector('#content-group'));
foreach ($contentGroup as $group) {
$groupName = $group->getText();
if ($groupName == 'Albums') {
$itemsContainer = $ytRow->findElements(WebDriverBy::cssSelector('#items'));
foreach ($itemsContainer as $item) {
$albumContainers = $item->findElements(WebDriverBy::cssSelector('ytmusic-two-row-item-renderer'));
if ($albumContainers) {
foreach ($albumContainers as $albumContainer) {
WebScraper::processAlbums($albumContainer, $artist_id);
}
}
}
}
}
}
}
} catch (\Exception $e) {
\Log::warning('Failed to scrape albums: ---------');

5
php/src/public/js/app.js
Unescape View File

@ -160,9 +160,10 @@ $(document).ready(function () {
{
data: 'id', orderable: false, render: (data, type, row) => {
let stateDiable = row.state === 'in_progress' ? 'disabled' : '';
let stateClass = row.state === 'in_progress' ? '' : 'btn-primary';
let stateClass = row.state === 'done' ? 'btn-success' : 'btn-primary';
let artist_name = row.name;
return `<button class="btn ${stateClass}" style="float: right;" data-artist_name="${artist_name}" data-artist_id="${data}" onclick="artist_queue_toggle(this)" ${stateDiable}><i class="las la-cloud-download-alt"></i> Download</button>`
let button_icon = row.state === 'done' ? '<i class="las la-redo-alt"></i>' : '<i class="las la-cloud-download-alt"></i>';
return `<button class="btn ${stateClass}" style="float: right;" data-artist_name="${artist_name}" data-artist_id="${data}" onclick="artist_queue_toggle(this)" ${stateDiable}>${button_icon} Download</button>`
}
}
],

1
php/src/routes/console.php
Unescape View File

@ -7,4 +7,3 @@ Artisan::command('inspire', function () {
$this->comment(Inspiring::quote());
})->purpose('Display an inspiring quote')->hourly();
Artisan::command('app:process-artist-queue')->everyMinute();

15
python/app.py
Unescape View File

@ -13,25 +13,28 @@ app = Flask(__name__)
redis = Redis(host='redis', port=6379)
# def process_artist_queue():
# requests.get('http://nginx/api/queue/artists/run')
# return
def process_artist_queue():
print('Running Artist Queue process..')
print('---')
requests.get('http://nginx/api/queue/artists/run')
return
def process_album_queue():
print('Running Album Queue Process..')
print('---')
response = requests.get('http://nginx/api/album/queue')
data = response.json()
artist = data.get('artist')
album = data.get('album')
artist = data.get('artist', False)
album = data.get('album', False)
queue = data.get('queue')
if artist and album and queue:
if not queue == False and artist and album:
result = download_album(album, artist)
requests.post('http://nginx/api/album/queue/update/%s' % queue.get('id'), json=result)
return
cron = BackgroundScheduler({'apscheduler.job_defaults.max_instances': 1}, daemon=True)
cron.add_job(process_album_queue, 'interval', minutes=1)
cron.add_job(process_artist_queue, 'interval', minutes=1)
cron.start()
if __name__ == "__main__":

Write Preview
Loading…
Cancel
Save