brett.spaulding
/
getDiscography
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: brett.spaulding:main
Branches Tags
brett.spaulding:main
brett.spaulding:refactor_total
..
compare: brett.spaulding:refactor_total
Branches Tags
brett.spaulding:main
brett.spaulding:refactor_total

No commits in common. 'main' and 'refactor_total' have entirely different histories.
main ... refactor_t

13 changed files with 94 additions and 174 deletions
Show Stats

8
docker-compose.yml
Unescape View File

@ -132,7 +132,7 @@ services:
chrome: chrome:
image: selenium/node-chrome:nightly image: selenium/node-chrome:nightly
shm_size: 4gb shm_size: 8gb
networks: networks:
- laravel - laravel
depends_on: depends_on:
@ -141,8 +141,8 @@ services:
- SE_EVENT_BUS_HOST=selenium-hub - SE_EVENT_BUS_HOST=selenium-hub
- SE_EVENT_BUS_PUBLISH_PORT=4442 - SE_EVENT_BUS_PUBLISH_PORT=4442
- SE_EVENT_BUS_SUBSCRIBE_PORT=4443 - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
- SE_NODE_MAX_SESSIONS=2 - SE_NODE_MAX_SESSIONS=15
- SE_NODE_MAX_SESSION=2 - SE_NODE_MAX_SESSION=15
# edge: # edge:
# image: selenium/node-edge:nightly # image: selenium/node-edge:nightly
@ -177,7 +177,7 @@ services:
networks: networks:
- laravel - laravel
environment: environment:
JAVA_OPTS: "-Xmx4g -Xms2g" JAVA_OPTS: "-Xmx8g -Xms2g"
container_name: selenium-hub container_name: selenium-hub
ports: ports:
- "4442:4442" - "4442:4442"

6
php/src/app/Console/Commands/ProcessArtistQueue.php
Unescape View File

@ -28,10 +28,10 @@ class ProcessArtistQueue extends Command
public function handle() public function handle()
{ {
// This queue will prompt the scraping of all artist albums, mark done when complete // This queue will prompt the scraping of all artist albums, mark done when complete
$artist_queue = ArtistQueue::where('state', 'pending')->get(); $artists = ArtistQueue::where('state', 'pending')->get();
$bar = new ProgressBar($this->output, count($artist_queue)); $bar = new ProgressBar($this->output, count($artists));
$bar->start(); $bar->start();
foreach ($artist_queue as $artist) { foreach ($artists as $artist) {
$artist->state = 'in_progress'; $artist->state = 'in_progress';
$artist->save(); $artist->save();
$artist->process_artist(); $artist->process_artist();

38
php/src/app/Http/Controllers/ApiController.php
Unescape View File

@ -2,7 +2,6 @@
namespace App\Http\Controllers; namespace App\Http\Controllers;
use App\Jobs\RunArtistQueue;
use App\Models\AlbumQueue; use App\Models\AlbumQueue;
use App\Models\Artist; use App\Models\Artist;
use App\Models\WebDriver; use App\Models\WebDriver;
@ -24,7 +23,7 @@ class ApiController extends Controller
'name' => $artist->name, 'name' => $artist->name,
'url_remote' => $artist->url_remote, 'url_remote' => $artist->url_remote,
'state' => $artist->state, 'state' => $artist->state,
'thumbnail' => str_replace('/var/www/html/public', '', $artist->image), 'thumbnail' => $artist->thumbnail,
]; ];
} }
$response = json_encode(array('data' => $data)); $response = json_encode(array('data' => $data));
@ -38,16 +37,14 @@ class ApiController extends Controller
foreach ($album_queue as $queue) { foreach ($album_queue as $queue) {
$album = $queue->album; $album = $queue->album;
$artist = $album->artist; $artist = $album->artist;
if ($album && $artist) { $response[] = [
$response[] = [ 'name' => $album->name,
'name' => $album->name, 'artist_id' => $artist->toArray(),
'artist_id' => $artist->toArray(), 'url_remote' => $album->url_remote,
'url_remote' => $album->url_remote, 'thumbnail' => $album->thumbnail,
'thumbnail' => str_replace('/var/www/html/public', '', $album->image), 'image' => $album->image,
'image' => str_replace('/var/www/html/public', '', $album->image), 'state' => $queue->state,
'state' => $queue->state, ];
];
}
} }
return json_encode($response); return json_encode($response);
} }
@ -59,7 +56,7 @@ class ApiController extends Controller
public function queue_artist_run() public function queue_artist_run()
{ {
ArtistQueue::run_queue(); Artisan::queue('app:process-artist-queue');
} }
public function search_artist(string $artist) public function search_artist(string $artist)
@ -82,16 +79,15 @@ class ApiController extends Controller
public function queue_waiting() public function queue_waiting()
{ {
$data = array('queue' => false);
$queue = AlbumQueue::where('state', 'pending')->first(); $queue = AlbumQueue::where('state', 'pending')->first();
if (!is_null($queue)) { $album = $queue->album;
$album = $queue->album; $artist = $album->artist;
$artist = $album->artist;
$queue->state = 'in_progress';
$queue->save();
$data = array('queue' => $queue->toArray(), 'album' => $album->toArray(), 'artist' => $artist->toArray());
} \Log::info('======================');
\Log::info('Queue running for album: ' . $album->name);
$queue->state = 'in_progress';
$queue->save();
$data = array('queue' => $queue->toArray(), 'album' => $album->toArray(), 'artist' => $artist->toArray());
return json_encode($data); return json_encode($data);
} }

5
php/src/app/Models/Album.php
Unescape View File

@ -21,11 +21,6 @@ class Album extends Model
$this->save(); $this->save();
} }
public function getAlbumImageLocation()
{
return str_replace('/var/www/html', '', $this->image);
}
public static function findByArtistTitle(Artist $artist, string $name) public static function findByArtistTitle(Artist $artist, string $name)
{ {
return self::where('name', '=', $name)->where('artist_id', '=', $artist->id)->first(); return self::where('name', '=', $name)->where('artist_id', '=', $artist->id)->first();

15
php/src/app/Models/AlbumQueue.php
Unescape View File

@ -9,25 +9,18 @@ class AlbumQueue extends Model
{ {
use HasFactory; use HasFactory;
public function enqueue($album): bool public function enqueue($album_id): bool
{ {
$result = false; $result = false;
$album_queued = AlbumQueue::where('album_id', $album->id)->first(); $album_queued = AlbumQueue::where('album_id', $album_id->id)->first();
if (is_null($album_queued)) { if (is_null($album_queued) && $album_id->state === 'pending') {
$this->album_id = $album->id; $this->album_id = $album_id->id;
$this->save(); $this->save();
$result = true; $result = true;
} }
return $result; return $result;
} }
public static function addQueue($album_id): bool
{
$queue = new AlbumQueue();
$queue->enqueue($album_id);
return true;
}
public function album() public function album()
{ {
return $this->belongsTo(Album::class); return $this->belongsTo(Album::class);

5
php/src/app/Models/Artist.php
Unescape View File

@ -30,11 +30,6 @@ class Artist extends Model
return self::where('id', '=', $id)->get(); return self::where('id', '=', $id)->get();
} }
public function getArtistImageLocation()
{
return str_replace('/var/www/html', '', $this->image);
}
public static function addArtist(string $name, string $thumbnail, string $url_remote, string $image) public static function addArtist(string $name, string $thumbnail, string $url_remote, string $image)
{ {
$artist = new Artist(); $artist = new Artist();

21
php/src/app/Models/ArtistQueue.php
Unescape View File

@ -24,17 +24,12 @@ class ArtistQueue extends Model
return $result; return $result;
} }
public function artist()
{
return $this->belongsTo(Artist::class);
}
public function process_artist() public function process_artist()
{ {
// Scrape the artist page for image, and album data (image, url, name) // Scrape the artist page for image, and album data (image, url, name)
$driver = WebDriver::setUp(); $driver = WebDriver::setUp();
$artist_id = $this->artist; $artist_id = Artist::where('id', $this->artist_id)->get()->first();
if ($artist_id->count() > 0) { if ($artist_id->count() > 0) {
try { try {
$album_count = WebScraper::scrapeAlbums($driver, $artist_id); $album_count = WebScraper::scrapeAlbums($driver, $artist_id);
@ -43,7 +38,6 @@ class ArtistQueue extends Model
} catch (Exception $e) { } catch (Exception $e) {
\Log::warning('Failed to scrape albums: ' . $e->getMessage()); \Log::warning('Failed to scrape albums: ' . $e->getMessage());
} finally { } finally {
$artist_id->change_state('done');
$driver->quit(); $driver->quit();
} }
} else { } else {
@ -51,16 +45,5 @@ class ArtistQueue extends Model
} }
} }
public static function run_queue()
{
// This queue will prompt the scraping of all artist albums, mark done when complete
$artist_queue = ArtistQueue::where('state', 'pending')->get();
foreach ($artist_queue as $queue) {
$queue->state = 'in_progress';
$queue->save();
$queue->process_artist();
$queue->state = 'done';
$queue->save();
}
}
} }

90
php/src/app/Models/WebScraper.php
Unescape View File

@ -5,7 +5,6 @@ namespace App\Models;
use App\Utils\ImageUrl; use App\Utils\ImageUrl;
use Facebook\WebDriver\Remote\RemoteWebDriver; use Facebook\WebDriver\Remote\RemoteWebDriver;
use Facebook\WebDriver\WebDriverBy; use Facebook\WebDriver\WebDriverBy;
use Facebook\WebDriver\WebDriverAction;
use Facebook\WebDriver\WebDriverExpectedCondition; use Facebook\WebDriver\WebDriverExpectedCondition;
class WebScraper class WebScraper
@ -102,29 +101,6 @@ class WebScraper
return $response; return $response;
} }
public static function processAlbums($albumContainer, $artist)
{
$albumLink = $albumContainer->findElement(WebDriverBy::cssSelector('a'));
$albumHref = $albumLink->getAttribute('href');
$albumTitle = $albumLink->getAttribute('title');
$albumThumbnail = $albumLink->findElement(WebDriverBy::cssSelector('img'))->getAttribute('src');
// Resize image and save to file, provide path to data
$imageUrl = ImageUrl::modifyGoogleImageUrl($albumThumbnail);
$imageFileUrl = ImageUrl::save_img_url($imageUrl, 'album');
$data = [
'name' => $albumTitle,
'artist_id' => $artist->id,
'thumbnail' => $albumThumbnail,
'url_remote' => $albumHref, // TODO: Check here if the image is a 'gif' and not a URL
'image' => $imageFileUrl,
];
$album_id = Album::findOrCreateByName($artist, $albumTitle, $data);
$queued = AlbumQueue::addQueue($album_id);
return $queued;
}
/** /**
* Scrape the album data from given artist page, create new album records and queue those records for download * Scrape the album data from given artist page, create new album records and queue those records for download
* *
@ -135,57 +111,37 @@ class WebScraper
$url = 'https://music.youtube.com/' . $artist_id->url_remote; $url = 'https://music.youtube.com/' . $artist_id->url_remote;
$driver->get($url); $driver->get($url);
$response = 0; $response = 0;
sleep(3);
try { try {
$albumBtn = $driver->findElements(WebDriverBy::xpath('//a[text()="Albums"]')); $albumBtn = $driver->findElement(WebDriverBy::xpath('//a[text()="Albums"]'));
if ($albumBtn) { if ($albumBtn) {
$albumBtn[0]->click(); $albumBtn->click();
sleep(5); sleep(3);
$itemsContainer = $driver->findElements(WebDriverBy::cssSelector('#items')); $itemsContainer = $driver->findElements(WebDriverBy::cssSelector('#items'));
foreach ($itemsContainer as $item) { foreach ($itemsContainer as $item) {
$albumContainers = $item->findElements(WebDriverBy::cssSelector('.ytmusic-grid-renderer')); $albumContainers = $item->findElements(WebDriverBy::cssSelector('.ytmusic-grid-renderer'));
if ($albumContainers) { if ($albumContainers) {
foreach ($albumContainers as $albumContainer) { foreach ($albumContainers as $albumContainer) {
$response += 1; $response += 1;
WebScraper::processAlbums($albumContainer, $artist_id); $albumLink = $albumContainer->findElement(WebDriverBy::cssSelector('a'));
} $albumHref = $albumLink->getAttribute('href');
} $albumTitle = $albumLink->getAttribute('title');
} $albumThumbnail = $albumLink->findElement(WebDriverBy::cssSelector('img'))->getAttribute('src');
} else {
$ytRows = $driver->findElements(WebDriverBy::cssSelector('ytmusic-carousel-shelf-renderer')); // Resize image and save to file, provide path to data
foreach ($ytRows as $ytRow) { $imageUrl = ImageUrl::modifyGoogleImageUrl($albumThumbnail);
$contentGroup = $ytRow->findElements(WebDriverBy::cssSelector('#content-group')); $imageFileUrl = ImageUrl::save_img_url($imageUrl, 'album');
foreach ($contentGroup as $group) {
$groupName = $group->getText(); $data = [
if ($groupName == 'Albums') { 'name' => $albumTitle,
// Sometimes we don't have the option to click the albums button to filter 'artist_id' => $artist_id->id,
// Yet, the albums are in a carousel and the images won't load unless they are in view 'thumbnail' => $albumThumbnail,
$caroselNextButton = $driver->findElements(WebDriverBy::cssSelector('#next-items-button')); 'url_remote' => $albumHref,
try { 'image' => $imageFileUrl,
if ($caroselNextButton) { ];
// Youtube is smart enough to block this without an action $album_id = Album::findOrCreateByName($artist_id, $albumTitle, $data);
for ($i = 0; $i <= 3; $i++) {
if ($caroselNextButton[0]->isEnabled()) { $album_queue = new AlbumQueue();
$action = $driver->action(); $album_queue->enqueue($album_id);
$action->moveToElement($caroselNextButton[0])->click()->perform();
sleep(5);
}
sleep(2);
}
}
} catch (\Exception $e) {
\Log::info($e);
}
$itemsContainer = $ytRow->findElements(WebDriverBy::cssSelector('#items'));
foreach ($itemsContainer as $item) {
$albumContainers = $item->findElements(WebDriverBy::cssSelector('ytmusic-two-row-item-renderer'));
if ($albumContainers) {
foreach ($albumContainers as $albumContainer) {
WebScraper::processAlbums($albumContainer, $artist_id);
}
}
}
} }
} }
} }

60
php/src/public/js/app.js
Unescape View File

@ -13,14 +13,12 @@ function requestQueue() {
} }
function template_artist_result(element) { function template_artist_result(element) {
let image_src = element.image.replace('/var/www/html/public', '');
console.log(image_src);
return ` return `
<div class="card w-100 p-2 mb-2"> <div class="card w-100 p-2 mb-2">
<div class="container-fluid"> <div class="container-fluid">
<div class="row"> <div class="row">
<div class="col-3"> <div class="col-3">
<img src="${image_src}" width="72px" height="72px" style="border-radius: 12px;"/> <img src="${element.thumbnail}" width="72px" height="72px" style="border-radius: 12px;"/>
</div> </div>
<div class="col-9 m-auto"> <div class="col-9 m-auto">
<h4>${element.name}</h4> <h4>${element.name}</h4>
@ -66,7 +64,7 @@ function artist_queue_toggle(element) {
url: `/api/queue/artist/${self.data('artist_id')}`, url: `/api/queue/artist/${self.data('artist_id')}`,
success: () => { success: () => {
proc_notification('success', 'Queued Download', `Artist ${artist_name} Queued for Download!`); proc_notification('success', 'Queued Download', `Artist ${artist_name} Queued for Download!`);
// ArtistTable.ajax.reload(); ArtistTable.ajax.reload();
}, },
error: (response) => { error: (response) => {
console.log(response); console.log(response);
@ -90,32 +88,39 @@ function bind_action_buttons() {
}); });
$('#download_btn').on('click', () => { $('#download_btn').on('click', () => {
loader.fadeIn(300);
let artist = $('#search_bar').val(); let artist = $('#search_bar').val();
// Send request to server // Send request to server
setTimeout(() => { setTimeout(() => {
if (artist) {
if (artist == '') { console.log('Sending search request...');
return proc_notification('error', 'Whoopsie!', 'You need to add an artist, c\'mon man!');; $.ajax({
url: `/artist/${artist}`,
success: (response) => {
console.log('Receiving response...');
console.log(response);
console.log('===========');
icon = 'success';
let html = construct_artist_result_html(response);
proc_notification(icon, 'Shazam!', html);
ArtistTable.ajax.reload();
$('#search_bar').val('');
loader.fadeOut(700);
},
error: (response) => {
console.log('Receiving response...');
console.log(response);
console.log('===========');
proc_notification(icon, 'What the flip?!', response.statusText);
loader.fadeOut(700);
}
});
} else {
proc_notification(icon, 'Whoopsie!', 'You need to add an artist, c\'mon man!');
loader.fadeOut(700);
} }
loader.fadeIn(300);
$.ajax({
url: `/artist/${artist}`,
success: (response) => {
let html = construct_artist_result_html(response);
proc_notification('success', 'Shazam!', html);
ArtistTable.ajax.reload();
$('#search_bar').val('');
loader.fadeOut(700);
},
error: (response) => {
proc_notification('error', 'What the flip?!', response.statusText);
loader.fadeOut(700);
}
});
}, 10); }, 10);
}); });
@ -155,10 +160,9 @@ $(document).ready(function () {
{ {
data: 'id', orderable: false, render: (data, type, row) => { data: 'id', orderable: false, render: (data, type, row) => {
let stateDiable = row.state === 'in_progress' ? 'disabled' : ''; let stateDiable = row.state === 'in_progress' ? 'disabled' : '';
let stateClass = row.state === 'done' ? 'btn-success' : 'btn-primary'; let stateClass = row.state === 'in_progress' ? '' : 'btn-primary';
let artist_name = row.name; let artist_name = row.name;
let button_icon = row.state === 'done' ? '<i class="las la-redo-alt"></i>' : '<i class="las la-cloud-download-alt"></i>'; return `<button class="btn ${stateClass}" style="float: right;" data-artist_name="${artist_name}" data-artist_id="${data}" onclick="artist_queue_toggle(this)" ${stateDiable}><i class="las la-cloud-download-alt"></i> Download</button>`
return `<button class="btn ${stateClass}" style="float: right;" data-artist_name="${artist_name}" data-artist_id="${data}" onclick="artist_queue_toggle(this)" ${stateDiable}>${button_icon} Download</button>`
} }
} }
], ],

2
php/src/resources/views/components/download-queue.blade.php
Unescape View File

@ -17,7 +17,7 @@
</div> </div>
</template> </template>
<!-- Album Art --> <!-- Album Art -->
<img :src="album.image" class="img-fluid rounded-start" <img :src="album.thumbnail" class="img-fluid rounded-start"
:alt="album.name" style="width: 100%; height: 100%; min-height: 180px;"> :alt="album.name" style="width: 100%; height: 100%; min-height: 180px;">
</div> </div>

2
php/src/resources/views/modals/modal-catalog.blade.php
Unescape View File

@ -13,7 +13,7 @@
<div id="modal_content"> <div id="modal_content">
<div class="card"> <div class="card">
<table id="artistsCatalogDatatable" class="stripe"> <table id="artistsCatalogDatatable">
<thead> <thead>
<tr> <tr>
<th></th> <th></th>

1
php/src/routes/console.php
Unescape View File

@ -7,3 +7,4 @@ Artisan::command('inspire', function () {
$this->comment(Inspiring::quote()); $this->comment(Inspiring::quote());
})->purpose('Display an inspiring quote')->hourly(); })->purpose('Display an inspiring quote')->hourly();
Artisan::command('app:process-artist-queue')->everyMinute();

15
python/app.py
Unescape View File

@ -13,28 +13,25 @@ app = Flask(__name__)
redis = Redis(host='redis', port=6379) redis = Redis(host='redis', port=6379)
def process_artist_queue(): # def process_artist_queue():
print('Running Artist Queue process..') # requests.get('http://nginx/api/queue/artists/run')
print('---') # return
requests.get('http://nginx/api/queue/artists/run')
return
def process_album_queue(): def process_album_queue():
print('Running Album Queue Process..') print('Running Album Queue Process..')
print('---') print('---')
response = requests.get('http://nginx/api/album/queue') response = requests.get('http://nginx/api/album/queue')
data = response.json() data = response.json()
artist = data.get('artist', False) artist = data.get('artist')
album = data.get('album', False) album = data.get('album')
queue = data.get('queue') queue = data.get('queue')
if not queue == False and artist and album: if artist and album and queue:
result = download_album(album, artist) result = download_album(album, artist)
requests.post('http://nginx/api/album/queue/update/%s' % queue.get('id'), json=result) requests.post('http://nginx/api/album/queue/update/%s' % queue.get('id'), json=result)
return return
cron = BackgroundScheduler({'apscheduler.job_defaults.max_instances': 1}, daemon=True) cron = BackgroundScheduler({'apscheduler.job_defaults.max_instances': 1}, daemon=True)
cron.add_job(process_album_queue, 'interval', minutes=1) cron.add_job(process_album_queue, 'interval', minutes=1)
cron.add_job(process_artist_queue, 'interval', minutes=1)
cron.start() cron.start()
if __name__ == "__main__": if __name__ == "__main__":

Write Preview
Loading…
Cancel
Save