diff --git a/php/src/.gitignore b/php/src/.gitignore index ef1a7de..46ad5f4 100644 --- a/php/src/.gitignore +++ b/php/src/.gitignore @@ -19,3 +19,4 @@ yarn-error.log /.idea /.vscode public/images/artist +public/images/album diff --git a/php/src/app/Console/Commands/ProcessArtistQueue.php b/php/src/app/Console/Commands/ProcessArtistQueue.php index a77765f..ad50f02 100644 --- a/php/src/app/Console/Commands/ProcessArtistQueue.php +++ b/php/src/app/Console/Commands/ProcessArtistQueue.php @@ -4,6 +4,7 @@ namespace App\Console\Commands; use App\Models\ArtistQueue; use Illuminate\Console\Command; +use Symfony\Component\Console\Helper\ProgressBar; class ProcessArtistQueue extends Command { @@ -26,9 +27,12 @@ class ProcessArtistQueue extends Command */ public function handle() { - $records = ArtistQueue::where('state', 'in_progress')->get(); + $records = ArtistQueue::where('state', 'pending')->get(); + $bar = new ProgressBar($this->output, count($records)); + $bar->start(); foreach ($records as $record) { - + $record->process_artist(); + $bar->advance(); } } } diff --git a/php/src/app/Models/Album.php b/php/src/app/Models/Album.php index e009242..0a00dd8 100644 --- a/php/src/app/Models/Album.php +++ b/php/src/app/Models/Album.php @@ -4,8 +4,51 @@ namespace App\Models; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; +use Mockery\Exception; class Album extends Model { use HasFactory; + + public function change_state(string $state) + { + $available_states = array("pending", "in_progress", "done"); + if (!in_array($state, $available_states)){ + throw new Exception('Invalid state'); + } + $this->state = $state; + $this->save(); + } + + public static function findByName($name) + { + return self::where('name', '=', $name)->get(); + } + + public static function findById($id) + { + return self::where('id', '=', $id)->get(); + } + + public static function addAlbum(string $name, string $thumbnail, string $url_remote, string $image, $artist_id) + { + $album = new Album(); + $album->name = $name; + $album->artist_id = $artist_id; + $album->url_remote = $url_remote; + $album->thumbnail = $thumbnail; + $album->image = $image; + $album->save(); + return $album; + } + + public static function findOrCreateByName(string $name, array $data = []) + { + $album = self::findByName($name)->first(); + if (!$album && $data) { + $album = self::addAlbum($data['name'], $data['thumbnail'], $data['url_remote'], $data['image'], $data['artist_id']); + } + return $album; + } + } diff --git a/php/src/app/Models/AlbumQueue.php b/php/src/app/Models/AlbumQueue.php index ebc2920..eb50660 100644 --- a/php/src/app/Models/AlbumQueue.php +++ b/php/src/app/Models/AlbumQueue.php @@ -9,12 +9,19 @@ class AlbumQueue extends Model { use HasFactory; - public function enqueue() + public function enqueue(int $id): bool { - // Add albums to queue for download + $result = false; + $album_id = Album::findById($id)->first(); + if ($album_id->count() > 0 && $album_id->state === 'pending') { + $this->album_id = $album_id->id; + $this->save(); + $result = true; + } + return $result; } - public function process_queue() + public function process_album() { // Either python pings to process the queue or laravel will send the data to python for processing } diff --git a/php/src/app/Models/ArtistQueue.php b/php/src/app/Models/ArtistQueue.php index cee8904..61b2388 100644 --- a/php/src/app/Models/ArtistQueue.php +++ b/php/src/app/Models/ArtistQueue.php @@ -29,9 +29,9 @@ class ArtistQueue extends Model { // Scrape the artist page for image, and album data (image, url, name) $driver = WebDriver::setUp(); - $artist_id = Artist::where('id', $this->artist_id)->get(); + $artist_id = Artist::where('id', $this->artist_id)->get()->first(); if ($artist_id->count() > 0) { - + $response = WebScraper::scrapeAlbums($driver, $artist_id); } else { throw new Exception('The Artist ID provided to the queue does not exist.'); } diff --git a/php/src/app/Models/WebScraper.php b/php/src/app/Models/WebScraper.php index fdfb96c..08037d6 100644 --- a/php/src/app/Models/WebScraper.php +++ b/php/src/app/Models/WebScraper.php @@ -25,7 +25,7 @@ class WebScraper // Resize image and save to file, provide path to data $imageUrl = ImageUrl::modifyGoogleImageUrl($artistThumbnail); - $imageFileUrl = ImageUrl::save_img_url($imageUrl); + $imageFileUrl = ImageUrl::save_img_url($imageUrl, 'artist'); $data = [ 'name' => $artistName, @@ -76,7 +76,7 @@ class WebScraper // Resize image and save to file, provide path to data $imageUrl = ImageUrl::modifyGoogleImageUrl($artistThumbnail); - $imageFileUrl = ImageUrl::save_img_url($imageUrl); + $imageFileUrl = ImageUrl::save_img_url($imageUrl, 'artist'); // Create if we don't have it yet $data = [ @@ -100,4 +100,50 @@ class WebScraper return $response; } + + /** + * Scrape the album data from given artist page, create new album records and queue those records for download + * + * @return RemoteWebDriver + */ + public static function scrapeAlbums($driver, $artist_id): array + { + $url = 'https://music.youtube.com/' . $artist_id->url_remote; + $driver->get($url); + $response = []; + $albumBtn = $driver->findElement(WebDriverBy::xpath('//a[text()="Albums"]')); + if ($albumBtn) { + $albumBtn->click(); + sleep(3); + $itemsContainer = $driver->findElements(WebDriverBy::cssSelector('#items')); + foreach ($itemsContainer as $item) { + $albumContainers = $item->findElements(WebDriverBy::cssSelector('.ytmusic-grid-renderer')); + if ($albumContainers) { + foreach ($albumContainers as $albumContainer) { + $albumLink = $albumContainer->findElement(WebDriverBy::cssSelector('a')); + $albumHref = $albumLink->getAttribute('href'); + $albumTitle = $albumLink->getAttribute('title'); + $albumThumbnail = $albumLink->findElement(WebDriverBy::cssSelector('img'))->getAttribute('src'); + + // Resize image and save to file, provide path to data + $imageUrl = ImageUrl::modifyGoogleImageUrl($albumThumbnail); + $imageFileUrl = ImageUrl::save_img_url($imageUrl, 'album'); + + $data = [ + 'name' => $albumTitle, + 'artist_id' => $artist_id->id, + 'thumbnail' => $albumThumbnail, + 'url_remote' => $albumHref, + 'image' => $imageFileUrl, + ]; + $album_id = Album::findOrCreateByName($albumTitle, $data); + + $album_queue = new AlbumQueue(); + $album_queue_id = $album_queue->enqueue($album_id->id); + } + } + } + } + return $response; + } } diff --git a/php/src/app/Utils/ImageUrl.php b/php/src/app/Utils/ImageUrl.php index 60e2224..66bd771 100644 --- a/php/src/app/Utils/ImageUrl.php +++ b/php/src/app/Utils/ImageUrl.php @@ -27,15 +27,17 @@ class ImageUrl * Save an image from a Google Image URL to the local filesystem. * * @param string $url The modified URL of the image to download. + * @param string $type The public directory to save to, needs to be either 'artist' or 'album'. * @return string The path to the saved image file, or empty string if the file already exists. */ - public static function save_img_url(string $url): string + public static function save_img_url(string $url, string $type): string { + // Get the filename from the URL $filename = basename($url); // Create a directory for the images (if it doesn't exist) - $imagesDir = public_path('images/artist'); + $imagesDir = public_path('images/' . $type); if (!is_dir($imagesDir)) { mkdir($imagesDir, 0777, true); } diff --git a/php/src/database/migrations/2024_08_11_185222_add_albums_field_thumbnail.php b/php/src/database/migrations/2024_08_11_185222_add_albums_field_thumbnail.php new file mode 100644 index 0000000..0ee97f7 --- /dev/null +++ b/php/src/database/migrations/2024_08_11_185222_add_albums_field_thumbnail.php @@ -0,0 +1,28 @@ +string('thumbnail')->nullable()->after('name'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('albums', function (Blueprint $table) { + // + }); + } +}; diff --git a/php/src/database/migrations/2024_08_11_185930_fix_albums_field__artist_id.php b/php/src/database/migrations/2024_08_11_185930_fix_albums_field__artist_id.php new file mode 100644 index 0000000..4f28156 --- /dev/null +++ b/php/src/database/migrations/2024_08_11_185930_fix_albums_field__artist_id.php @@ -0,0 +1,28 @@ +foreignId('artist_id')->change()->nullable()->constrained('artists'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('albums', function (Blueprint $table) { + // + }); + } +}; diff --git a/php/src/database/migrations/2024_08_11_190709_remove_albums__artist_id.php b/php/src/database/migrations/2024_08_11_190709_remove_albums__artist_id.php new file mode 100644 index 0000000..f6c2c8d --- /dev/null +++ b/php/src/database/migrations/2024_08_11_190709_remove_albums__artist_id.php @@ -0,0 +1,28 @@ +dropForeign(['artist_id']); + $table->dropColumn('artist_id'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('albums', function (Blueprint $table) { + // + }); + } +}; diff --git a/php/src/database/migrations/2024_08_11_190720_add_albums__artist_id.php b/php/src/database/migrations/2024_08_11_190720_add_albums__artist_id.php new file mode 100644 index 0000000..d73a912 --- /dev/null +++ b/php/src/database/migrations/2024_08_11_190720_add_albums__artist_id.php @@ -0,0 +1,28 @@ +foreignId('artist_id')->constrained('artists')->onDelete('cascade'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('albums', function (Blueprint $table) { + // + }); + } +};