diff --git a/.gitignore b/.gitignore index 3c130ad..d0327f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ -./navidrome/ -./music/ + +navidrome/ +music/ .idea/ diff --git a/__pycache__/database.cpython-37.pyc b/__pycache__/database.cpython-37.pyc index c1b33fc..f93de1d 100644 Binary files a/__pycache__/database.cpython-37.pyc and b/__pycache__/database.cpython-37.pyc differ diff --git a/app.py b/app.py index 8e49ad9..41bf64b 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,9 @@ +import json +from apscheduler.schedulers.background import BackgroundScheduler from database import Model from flask import Flask, render_template from redis import Redis +from utils.download import download_album from utils.processor import process_download @@ -9,6 +12,27 @@ redis = Redis(host='redis', port=6379) Album = Model('album') +def process_downloads(): + print('Processing Downloads..') + pending_downloads = Album.search([('downloaded', '=', False), ('downloading', '=', False)]) + if pending_downloads: + ready_album = pending_downloads[:1] + if ready_album: + album = ready_album[0] + print('...................') + print('Downloading Album..') + print(album) + Album.write(album['id'], {'downloading': True}) + download_album(album) + else: + return + + +cron = BackgroundScheduler({'apscheduler.job_defaults.max_instances': 3}, daemon=True) +cron.add_job(process_downloads, 'interval', minutes=1) +cron.start() + + @app.route('/') def index(): # redis.incr('hits') @@ -35,9 +59,9 @@ def get_artist(path): @app.route('/api/v1/get/queue') def get_queue(): album_ids = Album.search([('downloaded', '=', False)]) - data = {'album_ids': album_ids} - print('======================') - print(data) + data = {} + if album_ids: + data.update({'album_ids': album_ids}) return render_template('download_queue.html', **data) diff --git a/database.py b/database.py index 4f951ff..5797968 100644 --- a/database.py +++ b/database.py @@ -1,3 +1,4 @@ +import json import operator as oprtr from pysondb import PysonDB @@ -37,8 +38,6 @@ class Model: filtered_record_ids =[] for record in records: record_id = self.env.get_by_id(record) - print('===') - print(record_id) checklist = [] for param in params: field = param[0] @@ -48,6 +47,7 @@ class Model: passed = all(x for x in checklist) if passed: + record_id.update({'id': record}) filtered_record_ids.append(record_id) return filtered_record_ids @@ -77,7 +77,8 @@ class Model: return record_ids def write(self, record_id, vals): - self.env.update_by_id(record_id, vals) + record = self.env.update_by_id(record_id, vals) + return record def unlink(self, record_id): self.env.delete_by_id(record_id) diff --git a/database/album.json b/database/album.json index 4b6401d..882c823 100644 --- a/database/album.json +++ b/database/album.json @@ -9,85 +9,61 @@ "link" ], "data": { - "199074445401889600": { - "artist": "Inteus", - "downloaded": false, + "277061953508566187": { + "artist": "System Of A Down", + "downloaded": true, "downloading": false, - "link": "/playlist?list=OLAK5uy_nr-vlAyokNEb25RLBNe1XHsFo9gkvu2Pg&playnext=1&index=1", - "cover": "https://i9.ytimg.com/s_p/OLAK5uy_nr-vlAyokNEb25RLBNe1XHsFo9gkvu2Pg/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCN-N_qMG&rs=AOn4CLBo8tjoBHrUytlN1kfqTHYVMaVA4Q&v=1686079199", - "album": "Quick Revive" + "link": "/playlist?list=OLAK5uy_lvT9kmNao6-ZoqeO4Vs_dA5Ol80SXk3Ig&playnext=1&index=1", + "cover": "https://i9.ytimg.com/s_p/OLAK5uy_lvT9kmNao6-ZoqeO4Vs_dA5Ol80SXk3Ig/mqdefault.jpg?sqp=CMzWpaQGir7X7AMGCLH-l6QG&rs=AOn4CLAvH1W4KrlwCiNjnkCP0Od959EmRQ&v=1686503217", + "album": "Protect The Land - Genocidal Humanoidz" }, - "187732915376923467": { - "artist": "Inteus", - "downloaded": false, + "944002510455338759": { + "artist": "System Of A Down", + "downloaded": true, "downloading": false, - "link": "/playlist?list=OLAK5uy_lS0tCQXZ-1ppWJOMhaMel8GKGtnUzsvnU&playnext=1&index=1", - "cover": "https://i9.ytimg.com/s_p/OLAK5uy_lS0tCQXZ-1ppWJOMhaMel8GKGtnUzsvnU/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCPDOhaQG&rs=AOn4CLBa8Mmhg8B0A7ljEpCsig_bzTju0A&v=1686202224", - "album": "Winds of Paradise" + "link": "/playlist?list=OLAK5uy_k0JS7wXMiLcvCNgcbZunGaPiTt1KX3rY4&playnext=1&index=1", + "cover": "https://i9.ytimg.com/s_p/OLAK5uy_k0JS7wXMiLcvCNgcbZunGaPiTt1KX3rY4/mqdefault.jpg?sqp=CMzWpaQGir7X7AMGCMPCoKQG&rs=AOn4CLAXHNH_ikaut2-Lf_ZMvtbEw2QsJw&v=1686643011", + "album": "Hypnotize" }, - "206562046422345861": { - "artist": "Inteus", - "downloaded": false, + "408685349006703299": { + "artist": "System Of A Down", + "downloaded": true, "downloading": false, - "link": "/playlist?list=OLAK5uy_movr4fRkYisvKvX92G_D6fWfV1umrn3hs&playnext=1&index=1", - "cover": "https://i9.ytimg.com/s_p/OLAK5uy_movr4fRkYisvKvX92G_D6fWfV1umrn3hs/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCIKr_6MG&rs=AOn4CLDiszLRoh4cueTcK9h20CQ3KNpeXQ&v=1686099330", - "album": "Selection:3" + "link": "/playlist?list=OLAK5uy_kTJeZXfriqNa_00DjGWvAIICYRRiOISGs&playnext=1&index=1", + "cover": "https://i9.ytimg.com/s_p/OLAK5uy_kTJeZXfriqNa_00DjGWvAIICYRRiOISGs/mqdefault.jpg?sqp=CMzWpaQGir7X7AMGCOD8kKQG&rs=AOn4CLDilmmezso7Uga6UtwFkyZwWoajnQ&v=1686388320", + "album": "Mezmerize" }, - "321688139888393359": { - "artist": "Inteus", - "downloaded": false, + "294169533529189046": { + "artist": "System Of A Down", + "downloaded": true, "downloading": false, - "link": "/playlist?list=OLAK5uy_kW87DzVan7NyJ7j06XBmOHlg1WaFm38LA&playnext=1&index=1", - "cover": "https://i9.ytimg.com/s_p/OLAK5uy_kW87DzVan7NyJ7j06XBmOHlg1WaFm38LA/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCIDi_aMG&rs=AOn4CLA4dE1oa2TYSqb0pvfNTdK7ZSgERw&v=1686073600", - "album": "Bruh Moment (2020 Scraps)" + "link": "/playlist?list=OLAK5uy_nTgNbKalmllvDQkCl1AnGtl6qYlqHOV04&playnext=1&index=1", + "cover": "https://i9.ytimg.com/s_p/OLAK5uy_nTgNbKalmllvDQkCl1AnGtl6qYlqHOV04/mqdefault.jpg?sqp=CMzWpaQGir7X7AMGCLKGlKQG&rs=AOn4CLDLmCArMfYuSemTqvhr7ZFjgNRzdA&v=1686438706", + "album": "Steal This Album!" }, - "633598784099856320": { - "artist": "Inteus", - "downloaded": false, + "211435660709717567": { + "artist": "System Of A Down", + "downloaded": true, "downloading": false, - "link": "/playlist?list=OLAK5uy_n1DHIq4LZlVVBvyilCzrjxAyCmsBSRfY0&playnext=1&index=1", - "cover": "https://i9.ytimg.com/s_p/OLAK5uy_n1DHIq4LZlVVBvyilCzrjxAyCmsBSRfY0/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCJGzn6QG&rs=AOn4CLD_qLcb-Z1EabBDe3449f3d0mh3Cg&v=1686624657", - "album": "Yent Szn 3" + "link": "/playlist?list=OLAK5uy_kb1yvg1bLW2yKdKxO1qvzGK7pFl9TuFBw&playnext=1&index=1", + "cover": "https://i9.ytimg.com/s_p/OLAK5uy_kb1yvg1bLW2yKdKxO1qvzGK7pFl9TuFBw/mqdefault.jpg?sqp=CMzWpaQGir7X7AMGCOuEkaQG&rs=AOn4CLCiDAqW94eyFczMp1Lou8IRPhVNYw&v=1686389355", + "album": "Toxicity" }, - "131921253377227610": { - "artist": "Inteus", - "downloaded": false, + "661582160567415371": { + "artist": "System Of A Down", + "downloaded": true, "downloading": false, - "link": "/playlist?list=OLAK5uy_nm2zfpc5MDyzOp2ftFqdlHBQ-L3tE2BQM&playnext=1&index=1", - "cover": "https://i9.ytimg.com/s_p/OLAK5uy_nm2zfpc5MDyzOp2ftFqdlHBQ-L3tE2BQM/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCIPhgaQG&rs=AOn4CLBM_1qrbE_tAOEp51oGRYQMeTkRsw&v=1686139011", - "album": "Voyager" + "link": "/playlist?list=OLAK5uy_ld_QngiUfBzcKb1CnnoBS-b7xauS6N2Us&playnext=1&index=1", + "cover": "https://i9.ytimg.com/s_p/OLAK5uy_ld_QngiUfBzcKb1CnnoBS-b7xauS6N2Us/mqdefault.jpg?sqp=CMzWpaQGir7X7AMGCKCjjaQG&rs=AOn4CLCtH2z6aR7u9EemiwZEda7-S3ADuw&v=1686327712", + "album": "System Of A Down" }, - "271867710382714948": { - "artist": "Inteus", - "downloaded": false, + "157682831582935241": { + "artist": "System Of A Down", + "downloaded": true, "downloading": false, - "link": "/playlist?list=OLAK5uy_kEzRY57fWs2iqQv7mg9jA8XEwdctH3vdk&playnext=1&index=1", - "cover": "https://i9.ytimg.com/s_p/OLAK5uy_kEzRY57fWs2iqQv7mg9jA8XEwdctH3vdk/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCLKkkaQG&rs=AOn4CLCsG_4SxEMgAp96U-dAiavg1BZggQ&v=1686393394", - "album": "Chapter III" - }, - "322585091946466973": { - "artist": "Inteus", - "downloaded": false, - "downloading": false, - "link": "/playlist?list=OLAK5uy_nWWfDckTPSOY4nN_5bXMz1y83Qrc160sM&playnext=1&index=1", - "cover": "https://i9.ytimg.com/s_p/OLAK5uy_nWWfDckTPSOY4nN_5bXMz1y83Qrc160sM/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCILUmaQG&rs=AOn4CLBW1i3Fi_0G31s8k81YFjUNM6ui_w&v=1686530562", - "album": "Chapter II" - }, - "219693029017574393": { - "artist": "Inteus", - "downloaded": false, - "downloading": false, - "link": "/playlist?list=OLAK5uy_kwpt9qeeS577nxNUo8kh2S4R_3gFFf5Ys&playnext=1&index=1", - "cover": "https://i9.ytimg.com/s_p/OLAK5uy_kwpt9qeeS577nxNUo8kh2S4R_3gFFf5Ys/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCJeClqQG&rs=AOn4CLD82PHRl_xq8BKF2EtuLOC2d-WStg&v=1686470935", - "album": "Chapter 1" - }, - "106299219586508927": { - "artist": "Inteus", - "downloaded": false, - "downloading": false, - "link": "/playlist?list=OLAK5uy_kA-xQTNx8Az1t4HAFQ9vB1oTWG4A5cAIQ&playnext=1&index=1", - "cover": "https://i9.ytimg.com/s_p/OLAK5uy_kA-xQTNx8Az1t4HAFQ9vB1oTWG4A5cAIQ/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCNaakaQG&rs=AOn4CLDN92pGYYBFP0Vwi5c28qE7L43K_w&v=1686392150", - "album": "Selection: 2" + "link": "/playlist?list=OLAK5uy_nFkNOoOmhDf_bWKkazOrMDEqqh_Z7bVlU&playnext=1&index=1", + "cover": "https://i9.ytimg.com/s_p/OLAK5uy_nFkNOoOmhDf_bWKkazOrMDEqqh_Z7bVlU/mqdefault.jpg?sqp=CMzWpaQGir7X7AMGCPO9gKQG&rs=AOn4CLC7VbkYuPc5LGoGOU1G6SNc9lcoWA&v=1686118131", + "album": "B.Y.O.B." } } } \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 671499c..2f915d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +apscheduler==3.10.1 async-timeout==4.0.2 beautifulsoup4==4.12.2 Brotli==1.0.9 @@ -20,6 +21,7 @@ requests==2.31.0 requests-file==1.5.1 selenium==3.14.1 selenium-requests==1.3 +schedule==1.2.0 six==1.16.0 soupsieve==2.4.1 tldextract==3.4.4 diff --git a/static/img/vinyl-card.png b/static/img/vinyl-card.png index 6a7c06b..d5184b5 100644 Binary files a/static/img/vinyl-card.png and b/static/img/vinyl-card.png differ diff --git a/static/js/app.js b/static/js/app.js index 2cbba93..a3e169b 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1,5 +1,6 @@ const appModal = $('#modalDownloadQueue'); const appModalContent = $('#modal_content'); +let modalPolling = false; function proc_notification(icon, title, text) { Swal.fire({ @@ -9,15 +10,22 @@ function proc_notification(icon, title, text) { }) } -$('.queue_btn').on('click', () => { - console.log('Get Queue!'); +function fill_download_queue() { $.ajax({ url: '/api/v1/get/queue' - }).done( (res) => { - console.log(res); + }).done((res) => { appModalContent.html(res); - appModal.modal('toggle'); }) +} + +$('.queue_btn').on('click', () => { + console.log('Get Queue!'); + if (modalPolling) { + clearInterval(modalPolling); + } + fill_download_queue(); + modalPolling = setInterval(fill_download_queue, 4000); + appModal.modal('toggle'); }) $('#download_btn').on('click', () => { diff --git a/utils/__pycache__/download.cpython-37.pyc b/utils/__pycache__/download.cpython-37.pyc index 9b2eb6b..fae3ca0 100644 Binary files a/utils/__pycache__/download.cpython-37.pyc and b/utils/__pycache__/download.cpython-37.pyc differ diff --git a/utils/download.py b/utils/download.py index 99b9f8b..35d2f67 100644 --- a/utils/download.py +++ b/utils/download.py @@ -1,56 +1,62 @@ import os +from database import Model from const import * from .yt_dlp_logger import * import wget import yt_dlp +Album = Model('album') -def download_process_list(artist, processed_albums_data_list): + +def download_album(album): """ - Take a list of dictionaries that have the values needed to create a file-structure and save the downloaded files - :param artist: - :param processed_albums_data_list: + Take a list of albums and process the downloads + :param album: Dict of album data :return: """ + print('------') + print('Album Data') + print(album) + artist = album.get('artist') artist_path = MEDIA_FOLDER + '/%s' % artist if not os.path.exists(artist_path): os.mkdir(artist_path) - for item in processed_albums_data_list: - print('---') - # Create album folder - album = item.get('album') - album_path = artist_path + '/%s' % album - if not os.path.exists(album_path): - os.mkdir(album_path) - - # Save album cover - if item.get('cover'): - try: - download_file(item.get('cover'), album_path) - except Exception as e: - print("Warning: %s" % e) - - # Download album - print(item) - ydl_opts = { - 'logger': YtDlpLogger(), - 'progress_hooks': [yt_dlp_log_hook], - 'format': 'mp3/bestaudio/best', - 'outtmpl': album_path + '/%(title)s.%(ext)s', - # ℹ️ See help(yt_dlp.postprocessor) for a list of available Postprocessors and their arguments - 'postprocessors': [{ # Extract audio using ffmpeg - 'key': 'FFmpegExtractAudio', - 'preferredcodec': 'mp3', - }] - } - - with yt_dlp.YoutubeDL(ydl_opts) as ydl: - try: - error_code = ydl.download('https://youtube.com' + item.get('link')) - except Exception as e: - print('!!!!!!!!!') - print(e) + print('---') + # Create album folder + album_title = album.get('album') + album_path = artist_path + '/%s' % album_title + if not os.path.exists(album_path): + os.mkdir(album_path) + + # Save album cover + if album.get('cover'): + try: + download_file(album.get('cover'), album_path) + except Exception as e: + print("Warning: %s" % e) + + # Download album + ydl_opts = { + 'logger': YtDlpLogger(), + 'progress_hooks': [yt_dlp_log_hook], + 'format': 'mp3/bestaudio/best', + 'outtmpl': album_path + '/%(title)s.%(ext)s', + # ℹ️ See help(yt_dlp.postprocessor) for a list of available Postprocessors and their arguments + 'postprocessors': [{ # Extract audio using ffmpeg + 'key': 'FFmpegExtractAudio', + 'preferredcodec': 'mp3', + }] + } + + with yt_dlp.YoutubeDL(ydl_opts) as ydl: + try: + error_code = ydl.download('https://youtube.com' + album.get('link')) + except Exception as e: + print('!!!!!!!!!') + print(e) + + Album.write(album['id'], {'downloaded': True, 'downloading': False}) def download_file(url, output):