parent
303881038b
commit
e02285924a
@ -1 +1 @@
|
|||||||
from . import const
|
|
||||||
|
|||||||
Binary file not shown.
@ -0,0 +1,84 @@
|
|||||||
|
import operator as oprtr
|
||||||
|
from pysondb import PysonDB
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_condition(record_field, operator, condition):
|
||||||
|
return operator(record_field, condition)
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_operator(op):
|
||||||
|
if op == '>':
|
||||||
|
op = oprtr.gt
|
||||||
|
elif op == '<':
|
||||||
|
op = oprtr.lt
|
||||||
|
elif op == '=':
|
||||||
|
op = oprtr.eq
|
||||||
|
elif op == '!=':
|
||||||
|
op = oprtr.ne
|
||||||
|
else:
|
||||||
|
raise UserWarning('Invalid Operator: %s' % op)
|
||||||
|
return op
|
||||||
|
|
||||||
|
|
||||||
|
class Model:
|
||||||
|
# TODO: Modify some of this to be wrapped into an ENV wrapper that gets loaded in when the server starts and creates
|
||||||
|
# class objects that can be manipulated easier by things like update_by_id
|
||||||
|
|
||||||
|
def __init__(self, name):
|
||||||
|
self.env = PysonDB('/home/stonesoft/Apps/getDiscography/database/%s.json' % name)
|
||||||
|
|
||||||
|
def _search(self, records, params):
|
||||||
|
"""
|
||||||
|
Iterate through list of condition tuples and append results to a checklist that will evaluate at the end
|
||||||
|
ex params: [('name', '=', 'John'), ('zip', '!=', '12345')]
|
||||||
|
:param params: List of tuples
|
||||||
|
:return: Record to search recordset if True
|
||||||
|
"""
|
||||||
|
filtered_record_ids =[]
|
||||||
|
for record in records:
|
||||||
|
record_id = self.env.get_by_id(record)
|
||||||
|
print('===')
|
||||||
|
print(record_id)
|
||||||
|
checklist = []
|
||||||
|
for param in params:
|
||||||
|
field = param[0]
|
||||||
|
operator = evaluate_operator(param[1])
|
||||||
|
condition = param[2]
|
||||||
|
checklist.append(evaluate_condition(record_id[field], operator, condition))
|
||||||
|
|
||||||
|
passed = all(x for x in checklist)
|
||||||
|
if passed:
|
||||||
|
filtered_record_ids.append(record_id)
|
||||||
|
|
||||||
|
return filtered_record_ids
|
||||||
|
|
||||||
|
def search(self, params):
|
||||||
|
"""
|
||||||
|
:param params: List of tuples that will be evaluated and return a total list of records
|
||||||
|
:return: None, List or Single record
|
||||||
|
"""
|
||||||
|
records = self.env.get_all()
|
||||||
|
record_ids = self._search(records, params)
|
||||||
|
if not record_ids:
|
||||||
|
record_ids = None
|
||||||
|
|
||||||
|
return record_ids
|
||||||
|
|
||||||
|
def read(self, record_id):
|
||||||
|
data = self.env.get_by_id(record_id)
|
||||||
|
return data
|
||||||
|
|
||||||
|
def create(self, vals):
|
||||||
|
record = self.env.add(vals)
|
||||||
|
return record
|
||||||
|
|
||||||
|
def create_many(self, record_list):
|
||||||
|
record_ids = self.env.add_many(record_list)
|
||||||
|
return record_ids
|
||||||
|
|
||||||
|
def write(self, record_id, vals):
|
||||||
|
self.env.update_by_id(record_id, vals)
|
||||||
|
|
||||||
|
def unlink(self, record_id):
|
||||||
|
self.env.delete_by_id(record_id)
|
||||||
|
return True
|
||||||
@ -0,0 +1,93 @@
|
|||||||
|
{
|
||||||
|
"version": 2,
|
||||||
|
"keys": [
|
||||||
|
"album",
|
||||||
|
"artist",
|
||||||
|
"cover",
|
||||||
|
"downloaded",
|
||||||
|
"downloading",
|
||||||
|
"link"
|
||||||
|
],
|
||||||
|
"data": {
|
||||||
|
"199074445401889600": {
|
||||||
|
"artist": "Inteus",
|
||||||
|
"downloaded": false,
|
||||||
|
"downloading": false,
|
||||||
|
"link": "/playlist?list=OLAK5uy_nr-vlAyokNEb25RLBNe1XHsFo9gkvu2Pg&playnext=1&index=1",
|
||||||
|
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_nr-vlAyokNEb25RLBNe1XHsFo9gkvu2Pg/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCN-N_qMG&rs=AOn4CLBo8tjoBHrUytlN1kfqTHYVMaVA4Q&v=1686079199",
|
||||||
|
"album": "Quick Revive"
|
||||||
|
},
|
||||||
|
"187732915376923467": {
|
||||||
|
"artist": "Inteus",
|
||||||
|
"downloaded": false,
|
||||||
|
"downloading": false,
|
||||||
|
"link": "/playlist?list=OLAK5uy_lS0tCQXZ-1ppWJOMhaMel8GKGtnUzsvnU&playnext=1&index=1",
|
||||||
|
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_lS0tCQXZ-1ppWJOMhaMel8GKGtnUzsvnU/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCPDOhaQG&rs=AOn4CLBa8Mmhg8B0A7ljEpCsig_bzTju0A&v=1686202224",
|
||||||
|
"album": "Winds of Paradise"
|
||||||
|
},
|
||||||
|
"206562046422345861": {
|
||||||
|
"artist": "Inteus",
|
||||||
|
"downloaded": false,
|
||||||
|
"downloading": false,
|
||||||
|
"link": "/playlist?list=OLAK5uy_movr4fRkYisvKvX92G_D6fWfV1umrn3hs&playnext=1&index=1",
|
||||||
|
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_movr4fRkYisvKvX92G_D6fWfV1umrn3hs/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCIKr_6MG&rs=AOn4CLDiszLRoh4cueTcK9h20CQ3KNpeXQ&v=1686099330",
|
||||||
|
"album": "Selection:3"
|
||||||
|
},
|
||||||
|
"321688139888393359": {
|
||||||
|
"artist": "Inteus",
|
||||||
|
"downloaded": false,
|
||||||
|
"downloading": false,
|
||||||
|
"link": "/playlist?list=OLAK5uy_kW87DzVan7NyJ7j06XBmOHlg1WaFm38LA&playnext=1&index=1",
|
||||||
|
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_kW87DzVan7NyJ7j06XBmOHlg1WaFm38LA/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCIDi_aMG&rs=AOn4CLA4dE1oa2TYSqb0pvfNTdK7ZSgERw&v=1686073600",
|
||||||
|
"album": "Bruh Moment (2020 Scraps)"
|
||||||
|
},
|
||||||
|
"633598784099856320": {
|
||||||
|
"artist": "Inteus",
|
||||||
|
"downloaded": false,
|
||||||
|
"downloading": false,
|
||||||
|
"link": "/playlist?list=OLAK5uy_n1DHIq4LZlVVBvyilCzrjxAyCmsBSRfY0&playnext=1&index=1",
|
||||||
|
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_n1DHIq4LZlVVBvyilCzrjxAyCmsBSRfY0/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCJGzn6QG&rs=AOn4CLD_qLcb-Z1EabBDe3449f3d0mh3Cg&v=1686624657",
|
||||||
|
"album": "Yent Szn 3"
|
||||||
|
},
|
||||||
|
"131921253377227610": {
|
||||||
|
"artist": "Inteus",
|
||||||
|
"downloaded": false,
|
||||||
|
"downloading": false,
|
||||||
|
"link": "/playlist?list=OLAK5uy_nm2zfpc5MDyzOp2ftFqdlHBQ-L3tE2BQM&playnext=1&index=1",
|
||||||
|
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_nm2zfpc5MDyzOp2ftFqdlHBQ-L3tE2BQM/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCIPhgaQG&rs=AOn4CLBM_1qrbE_tAOEp51oGRYQMeTkRsw&v=1686139011",
|
||||||
|
"album": "Voyager"
|
||||||
|
},
|
||||||
|
"271867710382714948": {
|
||||||
|
"artist": "Inteus",
|
||||||
|
"downloaded": false,
|
||||||
|
"downloading": false,
|
||||||
|
"link": "/playlist?list=OLAK5uy_kEzRY57fWs2iqQv7mg9jA8XEwdctH3vdk&playnext=1&index=1",
|
||||||
|
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_kEzRY57fWs2iqQv7mg9jA8XEwdctH3vdk/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCLKkkaQG&rs=AOn4CLCsG_4SxEMgAp96U-dAiavg1BZggQ&v=1686393394",
|
||||||
|
"album": "Chapter III"
|
||||||
|
},
|
||||||
|
"322585091946466973": {
|
||||||
|
"artist": "Inteus",
|
||||||
|
"downloaded": false,
|
||||||
|
"downloading": false,
|
||||||
|
"link": "/playlist?list=OLAK5uy_nWWfDckTPSOY4nN_5bXMz1y83Qrc160sM&playnext=1&index=1",
|
||||||
|
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_nWWfDckTPSOY4nN_5bXMz1y83Qrc160sM/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCILUmaQG&rs=AOn4CLBW1i3Fi_0G31s8k81YFjUNM6ui_w&v=1686530562",
|
||||||
|
"album": "Chapter II"
|
||||||
|
},
|
||||||
|
"219693029017574393": {
|
||||||
|
"artist": "Inteus",
|
||||||
|
"downloaded": false,
|
||||||
|
"downloading": false,
|
||||||
|
"link": "/playlist?list=OLAK5uy_kwpt9qeeS577nxNUo8kh2S4R_3gFFf5Ys&playnext=1&index=1",
|
||||||
|
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_kwpt9qeeS577nxNUo8kh2S4R_3gFFf5Ys/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCJeClqQG&rs=AOn4CLD82PHRl_xq8BKF2EtuLOC2d-WStg&v=1686470935",
|
||||||
|
"album": "Chapter 1"
|
||||||
|
},
|
||||||
|
"106299219586508927": {
|
||||||
|
"artist": "Inteus",
|
||||||
|
"downloaded": false,
|
||||||
|
"downloading": false,
|
||||||
|
"link": "/playlist?list=OLAK5uy_kA-xQTNx8Az1t4HAFQ9vB1oTWG4A5cAIQ&playnext=1&index=1",
|
||||||
|
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_kA-xQTNx8Az1t4HAFQ9vB1oTWG4A5cAIQ/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCNaakaQG&rs=AOn4CLDN92pGYYBFP0Vwi5c28qE7L43K_w&v=1686392150",
|
||||||
|
"album": "Selection: 2"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,20 +1,48 @@
|
|||||||
|
const appModal = $('#modalDownloadQueue');
|
||||||
|
const appModalContent = $('#modal_content');
|
||||||
|
|
||||||
|
function proc_notification(icon, title, text) {
|
||||||
|
Swal.fire({
|
||||||
|
title: title,
|
||||||
|
icon: icon,
|
||||||
|
text: text
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
$('.queue_btn').on('click', () => {
|
||||||
|
console.log('Get Queue!');
|
||||||
|
$.ajax({
|
||||||
|
url: '/api/v1/get/queue'
|
||||||
|
}).done( (res) => {
|
||||||
|
console.log(res);
|
||||||
|
appModalContent.html(res);
|
||||||
|
appModal.modal('toggle');
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
$('#download_btn').on('click', () => {
|
$('#download_btn').on('click', () => {
|
||||||
let artist = $('#search_bar').val();
|
let artist = $('#search_bar').val();
|
||||||
// Prevent
|
// Prevent
|
||||||
$('#search_bar').val('');
|
$('#search_bar').val('');
|
||||||
|
let icon = 'error';
|
||||||
|
let title = 'What the flip?!';
|
||||||
|
let text = 'You need to add an artist bro..';
|
||||||
|
|
||||||
if (artist) {
|
if (artist) {
|
||||||
$("#loader-wrapper").fadeIn(300);
|
$("#loader-wrapper").fadeIn(300);
|
||||||
$.ajax({
|
$.ajax({
|
||||||
url: `/api/v1/get/${artist}`,
|
url: `/api/v1/get/artist/${artist}`,
|
||||||
}).done(function (res) {
|
}).done(function (res) {
|
||||||
console.log('---');
|
text = res.message;
|
||||||
console.log(res);
|
if (res.status === 200) {
|
||||||
console.log('---');
|
icon = 'success';
|
||||||
|
title = 'Shazam!';
|
||||||
|
}
|
||||||
$("#loader-wrapper").fadeOut(700);
|
$("#loader-wrapper").fadeOut(700);
|
||||||
|
proc_notification(icon, title, text);
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
console.log('No artist');
|
proc_notification(icon, title, text);
|
||||||
}
|
}
|
||||||
|
|
||||||
})
|
})
|
||||||
@ -0,0 +1,41 @@
|
|||||||
|
<div class="row">
|
||||||
|
{% for album_id in album_ids %}
|
||||||
|
<div class="col-12">
|
||||||
|
<div class="queue_card">
|
||||||
|
<div class="card mb-3 p-0">
|
||||||
|
<div class="row g-0">
|
||||||
|
<div class="col-md-3">
|
||||||
|
|
||||||
|
<div class="dl_queue_img">
|
||||||
|
{% if not album_id.downloading %}
|
||||||
|
<div class="icn-downloading">
|
||||||
|
<i class="la la-6x la-compact-disc icn-spinner text-white"></i>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
<img src="{{ album_id.cover }}" class="img-fluid rounded-start"
|
||||||
|
alt="{{ album_id.album }}" style="width: 100%; height: 100%; min-height: 180px;">
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
<div class="col-md-9">
|
||||||
|
<div class="card-body">
|
||||||
|
<h5 class="card-title">{{ album_id.album }}</h5>
|
||||||
|
<p class="card-text">{{ album_id.artist }}</p>
|
||||||
|
{% if album_id.downloading %}
|
||||||
|
<p>Downloading...</p>
|
||||||
|
{% else %}
|
||||||
|
<p>Waiting to Download</p>
|
||||||
|
{% endif %}
|
||||||
|
<p class="card-text"><small class="text-muted"><a
|
||||||
|
href="https://youtube.com{{ album_id.link }}">Album Page</a></small></p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% else %}
|
||||||
|
<p>Nothing waiting to download</p>
|
||||||
|
{% endfor %}
|
||||||
|
|
||||||
|
</div>
|
||||||
@ -1,2 +1,5 @@
|
|||||||
from . import browser
|
from . import browser
|
||||||
|
from . import download
|
||||||
from . import processor
|
from . import processor
|
||||||
|
from . import scraper
|
||||||
|
from . import yt_dlp_logger
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,80 @@
|
|||||||
|
import bs4
|
||||||
|
import time
|
||||||
|
|
||||||
|
from selenium.webdriver.support.ui import WebDriverWait
|
||||||
|
from selenium.webdriver.support import expected_conditions as EC
|
||||||
|
from selenium.webdriver.common.by import By
|
||||||
|
from .browser import new_browser
|
||||||
|
from const import *
|
||||||
|
|
||||||
|
|
||||||
|
def process_scraped_data(artist, res_value):
|
||||||
|
# Process the gathered HTML data for list of data
|
||||||
|
html = bs4.BeautifulSoup(res_value, features="html.parser")
|
||||||
|
albums = html.find_all('a')
|
||||||
|
albums_data_list = []
|
||||||
|
for album in albums:
|
||||||
|
album_data = {
|
||||||
|
'artist': artist.title(),
|
||||||
|
'downloaded': False,
|
||||||
|
'downloading': False,
|
||||||
|
}
|
||||||
|
if album.has_key('href'):
|
||||||
|
album_data.update({'link': album['href']})
|
||||||
|
|
||||||
|
album_image = album.find('img')
|
||||||
|
if album_image and album_image.has_key('src'):
|
||||||
|
album_data.update({'cover': album_image['src']})
|
||||||
|
|
||||||
|
album_title = album.find('div', {'id': 'card-title'})
|
||||||
|
if album_title and hasattr(album_title, 'text'):
|
||||||
|
album_data.update({'album': album_title.text.replace('\n', '').replace('/', '-')})
|
||||||
|
|
||||||
|
albums_data_list.append(album_data)
|
||||||
|
|
||||||
|
return albums_data_list
|
||||||
|
|
||||||
|
|
||||||
|
def scrape(artist):
|
||||||
|
browser = new_browser(headless=True)
|
||||||
|
url = QUERY_URL + artist
|
||||||
|
browser.maximize_window()
|
||||||
|
browser.implicitly_wait(1)
|
||||||
|
response = browser.get(url)
|
||||||
|
last_height = browser.execute_script("return document.body.scrollHeight")
|
||||||
|
browser.execute_script("window.scrollTo(0, 500);")
|
||||||
|
time.sleep(1)
|
||||||
|
scrape_data = ''
|
||||||
|
|
||||||
|
try:
|
||||||
|
# get the financial value when it's populated to the page
|
||||||
|
value_element = WebDriverWait(browser, 5).until(
|
||||||
|
EC.presence_of_element_located(locator=(By.XPATH, '//div[@id="shelf-container"]'))
|
||||||
|
)
|
||||||
|
element = browser.find_element(By.XPATH, ALBUM_CONTAINER_ITEMS_XPATH)
|
||||||
|
if element:
|
||||||
|
time.sleep(1)
|
||||||
|
scrape_data += element.get_attribute('outerHTML')
|
||||||
|
btn_right = browser.find_element(By.XPATH, BTN_RIGHT_FULL_XPATH)
|
||||||
|
btn_right_displayed = True
|
||||||
|
safety_index = 0
|
||||||
|
while btn_right_displayed:
|
||||||
|
# actions = ActionChains(browser)
|
||||||
|
# actions.move_to_element(btn_right).perform()
|
||||||
|
safety_index += 1
|
||||||
|
time.sleep(1)
|
||||||
|
browser.execute_script(click_script)
|
||||||
|
time.sleep(1)
|
||||||
|
element = browser.find_element(By.XPATH, ALBUM_CONTAINER_ITEMS_XPATH)
|
||||||
|
scrape_data += element.get_attribute('outerHTML')
|
||||||
|
time.sleep(1)
|
||||||
|
btn_right_displayed = btn_right.is_displayed()
|
||||||
|
if safety_index > 5:
|
||||||
|
btn_right_displayed = False
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# after 5 seconds, give up
|
||||||
|
browser.quit()
|
||||||
|
|
||||||
|
return scrape_data
|
||||||
Loading…
Reference in new issue