parent
303881038b
commit
e02285924a
@ -1 +1 @@
|
||||
from . import const
|
||||
|
||||
|
||||
Binary file not shown.
@ -0,0 +1,84 @@
|
||||
import operator as oprtr
|
||||
from pysondb import PysonDB
|
||||
|
||||
|
||||
def evaluate_condition(record_field, operator, condition):
|
||||
return operator(record_field, condition)
|
||||
|
||||
|
||||
def evaluate_operator(op):
|
||||
if op == '>':
|
||||
op = oprtr.gt
|
||||
elif op == '<':
|
||||
op = oprtr.lt
|
||||
elif op == '=':
|
||||
op = oprtr.eq
|
||||
elif op == '!=':
|
||||
op = oprtr.ne
|
||||
else:
|
||||
raise UserWarning('Invalid Operator: %s' % op)
|
||||
return op
|
||||
|
||||
|
||||
class Model:
|
||||
# TODO: Modify some of this to be wrapped into an ENV wrapper that gets loaded in when the server starts and creates
|
||||
# class objects that can be manipulated easier by things like update_by_id
|
||||
|
||||
def __init__(self, name):
|
||||
self.env = PysonDB('/home/stonesoft/Apps/getDiscography/database/%s.json' % name)
|
||||
|
||||
def _search(self, records, params):
|
||||
"""
|
||||
Iterate through list of condition tuples and append results to a checklist that will evaluate at the end
|
||||
ex params: [('name', '=', 'John'), ('zip', '!=', '12345')]
|
||||
:param params: List of tuples
|
||||
:return: Record to search recordset if True
|
||||
"""
|
||||
filtered_record_ids =[]
|
||||
for record in records:
|
||||
record_id = self.env.get_by_id(record)
|
||||
print('===')
|
||||
print(record_id)
|
||||
checklist = []
|
||||
for param in params:
|
||||
field = param[0]
|
||||
operator = evaluate_operator(param[1])
|
||||
condition = param[2]
|
||||
checklist.append(evaluate_condition(record_id[field], operator, condition))
|
||||
|
||||
passed = all(x for x in checklist)
|
||||
if passed:
|
||||
filtered_record_ids.append(record_id)
|
||||
|
||||
return filtered_record_ids
|
||||
|
||||
def search(self, params):
|
||||
"""
|
||||
:param params: List of tuples that will be evaluated and return a total list of records
|
||||
:return: None, List or Single record
|
||||
"""
|
||||
records = self.env.get_all()
|
||||
record_ids = self._search(records, params)
|
||||
if not record_ids:
|
||||
record_ids = None
|
||||
|
||||
return record_ids
|
||||
|
||||
def read(self, record_id):
|
||||
data = self.env.get_by_id(record_id)
|
||||
return data
|
||||
|
||||
def create(self, vals):
|
||||
record = self.env.add(vals)
|
||||
return record
|
||||
|
||||
def create_many(self, record_list):
|
||||
record_ids = self.env.add_many(record_list)
|
||||
return record_ids
|
||||
|
||||
def write(self, record_id, vals):
|
||||
self.env.update_by_id(record_id, vals)
|
||||
|
||||
def unlink(self, record_id):
|
||||
self.env.delete_by_id(record_id)
|
||||
return True
|
||||
@ -0,0 +1,93 @@
|
||||
{
|
||||
"version": 2,
|
||||
"keys": [
|
||||
"album",
|
||||
"artist",
|
||||
"cover",
|
||||
"downloaded",
|
||||
"downloading",
|
||||
"link"
|
||||
],
|
||||
"data": {
|
||||
"199074445401889600": {
|
||||
"artist": "Inteus",
|
||||
"downloaded": false,
|
||||
"downloading": false,
|
||||
"link": "/playlist?list=OLAK5uy_nr-vlAyokNEb25RLBNe1XHsFo9gkvu2Pg&playnext=1&index=1",
|
||||
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_nr-vlAyokNEb25RLBNe1XHsFo9gkvu2Pg/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCN-N_qMG&rs=AOn4CLBo8tjoBHrUytlN1kfqTHYVMaVA4Q&v=1686079199",
|
||||
"album": "Quick Revive"
|
||||
},
|
||||
"187732915376923467": {
|
||||
"artist": "Inteus",
|
||||
"downloaded": false,
|
||||
"downloading": false,
|
||||
"link": "/playlist?list=OLAK5uy_lS0tCQXZ-1ppWJOMhaMel8GKGtnUzsvnU&playnext=1&index=1",
|
||||
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_lS0tCQXZ-1ppWJOMhaMel8GKGtnUzsvnU/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCPDOhaQG&rs=AOn4CLBa8Mmhg8B0A7ljEpCsig_bzTju0A&v=1686202224",
|
||||
"album": "Winds of Paradise"
|
||||
},
|
||||
"206562046422345861": {
|
||||
"artist": "Inteus",
|
||||
"downloaded": false,
|
||||
"downloading": false,
|
||||
"link": "/playlist?list=OLAK5uy_movr4fRkYisvKvX92G_D6fWfV1umrn3hs&playnext=1&index=1",
|
||||
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_movr4fRkYisvKvX92G_D6fWfV1umrn3hs/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCIKr_6MG&rs=AOn4CLDiszLRoh4cueTcK9h20CQ3KNpeXQ&v=1686099330",
|
||||
"album": "Selection:3"
|
||||
},
|
||||
"321688139888393359": {
|
||||
"artist": "Inteus",
|
||||
"downloaded": false,
|
||||
"downloading": false,
|
||||
"link": "/playlist?list=OLAK5uy_kW87DzVan7NyJ7j06XBmOHlg1WaFm38LA&playnext=1&index=1",
|
||||
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_kW87DzVan7NyJ7j06XBmOHlg1WaFm38LA/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCIDi_aMG&rs=AOn4CLA4dE1oa2TYSqb0pvfNTdK7ZSgERw&v=1686073600",
|
||||
"album": "Bruh Moment (2020 Scraps)"
|
||||
},
|
||||
"633598784099856320": {
|
||||
"artist": "Inteus",
|
||||
"downloaded": false,
|
||||
"downloading": false,
|
||||
"link": "/playlist?list=OLAK5uy_n1DHIq4LZlVVBvyilCzrjxAyCmsBSRfY0&playnext=1&index=1",
|
||||
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_n1DHIq4LZlVVBvyilCzrjxAyCmsBSRfY0/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCJGzn6QG&rs=AOn4CLD_qLcb-Z1EabBDe3449f3d0mh3Cg&v=1686624657",
|
||||
"album": "Yent Szn 3"
|
||||
},
|
||||
"131921253377227610": {
|
||||
"artist": "Inteus",
|
||||
"downloaded": false,
|
||||
"downloading": false,
|
||||
"link": "/playlist?list=OLAK5uy_nm2zfpc5MDyzOp2ftFqdlHBQ-L3tE2BQM&playnext=1&index=1",
|
||||
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_nm2zfpc5MDyzOp2ftFqdlHBQ-L3tE2BQM/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCIPhgaQG&rs=AOn4CLBM_1qrbE_tAOEp51oGRYQMeTkRsw&v=1686139011",
|
||||
"album": "Voyager"
|
||||
},
|
||||
"271867710382714948": {
|
||||
"artist": "Inteus",
|
||||
"downloaded": false,
|
||||
"downloading": false,
|
||||
"link": "/playlist?list=OLAK5uy_kEzRY57fWs2iqQv7mg9jA8XEwdctH3vdk&playnext=1&index=1",
|
||||
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_kEzRY57fWs2iqQv7mg9jA8XEwdctH3vdk/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCLKkkaQG&rs=AOn4CLCsG_4SxEMgAp96U-dAiavg1BZggQ&v=1686393394",
|
||||
"album": "Chapter III"
|
||||
},
|
||||
"322585091946466973": {
|
||||
"artist": "Inteus",
|
||||
"downloaded": false,
|
||||
"downloading": false,
|
||||
"link": "/playlist?list=OLAK5uy_nWWfDckTPSOY4nN_5bXMz1y83Qrc160sM&playnext=1&index=1",
|
||||
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_nWWfDckTPSOY4nN_5bXMz1y83Qrc160sM/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCILUmaQG&rs=AOn4CLBW1i3Fi_0G31s8k81YFjUNM6ui_w&v=1686530562",
|
||||
"album": "Chapter II"
|
||||
},
|
||||
"219693029017574393": {
|
||||
"artist": "Inteus",
|
||||
"downloaded": false,
|
||||
"downloading": false,
|
||||
"link": "/playlist?list=OLAK5uy_kwpt9qeeS577nxNUo8kh2S4R_3gFFf5Ys&playnext=1&index=1",
|
||||
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_kwpt9qeeS577nxNUo8kh2S4R_3gFFf5Ys/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCJeClqQG&rs=AOn4CLD82PHRl_xq8BKF2EtuLOC2d-WStg&v=1686470935",
|
||||
"album": "Chapter 1"
|
||||
},
|
||||
"106299219586508927": {
|
||||
"artist": "Inteus",
|
||||
"downloaded": false,
|
||||
"downloading": false,
|
||||
"link": "/playlist?list=OLAK5uy_kA-xQTNx8Az1t4HAFQ9vB1oTWG4A5cAIQ&playnext=1&index=1",
|
||||
"cover": "https://i9.ytimg.com/s_p/OLAK5uy_kA-xQTNx8Az1t4HAFQ9vB1oTWG4A5cAIQ/mqdefault.jpg?sqp=CJT0pKQGir7X7AMGCNaakaQG&rs=AOn4CLDN92pGYYBFP0Vwi5c28qE7L43K_w&v=1686392150",
|
||||
"album": "Selection: 2"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,20 +1,48 @@
|
||||
const appModal = $('#modalDownloadQueue');
|
||||
const appModalContent = $('#modal_content');
|
||||
|
||||
function proc_notification(icon, title, text) {
|
||||
Swal.fire({
|
||||
title: title,
|
||||
icon: icon,
|
||||
text: text
|
||||
})
|
||||
}
|
||||
|
||||
$('.queue_btn').on('click', () => {
|
||||
console.log('Get Queue!');
|
||||
$.ajax({
|
||||
url: '/api/v1/get/queue'
|
||||
}).done( (res) => {
|
||||
console.log(res);
|
||||
appModalContent.html(res);
|
||||
appModal.modal('toggle');
|
||||
})
|
||||
})
|
||||
|
||||
$('#download_btn').on('click', () => {
|
||||
let artist = $('#search_bar').val();
|
||||
// Prevent
|
||||
$('#search_bar').val('');
|
||||
let icon = 'error';
|
||||
let title = 'What the flip?!';
|
||||
let text = 'You need to add an artist bro..';
|
||||
|
||||
if (artist) {
|
||||
$("#loader-wrapper").fadeIn(300);
|
||||
$.ajax({
|
||||
url: `/api/v1/get/${artist}`,
|
||||
url: `/api/v1/get/artist/${artist}`,
|
||||
}).done(function (res) {
|
||||
console.log('---');
|
||||
console.log(res);
|
||||
console.log('---');
|
||||
text = res.message;
|
||||
if (res.status === 200) {
|
||||
icon = 'success';
|
||||
title = 'Shazam!';
|
||||
}
|
||||
$("#loader-wrapper").fadeOut(700);
|
||||
proc_notification(icon, title, text);
|
||||
});
|
||||
} else {
|
||||
console.log('No artist');
|
||||
proc_notification(icon, title, text);
|
||||
}
|
||||
|
||||
})
|
||||
@ -0,0 +1,41 @@
|
||||
<div class="row">
|
||||
{% for album_id in album_ids %}
|
||||
<div class="col-12">
|
||||
<div class="queue_card">
|
||||
<div class="card mb-3 p-0">
|
||||
<div class="row g-0">
|
||||
<div class="col-md-3">
|
||||
|
||||
<div class="dl_queue_img">
|
||||
{% if not album_id.downloading %}
|
||||
<div class="icn-downloading">
|
||||
<i class="la la-6x la-compact-disc icn-spinner text-white"></i>
|
||||
</div>
|
||||
{% endif %}
|
||||
<img src="{{ album_id.cover }}" class="img-fluid rounded-start"
|
||||
alt="{{ album_id.album }}" style="width: 100%; height: 100%; min-height: 180px;">
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="col-md-9">
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">{{ album_id.album }}</h5>
|
||||
<p class="card-text">{{ album_id.artist }}</p>
|
||||
{% if album_id.downloading %}
|
||||
<p>Downloading...</p>
|
||||
{% else %}
|
||||
<p>Waiting to Download</p>
|
||||
{% endif %}
|
||||
<p class="card-text"><small class="text-muted"><a
|
||||
href="https://youtube.com{{ album_id.link }}">Album Page</a></small></p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% else %}
|
||||
<p>Nothing waiting to download</p>
|
||||
{% endfor %}
|
||||
|
||||
</div>
|
||||
@ -1,2 +1,5 @@
|
||||
from . import browser
|
||||
from . import download
|
||||
from . import processor
|
||||
from . import scraper
|
||||
from . import yt_dlp_logger
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,80 @@
|
||||
import bs4
|
||||
import time
|
||||
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.webdriver.common.by import By
|
||||
from .browser import new_browser
|
||||
from const import *
|
||||
|
||||
|
||||
def process_scraped_data(artist, res_value):
|
||||
# Process the gathered HTML data for list of data
|
||||
html = bs4.BeautifulSoup(res_value, features="html.parser")
|
||||
albums = html.find_all('a')
|
||||
albums_data_list = []
|
||||
for album in albums:
|
||||
album_data = {
|
||||
'artist': artist.title(),
|
||||
'downloaded': False,
|
||||
'downloading': False,
|
||||
}
|
||||
if album.has_key('href'):
|
||||
album_data.update({'link': album['href']})
|
||||
|
||||
album_image = album.find('img')
|
||||
if album_image and album_image.has_key('src'):
|
||||
album_data.update({'cover': album_image['src']})
|
||||
|
||||
album_title = album.find('div', {'id': 'card-title'})
|
||||
if album_title and hasattr(album_title, 'text'):
|
||||
album_data.update({'album': album_title.text.replace('\n', '').replace('/', '-')})
|
||||
|
||||
albums_data_list.append(album_data)
|
||||
|
||||
return albums_data_list
|
||||
|
||||
|
||||
def scrape(artist):
|
||||
browser = new_browser(headless=True)
|
||||
url = QUERY_URL + artist
|
||||
browser.maximize_window()
|
||||
browser.implicitly_wait(1)
|
||||
response = browser.get(url)
|
||||
last_height = browser.execute_script("return document.body.scrollHeight")
|
||||
browser.execute_script("window.scrollTo(0, 500);")
|
||||
time.sleep(1)
|
||||
scrape_data = ''
|
||||
|
||||
try:
|
||||
# get the financial value when it's populated to the page
|
||||
value_element = WebDriverWait(browser, 5).until(
|
||||
EC.presence_of_element_located(locator=(By.XPATH, '//div[@id="shelf-container"]'))
|
||||
)
|
||||
element = browser.find_element(By.XPATH, ALBUM_CONTAINER_ITEMS_XPATH)
|
||||
if element:
|
||||
time.sleep(1)
|
||||
scrape_data += element.get_attribute('outerHTML')
|
||||
btn_right = browser.find_element(By.XPATH, BTN_RIGHT_FULL_XPATH)
|
||||
btn_right_displayed = True
|
||||
safety_index = 0
|
||||
while btn_right_displayed:
|
||||
# actions = ActionChains(browser)
|
||||
# actions.move_to_element(btn_right).perform()
|
||||
safety_index += 1
|
||||
time.sleep(1)
|
||||
browser.execute_script(click_script)
|
||||
time.sleep(1)
|
||||
element = browser.find_element(By.XPATH, ALBUM_CONTAINER_ITEMS_XPATH)
|
||||
scrape_data += element.get_attribute('outerHTML')
|
||||
time.sleep(1)
|
||||
btn_right_displayed = btn_right.is_displayed()
|
||||
if safety_index > 5:
|
||||
btn_right_displayed = False
|
||||
time.sleep(1)
|
||||
|
||||
finally:
|
||||
# after 5 seconds, give up
|
||||
browser.quit()
|
||||
|
||||
return scrape_data
|
||||
Loading…
Reference in new issue