updated to running
This commit is contained in:
@@ -13,24 +13,18 @@ from selenium import webdriver as wd
|
||||
|
||||
from events.models import Event as DSEvent, Organization, Promo, Scraper, Calendar
|
||||
|
||||
tz = pytz.timezone("US/Central")
|
||||
td = relativedelta.relativedelta(months=1)
|
||||
odt = datetime.now() + td
|
||||
# tz = pytz.timezone("US/Central")
|
||||
plus_one_month = relativedelta.relativedelta(months=1)
|
||||
odt_next_month = datetime.now() + plus_one_month
|
||||
|
||||
|
||||
def getReady():
|
||||
sys.path.append('../../')
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
|
||||
django.setup()
|
||||
return
|
||||
|
||||
|
||||
def getScraper(venue, cal):
|
||||
# Get Scraper name, item count and online_calendar (virtcal)
|
||||
def getScraper(venue, webite, cal):
|
||||
virtcal = Calendar.objects.get(shortcode='000')
|
||||
try:
|
||||
scraper, created = Scraper.objects.get_or_create(
|
||||
name=venue.name,
|
||||
website=venue.website,
|
||||
website=website,
|
||||
calendar = Calendar.objects.get(shortcode=cal),
|
||||
items = 0,
|
||||
new_items = 0,
|
||||
@@ -43,9 +37,9 @@ def getScraper(venue, cal):
|
||||
scraper.items = len(num_of_events)
|
||||
scraper.save()
|
||||
print("Scraper: ", scraper)
|
||||
pass
|
||||
return scraper, scraper.items, virtcal
|
||||
|
||||
# Update item_count of the Scraper at the end of the scrape
|
||||
def updateScraper(scraper, item_count_start):
|
||||
num_of_events = DSEvent.objects.filter(scraper=scraper)
|
||||
scraper.items = len(num_of_events)
|
||||
@@ -54,19 +48,18 @@ def updateScraper(scraper, item_count_start):
|
||||
scraper.save()
|
||||
return
|
||||
|
||||
# Get site HTML content for XPATH travel
|
||||
def getSource(browser, link):
|
||||
browser.get(link)
|
||||
sleep(5)
|
||||
ps = html.fromstring(browser.page_source)
|
||||
return ps
|
||||
|
||||
# Get Selenium Web Drive, with params for Chrome or Firefox
|
||||
# Or in production to run headless
|
||||
def getBrowser(run_env):
|
||||
if run_env == 'dev':
|
||||
print("Chrome is a go!")
|
||||
# chromeOptions = wd.ChromeOptions()
|
||||
# chromeOptions.binary_location = "/Application/Google\ Chrome.app"
|
||||
# chromeDriver = "/opt/homebrew/bin/chromedriver"
|
||||
# br = wd.Chrome(chromeDriver, options=chromeOptions)
|
||||
br = wd.Chrome()
|
||||
return br
|
||||
elif run_env == "def":
|
||||
@@ -85,19 +78,28 @@ def getBrowser(run_env):
|
||||
print("Failed", sys.argv, arg1)
|
||||
quit()
|
||||
|
||||
# Create Dated URL with zero-padded numbers
|
||||
def createBasicURL(site_url):
|
||||
month = datetime.now().month
|
||||
next_month = odt.month
|
||||
next_month = odt_next_month.month
|
||||
year = datetime.now().year
|
||||
links = [
|
||||
if next_month == 1:
|
||||
next_year = year+1
|
||||
links = [
|
||||
site_url + str(month) + "/" + str(year),
|
||||
site_url + str(next_month) + "/" + str(year)
|
||||
site_url + str(next_month) + "/" + str(next_year)
|
||||
]
|
||||
else:
|
||||
links = [
|
||||
site_url + str(month) + "/" + str(year),
|
||||
site_url + str(next_month) + "/" + str(year)
|
||||
]
|
||||
return links
|
||||
|
||||
# Create Dated URL without zero-padded numbers
|
||||
def createURLNoZero(site_url):
|
||||
month = datetime.now().month
|
||||
next_month = odt.month
|
||||
next_month = odt_next_month.month
|
||||
year = datetime.now().year
|
||||
links = [
|
||||
site_url + str(year) + "/" + str(month),
|
||||
@@ -108,13 +110,14 @@ def createURLNoZero(site_url):
|
||||
links.append(site_url + str(year) + "/" + str(next_month))
|
||||
return links
|
||||
|
||||
# Create Dated URL Link with zero-padding
|
||||
def createURL(site_url):
|
||||
month = datetime.now().month
|
||||
if month < 10:
|
||||
month = "0" + str(month)
|
||||
else:
|
||||
month = str(month)
|
||||
next_month = odt.month
|
||||
next_month = odt_next_month.month
|
||||
if next_month < 10:
|
||||
next_month = "0" + str(next_month)
|
||||
else:
|
||||
@@ -129,6 +132,7 @@ def createURL(site_url):
|
||||
links.append(site_url + str(year) + "/" + next_month)
|
||||
return links
|
||||
|
||||
# Create Dated URL with dashes
|
||||
def createDashURL(site_url):
|
||||
month = datetime.now().month
|
||||
if month < 10:
|
||||
@@ -148,16 +152,16 @@ def createDashURL(site_url):
|
||||
print(links)
|
||||
return links
|
||||
|
||||
# Add Calendar to Event Object (maybe extraneous)
|
||||
def add_calendar(event, calendar):
|
||||
print("\nNEW EVENT DIGITOOLS: ", event, type(event), calendar)
|
||||
if type(event) is tuple:
|
||||
print("STS: ", event)
|
||||
event = event[0]
|
||||
cal = Calendar.objects.get(shortcode=calendar)
|
||||
event.calendar.add(cal)
|
||||
event.save()
|
||||
return event
|
||||
|
||||
# Add Calendars to Event Object ??
|
||||
def add_calendars(event, data):
|
||||
if type(data['calendars']) is not list:
|
||||
event.calendar.add(data['calendars'])
|
||||
@@ -167,6 +171,7 @@ def add_calendars(event, data):
|
||||
event.save()
|
||||
return event
|
||||
|
||||
# Create Basic DigiSnaxx Event
|
||||
def createBasicEvent(event, event_type, venue):
|
||||
new_event, created = DSEvent.objects.update_or_create(
|
||||
event_type = event_type,
|
||||
@@ -178,8 +183,10 @@ def createBasicEvent(event, event_type, venue):
|
||||
venue = venue
|
||||
)
|
||||
new_event = add_calendars(new_event, event)
|
||||
print("\n+new event+\n")
|
||||
return new_event, created
|
||||
|
||||
# Create iCal Event
|
||||
def createBasiciCalEvent(event, event_type, venue):
|
||||
new_event, created = DSEvent.objects.update_or_create(
|
||||
event_type = event_type,
|
||||
@@ -191,8 +198,11 @@ def createBasiciCalEvent(event, event_type, venue):
|
||||
venue = venue
|
||||
)
|
||||
new_event = add_calendars(new_event, event)
|
||||
print("\n+new event+\n")
|
||||
return new_event, created
|
||||
|
||||
# Create Detailed Event with Details & Guests
|
||||
# Details in JSON Format
|
||||
def createDetailedEvent(event, event_type, venue, scraper):
|
||||
new_event, created = DSEvent.objects.update_or_create(
|
||||
event_type = event_type,
|
||||
@@ -206,31 +216,10 @@ def createDetailedEvent(event, event_type, venue, scraper):
|
||||
venue = venue
|
||||
)
|
||||
new_event = add_calendars(new_event, event)
|
||||
print("\n+new event+\n")
|
||||
return new_event, created
|
||||
|
||||
def createBasicArticle(article, event_type, organization):
|
||||
new_article, created = Promo.objects.update_or_create(
|
||||
promo_type = 'Ja',
|
||||
title = article['title'],
|
||||
target_link = article['link'],
|
||||
published = True,
|
||||
organization = organization
|
||||
)
|
||||
return new_article, created
|
||||
|
||||
def createIcalEvent(event, scraper, venue, event_type):
|
||||
new_date = event['dateStart']
|
||||
new_event = {}
|
||||
new_event['scraper'] = scraper
|
||||
new_event['calendar'] = scraper.calendar
|
||||
new_event['title'] = event['strSummary'],
|
||||
new_event['date'] = str(new_date)[:-6],
|
||||
new_event['dateStamp'] = str(new_date)[:-6],
|
||||
new_event['link'] = venue.website
|
||||
print("NEW iCal Event")
|
||||
ppr(new_event)
|
||||
createBasiciCalEvent(new_event, event_type, venue)
|
||||
|
||||
# Create iCal event from DF_Online & Medellin
|
||||
def createCleanIcalEvent(event, scraper, venue, event_type):
|
||||
new_date = event['dateStart']
|
||||
new_event = {}
|
||||
@@ -242,6 +231,7 @@ def createCleanIcalEvent(event, scraper, venue, event_type):
|
||||
new_event['link'] = venue.website
|
||||
createBasiciCalEvent(new_event, event_type, venue)
|
||||
|
||||
# Get events from iCal
|
||||
def getiCalEvents(gcal, scraper, venue, event_type):
|
||||
events = []
|
||||
for component in gcal.walk():
|
||||
@@ -259,13 +249,8 @@ def getiCalEvents(gcal, scraper, venue, event_type):
|
||||
try:
|
||||
event['dateStart'] = event['dateStart'].dt
|
||||
except Exception as e:
|
||||
event['dateStart'] = event['dateStart'].dt
|
||||
print("what? ", e)
|
||||
|
||||
event['dateEnd'] = (component.get('DTEND'))
|
||||
if event['dateEnd'] is not None:
|
||||
event['dateEnd'] = event['dateEnd'].dt
|
||||
else:
|
||||
event['dateEnd'] = event['dateStart']
|
||||
if event['strSummary'] != 'None':
|
||||
event['details'] = {
|
||||
"description" : event['strDesc'],
|
||||
@@ -274,55 +259,110 @@ def getiCalEvents(gcal, scraper, venue, event_type):
|
||||
events.append(event)
|
||||
return events
|
||||
|
||||
def returniCalEvents(gcal, scraper, venue, event_type):
|
||||
events = []
|
||||
# Build iCal Events and Send to Create
|
||||
def buildiCalEvents(events, event_type, scraper):
|
||||
for event in events:
|
||||
e = {}
|
||||
e['calendars'] = event['calendars']
|
||||
try:
|
||||
e['dateStamp'] = event['dateStart'][0]
|
||||
except:
|
||||
e['dateStamp'] = event['dateStart']
|
||||
e['title'] = event['strSummary']
|
||||
e['scraper'] = scraper
|
||||
e['link'] = venue.website
|
||||
try:
|
||||
createBasicEvent(e, event_type, venue)
|
||||
scraper.items+=1
|
||||
except Exception as e:
|
||||
print("Error: ", e)
|
||||
scraper.save()
|
||||
return
|
||||
|
||||
def getMDEVenue(venue):
|
||||
if venue.name == "DANCEFREE":
|
||||
venue.website = "https://www.instagram.com/dancefreeco"
|
||||
if venue.name == "Vintrash":
|
||||
venue.website = "https://www.instagram.com/vintrashbar"
|
||||
if venue.name == "The Wandering Paisa":
|
||||
venue.website = "https://wanderingpaisahostel.com"
|
||||
if venue.name == "Dulce Posion":
|
||||
venue.website = "https://www.instagram.com/dulceposionr"
|
||||
if venue.name == "Blood Dance Company":
|
||||
venue.website = "https://www.instagram.com/blooddancecompany"
|
||||
if venue.name == "OLSA Certified Spanish School":
|
||||
venue.website = "https://www.olsafoundation.org/"
|
||||
if event['strSummary'] == "Merli Rooftop Language Exchange":
|
||||
venue.website = "https://calendar.google.com/calendar/embed?src=46ae0446724b1b3ee83cbd7dbc0db6a235bf97509ad860ca91eada3c267b5e41%40group.calendar.google.com&ctz=America%2FBogota"
|
||||
if "Concious Warrior" in event['strSummary']:
|
||||
venue.website = "https://www.consciouscolombia.com/"
|
||||
venue.save()
|
||||
return
|
||||
|
||||
# Get iCal events for Medellin & OnlineEvents
|
||||
def getiCalRepeateEvents(gcal, scraper, venue, event_type, cal):
|
||||
for component in gcal.walk():
|
||||
event = {}
|
||||
event['scraper'] = scraper
|
||||
event['calendars'] = [scraper.calendar]
|
||||
event['strSummary'] = f"{(component.get('SUMMARY'))}"
|
||||
event['strDesc'] = component.get('DESCRIPTION')
|
||||
event['strLocation'] = component.get('LOCATION')
|
||||
event['strLocation'] = str(component.get('LOCATION'))
|
||||
event['dateStart'] = component.get('DTSTART')
|
||||
event['dateStamp'] = component.get('DTSTAMP')
|
||||
if event['dateStamp'] is not None:
|
||||
event['dateStamp'] = event['dateStamp'].dt
|
||||
if event['dateStart'] is not None:
|
||||
try:
|
||||
event['dateStart'] = event['dateStart'].dt
|
||||
except Exception as e:
|
||||
event['dateStart'] = event['dateStart'].dt
|
||||
|
||||
event['dateEnd'] = (component.get('DTEND'))
|
||||
if event['dateEnd'] is not None:
|
||||
event['dateEnd'] = event['dateEnd'].dt
|
||||
else:
|
||||
event['dateEnd'] = event['dateStart']
|
||||
if event['strSummary'] != 'None':
|
||||
event['details'] = {
|
||||
"description" : event['strDesc'],
|
||||
"Location" : event['strLocation'],
|
||||
}
|
||||
events.append(event)
|
||||
now_now = datetime.today().date()
|
||||
try:
|
||||
print("1Event: ", event['dateStart'])
|
||||
if event['dateStart'] > now_now:
|
||||
print("Check Check: ", event['dateStart'])
|
||||
createIcalEvent(event, scraper, venue, event_type)
|
||||
else:
|
||||
print("WHAT?")
|
||||
ppr(event)
|
||||
except Exception as e:
|
||||
if event['dateStamp'] != None:
|
||||
event['dateStart'] = event['dateStart'].dt
|
||||
event['dateStart'] = datetime.strptime(str(event['dateStart'])[:-6], '%Y-%m-%d %H:%M:%S')
|
||||
rules = component.get('RRule')
|
||||
try:
|
||||
event['dateStart'] = event['dateStart'].date()
|
||||
print("2Event: ", event['dateStart'])
|
||||
if event['dateStart'] > now_now:
|
||||
print("Check Check: ", event['dateStart'])
|
||||
createIcalEvent(event, scraper, venue, event_type)
|
||||
else:
|
||||
print("WHAT?")
|
||||
ppr(event)
|
||||
if rules['FREQ'][0] == 'WEEKLY':
|
||||
if datetime.today().weekday() != 0:
|
||||
event = digitools.splitLocation(event, "Medellin")
|
||||
date = datetime.today().date() - timedelta(days=datetime.today().weekday())
|
||||
date = datetime.combine(date, event['dateStart'].time())
|
||||
days = ["SU", "MO", "TU", "WE", "TH", "FR", "SA"]
|
||||
for day in rules['BYDAY']:
|
||||
day = days.index(day)
|
||||
if cal == 'mde':
|
||||
getVenue(event['venue'])
|
||||
iCalEventRepeatFilter(day, date, event, scraper, event['venue'], "Ed")
|
||||
|
||||
except Exception as e:
|
||||
print("The Error: ", e)
|
||||
pass
|
||||
print("Error: ", e, "\n\n\n\n")
|
||||
pass
|
||||
|
||||
def iCalEventRepeatFilterteEvent(day, date, event, scraper, venue, event_type):
|
||||
days = [day-1, day+6, day+13]
|
||||
for day in days:
|
||||
event['dateStamp'] = date + timedelta(days=day)
|
||||
event['dateStart'] = event['dateStamp']
|
||||
digitools.createCleanIcalEvent(event, scraper, venue, event_type)
|
||||
return
|
||||
|
||||
def splitLocation(event, **kwargs):
|
||||
loc_split = event['strLocation'].split(',')
|
||||
venue_name = loc_split[0]
|
||||
venue, created = Organization.objects.get_or_create(
|
||||
name=venue_name,
|
||||
)
|
||||
event['venue'] = venue
|
||||
if city:
|
||||
venue.city = kwargs['city']
|
||||
venue.save()
|
||||
return event
|
||||
|
||||
# ARCHIVED Methods
|
||||
def createBasicArticle(article, event_type, organization):
|
||||
new_article, created = Promo.objects.update_or_create(
|
||||
promo_type = 'Ja',
|
||||
title = article['title'],
|
||||
target_link = article['link'],
|
||||
published = True,
|
||||
organization = organization
|
||||
)
|
||||
return new_article, created
|
||||
Reference in New Issue
Block a user