adding digitools.smallset
This commit is contained in:
90
zArchive/Templates/TemplateScraper.py
Normal file
90
zArchive/Templates/TemplateScraper.py
Normal file
@@ -0,0 +1,90 @@
|
||||
import os, sys
|
||||
from datetime import datetime
|
||||
from dateutil import relativedelta
|
||||
|
||||
import django
|
||||
sys.path.append('../../../')
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
|
||||
django.setup()
|
||||
|
||||
from time import sleep
|
||||
from pprint import pprint as ppr
|
||||
import pytz
|
||||
|
||||
from events.models import Organization, Scraper
|
||||
from events.digitools import getBrowser, createURL, createBasicEvent, getSource
|
||||
|
||||
count = 0
|
||||
|
||||
venue, created = Organization.objects.get_or_create(
|
||||
name="Venue Name",
|
||||
city="Minneapolis",
|
||||
website="Event Website",
|
||||
)
|
||||
|
||||
try:
|
||||
scraper, created = Scraper.objects.get_or_create(
|
||||
name=venue.name,
|
||||
website=venue.website,
|
||||
items = 0,
|
||||
last_ran = datetime.now(),
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
scraper = Scraper.objects.get(name=venue.name)
|
||||
print("Scraper: ", scraper)
|
||||
|
||||
event_type = ""
|
||||
|
||||
# Time Signatures
|
||||
tz = pytz.timezone("US/Central")
|
||||
DATETIME_FORMAT = '%b %d %Y %I:%M %p'
|
||||
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
|
||||
|
||||
def get_events(ps, event_type):
|
||||
contents = ps.xpath('.//*/li[@class="event"]')
|
||||
for c in contents:
|
||||
try:
|
||||
event = {}
|
||||
day = c.xpath('.//*/span[@class="day"]/text()')[0]
|
||||
month = c.xpath('.//*/span[@class="mth"]/text()')[0]
|
||||
year = datetime.now().year
|
||||
if month == "Jan":
|
||||
year = int(year) + 1
|
||||
event['calendar'] = venue.calendar
|
||||
event['title'] = c.xpath('.//*/span[@class="event_title"]/a/text()')[0]
|
||||
event['date'] = [month, day, str(year), c.xpath('.//*/span[@class="event_time"]/text()')[0].strip()]
|
||||
event['date'] = " ".join(event['date'])
|
||||
event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
|
||||
event['link'] = c.xpath('.//*/span[@class="event_title"]/a/@href')[0]
|
||||
print("Event Dict Created")
|
||||
createBasicEvent(event, event_type, venue)
|
||||
scraper.items+=1
|
||||
except Exception as e:
|
||||
print(e)
|
||||
ppr(event)
|
||||
print("\n\n+++\n\n")
|
||||
|
||||
if len(sys.argv) >= 2:
|
||||
arg1 = sys.argv[1]
|
||||
br = getBrowser(arg1)
|
||||
else:
|
||||
print("No run_env")
|
||||
quit()
|
||||
|
||||
ps = getSource(br, venue.website)
|
||||
get_events(ps, "Ed")
|
||||
sleep(3)
|
||||
|
||||
|
||||
scraper.save()
|
||||
|
||||
# Get Event Page Link(s)
|
||||
# links = createURL("https://acmecomedycompany.com/the-club/calendar/")
|
||||
|
||||
# for link in links:
|
||||
# ps = getSource(br, link)
|
||||
# get_events(ps, "Ed")
|
||||
# sleep(3)
|
||||
|
||||
br.close()
|
||||
84
zArchive/Templates/ical_run.py.template
Normal file
84
zArchive/Templates/ical_run.py.template
Normal file
@@ -0,0 +1,84 @@
|
||||
import requests, os, sys
|
||||
from icalendar import Calendar as iCalendar, Event
|
||||
|
||||
from pprint import pprint as ppr
|
||||
import pytz
|
||||
|
||||
import django
|
||||
sys.path.append('../')
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
|
||||
django.setup()
|
||||
|
||||
from events.models import Event as DSEvent, Organization
|
||||
|
||||
venue, created = Organization.objects.get_or_create(
|
||||
name="location",
|
||||
city="Minneapolis",
|
||||
website="",
|
||||
)
|
||||
|
||||
event_type = ""
|
||||
|
||||
calendar_url = [
|
||||
'https://calendar.google.com/calendar/ical/js94epu90r2et31aopons1ifm8%40group.calendar.google.com/public/basic.ics',
|
||||
'https://calendar.google.com/calendar/ical/6rpooudjg01vc8bjek1snu2ro0%40group.calendar.google.com/public/basic.ics',
|
||||
'https://calendar.google.com/calendar/ical/teflgutelllvla7r6vfcmjdjjo%40group.calendar.google.com/public/basic.ics'
|
||||
]
|
||||
|
||||
objIcalData = requests.get(calendar_url[1])
|
||||
|
||||
gcal = iCalendar.from_ical(objIcalData.text)
|
||||
|
||||
cfpa_events = []
|
||||
tz = pytz.timezone("US/Central")
|
||||
|
||||
for component in gcal.walk():
|
||||
event = {}
|
||||
event['strSummary'] = f"{(component.get('SUMMARY'))}"
|
||||
event['strDesc'] = component.get('DESCRIPTION')
|
||||
event['strLocation'] = component.get('LOCATION')
|
||||
event['dateStart'] = component.get('DTSTART')
|
||||
event['dateStamp'] = component.get('DTSTAMP')
|
||||
if event['dateStamp'] is not None:
|
||||
event['dateStamp'] = event['dateStamp'].dt
|
||||
if event['dateStart'] is not None:
|
||||
try:
|
||||
event['dateStart'] = event['dateStart'].dt.astimezone(pytz.utc)
|
||||
except Exception as e:
|
||||
event['dateStart'] = event['dateStart'].dt
|
||||
|
||||
event['dateEnd'] = (component.get('DTEND'))
|
||||
if event['dateEnd'] is not None:
|
||||
event['dateEnd'] = event['dateEnd'].dt
|
||||
else:
|
||||
event['dateEnd'] = event['dateStart']
|
||||
if event['strSummary'] != 'None':
|
||||
event['details'] = {
|
||||
"description" : event['strDesc'],
|
||||
"DateTime" : event['dateStart'],
|
||||
"Location" : event['strLocation'],
|
||||
}
|
||||
cfpa_events.append(event)
|
||||
new_event = DSEvent.objects.update_or_create(
|
||||
event_type = event_type,
|
||||
show_title = event['strSummary'],
|
||||
show_link = event['link'],
|
||||
show_date = event['dateStart'],
|
||||
show_day = event['dateStart'].date(),
|
||||
more_details = event["details"],
|
||||
venue = venue
|
||||
)
|
||||
|
||||
|
||||
|
||||
# {'dateEnd': datetime.datetime(2022, 10, 22, 18, 30, tzinfo=<UTC>),
|
||||
# 'dateStamp': datetime.datetime(2023, 3, 23, 1, 57, 45, tzinfo=<UTC>),
|
||||
# 'dateStart': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
|
||||
# 'details': {'DateTime': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
|
||||
# 'Location': vText('b'''),
|
||||
# 'description': None},
|
||||
# 'strDesc': None,
|
||||
# 'strLocation': vText('b'''),
|
||||
# 'strSummary': 'Nia Class with Beth Giles'}
|
||||
|
||||
|
||||
84
zArchive/Templates/ical_template.py
Normal file
84
zArchive/Templates/ical_template.py
Normal file
@@ -0,0 +1,84 @@
|
||||
import requests, os, sys
|
||||
from icalendar import Calendar as iCalendar, Event
|
||||
|
||||
from pprint import pprint as ppr
|
||||
import pytz
|
||||
|
||||
import django
|
||||
sys.path.append('../')
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
|
||||
django.setup()
|
||||
|
||||
from events.models import Event as DSEvent, Organization
|
||||
|
||||
venue, created = Organization.objects.get_or_create(
|
||||
name="location",
|
||||
city="Minneapolis",
|
||||
website="",
|
||||
)
|
||||
|
||||
event_type = ""
|
||||
|
||||
calendar_url = [
|
||||
'https://calendar.google.com/calendar/ical/js94epu90r2et31aopons1ifm8%40group.calendar.google.com/public/basic.ics',
|
||||
'https://calendar.google.com/calendar/ical/6rpooudjg01vc8bjek1snu2ro0%40group.calendar.google.com/public/basic.ics',
|
||||
'https://calendar.google.com/calendar/ical/teflgutelllvla7r6vfcmjdjjo%40group.calendar.google.com/public/basic.ics'
|
||||
]
|
||||
|
||||
objIcalData = requests.get(calendar_url[1])
|
||||
|
||||
gcal = iCalendar.from_ical(objIcalData.text)
|
||||
|
||||
cfpa_events = []
|
||||
tz = pytz.timezone("US/Central")
|
||||
|
||||
for component in gcal.walk():
|
||||
event = {}
|
||||
event['strSummary'] = f"{(component.get('SUMMARY'))}"
|
||||
event['strDesc'] = component.get('DESCRIPTION')
|
||||
event['strLocation'] = component.get('LOCATION')
|
||||
event['dateStart'] = component.get('DTSTART')
|
||||
event['dateStamp'] = component.get('DTSTAMP')
|
||||
if event['dateStamp'] is not None:
|
||||
event['dateStamp'] = event['dateStamp'].dt
|
||||
if event['dateStart'] is not None:
|
||||
try:
|
||||
event['dateStart'] = event['dateStart'].dt.astimezone(pytz.utc)
|
||||
except Exception as e:
|
||||
event['dateStart'] = event['dateStart'].dt
|
||||
|
||||
event['dateEnd'] = (component.get('DTEND'))
|
||||
if event['dateEnd'] is not None:
|
||||
event['dateEnd'] = event['dateEnd'].dt
|
||||
else:
|
||||
event['dateEnd'] = event['dateStart']
|
||||
if event['strSummary'] != 'None':
|
||||
event['details'] = {
|
||||
"description" : event['strDesc'],
|
||||
"DateTime" : event['dateStart'],
|
||||
"Location" : event['strLocation'],
|
||||
}
|
||||
cfpa_events.append(event)
|
||||
new_event = DSEvent.objects.update_or_create(
|
||||
event_type = event_type,
|
||||
show_title = event['strSummary'],
|
||||
show_link = event['link'],
|
||||
show_date = event['dateStart'],
|
||||
show_day = event['dateStart'].date(),
|
||||
more_details = event["details"],
|
||||
venue = venue
|
||||
)
|
||||
|
||||
|
||||
|
||||
# {'dateEnd': datetime.datetime(2022, 10, 22, 18, 30, tzinfo=<UTC>),
|
||||
# 'dateStamp': datetime.datetime(2023, 3, 23, 1, 57, 45, tzinfo=<UTC>),
|
||||
# 'dateStart': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
|
||||
# 'details': {'DateTime': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
|
||||
# 'Location': vText('b'''),
|
||||
# 'description': None},
|
||||
# 'strDesc': None,
|
||||
# 'strLocation': vText('b'''),
|
||||
# 'strSummary': 'Nia Class with Beth Giles'}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user