diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f623792 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*/__pycache__ +*/*/__pycache__ diff --git a/db.sqlite3 b/db.sqlite3 index 09ae2e4..31f8632 100644 Binary files a/db.sqlite3 and b/db.sqlite3 differ diff --git a/ds_events/settings.py b/ds_events/settings.py index 24c2d98..57ad182 100644 --- a/ds_events/settings.py +++ b/ds_events/settings.py @@ -49,7 +49,7 @@ INSTALLED_APPS = [ 'rest_framework.authtoken', 'socials', 'events', - 'academia_nuts', + # 'academia_nuts', # 'leg_info', ] diff --git a/events/admin.py b/events/admin.py index ccdb6a4..e42225d 100644 --- a/events/admin.py +++ b/events/admin.py @@ -13,8 +13,8 @@ class OrganizationAdmin(admin.ModelAdmin): class EventAdmin(admin.ModelAdmin): # prepopulated_fields = {"slug": ("shortname",)} - list_display = ( "show_title", "event_type", "show_date",) - list_filter = ("venue", "event_type") + list_display = ( "show_title", "event_type", "show_date", "calendar") + list_filter = ("calendar", "venue", "event_type",) class PromoAdmin(admin.ModelAdmin): # prepopulated_fields = {"slug": ("shortname",)} diff --git a/events/digitools.py b/events/digitools.py index 9bf6938..bbfe312 100644 --- a/events/digitools.py +++ b/events/digitools.py @@ -197,7 +197,29 @@ def createBasicArticle(article, event_type, organization): ) return new_article, created -def getiCalEvents(gcal, scraper): +def createIcalEvent(event, scraper, venue, event_type): + new_date = event['dateStart'] + new_event = {} + new_event['scraper'] = scraper + new_event['calendar'] = scraper.calendar + new_event['title'] = event['strSummary'], + new_event['date'] = str(new_date)[:-6], + new_event['dateStamp'] = str(new_date)[:-6], + new_event['link'] = venue.website + createBasiciCalEvent(new_event, event_type, venue) + +def createCleanIcalEvent(event, scraper, venue, event_type): + new_date = event['dateStart'] + new_event = {} + new_event['scraper'] = scraper + new_event['calendar'] = scraper.calendar + new_event['title'] = event['strSummary'], + new_event['date'] = str(new_date), + new_event['dateStamp'] = str(new_date), + new_event['link'] = venue.website + createBasiciCalEvent(new_event, event_type, venue) + +def getiCalEvents(gcal, scraper, venue): for component in gcal.walk(): event = {} event['scraper'] = scraper @@ -229,30 +251,13 @@ def getiCalEvents(gcal, scraper): try: print("1Event: ", event['dateStart']) if event['dateStart'] > now_now: - new_date = event['dateStart']-td - new_event = {} - new_event['scraper'] = scraper - new_event['calendar'] = scraper.calendar - new_event['title'] = event['strSummary'], - new_event['date'] = str(new_date), - new_event['dateStamp'] = str(new_date), - new_event['link'] = venue.website - createBasiciCalEvent(new_event, "Mu", venue) + createIcalEvent(event, scraper, venue) except Exception as e: try: event['dateStart'] = event['dateStart'].date() print("1Event: ", event['dateStart']) if event['dateStart'] > now_now: - new_date = event['dateStart']-td - print("The new Date: ", new_date, type(new_date)) - new_event = {} - new_event['scraper'] = scraper - new_event['calendar'] = scraper.calendar - new_event['title'] = event['strSummary'], - new_event['date'] = new_date, - new_event['dateStamp'] = new_date, - new_event['link'] = venue.website - createBasiciCalEvent(new_event, "Mu", venue) + createIcalEvent(event, scraper, venue) except Exception as e: print("The Error: ", e) pass \ No newline at end of file diff --git a/events/scrapers/Working/iCal/ical.CAFAC.mpls.py b/events/scrapers/Working/iCal/ical.CAFAC.mpls.py index ffc67c3..0e43759 100644 --- a/events/scrapers/Working/iCal/ical.CAFAC.mpls.py +++ b/events/scrapers/Working/iCal/ical.CAFAC.mpls.py @@ -46,5 +46,5 @@ event_type = "Ed" objIcalData = requests.get(scraper.website) gcal = iCalendar.from_ical(objIcalData.text) tz = pytz.timezone("US/Central") -digitools.getiCalEvents(gcal, scraper) +digitools.getiCalEvents(gcal, scraper, venue) digitools.updateScraper(scraper, item_count_start) diff --git a/events/scrapers/Working/iCal/ical_ingles.py b/events/scrapers/Working/iCal/ical_ingles.py new file mode 100644 index 0000000..6d38867 --- /dev/null +++ b/events/scrapers/Working/iCal/ical_ingles.py @@ -0,0 +1,118 @@ +import requests, os, sys +from icalendar import Calendar as iCalendar, Event + +from datetime import datetime, timedelta +from dateutil import relativedelta +td = relativedelta.relativedelta(hours=5) + +from pprint import pprint as ppr +import pytz + +import django +sys.path.append('../../../../') +os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings' +django.setup() + +from events.models import Event as DSEvent, Organization, Scraper, Calendar +import events.digitools as digitools +from dateutil import relativedelta +td = relativedelta.relativedelta(hours=5) + + +venue, created = Organization.objects.get_or_create( + name="idioki", + city="Medellin", + website="https://idioki.com/", + ) +event_type = "Mu" + +try: + scraper, created = Scraper.objects.get_or_create( + name="idioki", + website="https://calendar.google.com/calendar/ical/46ae0446724b1b3ee83cbd7dbc0db6a235bf97509ad860ca91eada3c267b5e41%40group.calendar.google.com/public/basic.ics", + calendar = Calendar.objects.get(shortcode='mde'), + items = 0, + new_items = 0, + last_ran = datetime.now(), + ) +except Exception as e: + print(e) + scraper = Scraper.objects.get(name=venue.name) + +item_count_start = scraper.items + +objIcalData = requests.get(scraper.website) +gcal = iCalendar.from_ical(objIcalData.text) +tz = pytz.timezone("US/Central") + +def createEvent(day, date, event, scraper, venue, event_type): + days = [day-1, day+6, day+13] + for day in days: + event['dateStamp'] = date + timedelta(days=day) + event['dateStart'] = event['dateStamp'] + digitools.createCleanIcalEvent(event, scraper, venue, event_type) + return + +def splitLocation(event): + loc_split = event['strLocation'].split(',') + venue_name = loc_split[0] + venue, created = Organization.objects.get_or_create( + name=venue_name, + city="Medellin", + website="https://idioki.com/", + ) + event['venue'] = venue + return event + +counter = 0 + +for component in gcal.walk(): + event = {} + event['scraper'] = scraper + event['calendar'] = scraper.calendar + event['strSummary'] = f"{(component.get('SUMMARY'))}" + event['strDesc'] = component.get('DESCRIPTION') + event['strLocation'] = str(component.get('LOCATION')) + # startDate = component.get('DTSTART') + # startTime = startDate.time() + event['dateStart'] = component.get('DTSTART') + event['dateStamp'] = component.get('DTSTAMP') + + if event['strSummary'] != 'None': + event['details'] = { + "description" : event['strDesc'], + "Location" : event['strLocation'], + } + + if event['dateStamp'] != None: + event['dateStart'] = event['dateStart'].dt + event['dateStart'] = datetime.strptime(str(event['dateStart'])[:-6], '%Y-%m-%d %H:%M:%S') + rules = component.get('RRule') + if rules['FREQ'][0] == 'WEEKLY': + if datetime.today().weekday() != 0: + event = splitLocation(event) + date = datetime.today().date() - timedelta(days=datetime.today().weekday()) + date = datetime.combine(date, event['dateStart'].time()) + days = ["SU", "MO", "TU", "WE", "TH", "FR", "SA"] + for day in rules['BYDAY']: + day = days.index(day) + loc_split = event['strLocation'].split(",") + venue, created = Organization.objects.get_or_create( + name=loc_split[0], + city="Medellin", + website="https://idioki.com/", + ) + createEvent(day, date, event, scraper, venue, "Ed") + +digitools.updateScraper(scraper, item_count_start) + +new_time = datetime.now() - timedelta(days=1) +right_bound_time = datetime.now() + timedelta(days=45) +events = DSEvent.objects.filter(show_date__lte=new_time) +events1 = DSEvent.objects.filter(show_date__gte=right_bound_time) + +for e in events: + e.delete() + +for e in events1: + e.delete() \ No newline at end of file diff --git a/events/scrapers/Working/iCal/ical_run.KJHideaway.StPaul.py b/events/scrapers/Working/iCal/ical_run.KJHideaway.StPaul.py index 6390dce..4946118 100644 --- a/events/scrapers/Working/iCal/ical_run.KJHideaway.StPaul.py +++ b/events/scrapers/Working/iCal/ical_run.KJHideaway.StPaul.py @@ -11,6 +11,8 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings' django.setup() from events.models import Event as DSEvent, Organization, Scraper, Calendar +import events.digitools as digitools + from dateutil import relativedelta td = relativedelta.relativedelta(hours=5) @@ -40,5 +42,5 @@ event_type = "Mu" objIcalData = requests.get(scraper.website) gcal = iCalendar.from_ical(objIcalData.text) tz = pytz.timezone("US/Central") -digitools.getiCalEvents(gcal, scraper) +digitools.getiCalEvents(gcal, scraper, venue) digitools.updateScraper(scraper, item_count_start) \ No newline at end of file diff --git a/events/scrapers/Working/iCal/ical_run.SocialableCider.mpls.py b/events/scrapers/Working/iCal/ical_run.SocialableCider.mpls.py index 386453d..bde0fa6 100644 --- a/events/scrapers/Working/iCal/ical_run.SocialableCider.mpls.py +++ b/events/scrapers/Working/iCal/ical_run.SocialableCider.mpls.py @@ -44,5 +44,5 @@ item_count_start = scraper.items objIcalData = requests.get(scraper.website) gcal = iCalendar.from_ical(objIcalData.text) tz = pytz.timezone("US/Central") -digitools.getiCalEvents(gcal, scraper) +digitools.getiCalEvents(gcal, scraper, venue) digitools.updateScraper(scraper, item_count_start) \ No newline at end of file diff --git a/events/scrapers/Working/iCal/ical_run.bunkers.py b/events/scrapers/Working/iCal/ical_run.bunkers.py index 05afd31..982f86d 100644 --- a/events/scrapers/Working/iCal/ical_run.bunkers.py +++ b/events/scrapers/Working/iCal/ical_run.bunkers.py @@ -44,5 +44,5 @@ event_type = "Mu" objIcalData = requests.get(scraper.website) gcal = iCalendar.from_ical(objIcalData.text) tz = pytz.timezone("US/Central") -digitools.getiCalEvents(gcal, scraper) +digitools.getiCalEvents(gcal, scraper, venue) digitools.updateScraper(scraper, item_count_start) diff --git a/events/scrapers/Working/iCal/ical_run.cfpa.py b/events/scrapers/Working/iCal/ical_run.cfpa.py index aba29e5..77b9afd 100644 --- a/events/scrapers/Working/iCal/ical_run.cfpa.py +++ b/events/scrapers/Working/iCal/ical_run.cfpa.py @@ -44,5 +44,5 @@ event_type = "Ed" objIcalData = requests.get(scraper.website) gcal = iCalendar.from_ical(objIcalData.text) tz = pytz.timezone("US/Central") -digitools.getiCalEvents(gcal, scraper) +digitools.getiCalEvents(gcal, scraper, venue) digitools.updateScraper(scraper, item_count_start) \ No newline at end of file diff --git a/events/scrapers/Working/iCal/ical_run.eagles.py b/events/scrapers/Working/iCal/ical_run.eagles.py index 21003c6..2a92be5 100644 --- a/events/scrapers/Working/iCal/ical_run.eagles.py +++ b/events/scrapers/Working/iCal/ical_run.eagles.py @@ -42,5 +42,5 @@ event_type = "Mu" objIcalData = requests.get(scraper.website) gcal = iCalendar.from_ical(objIcalData.text) tz = pytz.timezone("US/Central") -digitools.getiCalEvents(gcal, scraper) +digitools.getiCalEvents(gcal, scraper, venue) digitools.updateScraper(scraper, item_count_start) \ No newline at end of file diff --git a/events/scrapers/Working/iCal/ical_run.terminalbar-mpls.py b/events/scrapers/Working/iCal/ical_run.terminalbar-mpls.py index c8d9d35..5278fb0 100644 --- a/events/scrapers/Working/iCal/ical_run.terminalbar-mpls.py +++ b/events/scrapers/Working/iCal/ical_run.terminalbar-mpls.py @@ -46,5 +46,5 @@ event_type = "Mu" objIcalData = requests.get(scraper.website) gcal = iCalendar.from_ical(objIcalData.text) tz = pytz.timezone("US/Central") -digitools.getiCalEvents(gcal, scraper) +digitools.getiCalEvents(gcal, scraper, venue) digitools.updateScraper(scraper, item_count_start) \ No newline at end of file