scraper updates

.gitignore

@@ -0,0 +1,2 @@
+*/__pycache__
+*/*/__pycache__

ds_events/settings.py

@@ -49,7 +49,7 @@ INSTALLED_APPS = [
     'rest_framework.authtoken',
     'socials',
     'events',
-    'academia_nuts',
+    # 'academia_nuts',
     # 'leg_info',
 ]
 

events/admin.py

@@ -13,8 +13,8 @@ class OrganizationAdmin(admin.ModelAdmin):
 
 class EventAdmin(admin.ModelAdmin):
 #    prepopulated_fields = {"slug": ("shortname",)}
-    list_display = ( "show_title", "event_type", "show_date",)
+    list_display = ( "show_title", "event_type", "show_date", "calendar")
-    list_filter = ("venue", "event_type")
+    list_filter = ("calendar", "venue", "event_type",)
 
 class PromoAdmin(admin.ModelAdmin):
 #    prepopulated_fields = {"slug": ("shortname",)}

events/digitools.py

@@ -197,7 +197,29 @@ def createBasicArticle(article, event_type, organization):
     )
     return new_article, created
 
-def getiCalEvents(gcal, scraper):
+def createIcalEvent(event, scraper, venue, event_type):
+    new_date = event['dateStart']
+    new_event = {}
+    new_event['scraper'] = scraper
+    new_event['calendar'] = scraper.calendar
+    new_event['title'] = event['strSummary'],
+    new_event['date'] = str(new_date)[:-6],
+    new_event['dateStamp'] = str(new_date)[:-6],
+    new_event['link'] = venue.website
+    createBasiciCalEvent(new_event, event_type, venue)
+
+def createCleanIcalEvent(event, scraper, venue, event_type):
+    new_date = event['dateStart']
+    new_event = {}
+    new_event['scraper'] = scraper
+    new_event['calendar'] = scraper.calendar
+    new_event['title'] = event['strSummary'],
+    new_event['date'] = str(new_date),
+    new_event['dateStamp'] = str(new_date),
+    new_event['link'] = venue.website
+    createBasiciCalEvent(new_event, event_type, venue)
+
+def getiCalEvents(gcal, scraper, venue):
     for component in gcal.walk():
         event = {}
         event['scraper'] = scraper
@@ -229,30 +251,13 @@ def getiCalEvents(gcal, scraper):
             try:
                 print("1Event: ", event['dateStart'])
                 if event['dateStart'] > now_now:
-                    new_date = event['dateStart']-td
+                    createIcalEvent(event, scraper, venue)
-                    new_event = {}
-                    new_event['scraper'] = scraper
-                    new_event['calendar'] = scraper.calendar
-                    new_event['title'] = event['strSummary'],
-                    new_event['date'] = str(new_date),
-                    new_event['dateStamp'] = str(new_date),
-                    new_event['link'] = venue.website
-                    createBasiciCalEvent(new_event, "Mu", venue)
             except Exception as e:
                 try:
                     event['dateStart'] = event['dateStart'].date()
                     print("1Event: ", event['dateStart'])
                     if event['dateStart'] > now_now:
-                        new_date = event['dateStart']-td
+                        createIcalEvent(event, scraper, venue)
-                        print("The new Date: ", new_date, type(new_date))
-                        new_event = {}
-                        new_event['scraper'] = scraper
-                        new_event['calendar'] = scraper.calendar
-                        new_event['title'] = event['strSummary'],
-                        new_event['date'] = new_date,
-                        new_event['dateStamp'] = new_date,
-                        new_event['link'] = venue.website
-                        createBasiciCalEvent(new_event, "Mu", venue)
                 except Exception as e:
                     print("The Error: ", e)
                     pass

events/scrapers/Working/iCal/ical.CAFAC.mpls.py

@@ -46,5 +46,5 @@ event_type = "Ed"
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 tz = pytz.timezone("US/Central")
-digitools.getiCalEvents(gcal, scraper)
+digitools.getiCalEvents(gcal, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_ingles.py

@@ -0,0 +1,118 @@
+import requests, os, sys
+from icalendar import Calendar as iCalendar, Event
+
+from datetime import datetime, timedelta
+from dateutil import relativedelta
+td = relativedelta.relativedelta(hours=5)
+
+from pprint import pprint as ppr
+import pytz
+
+import django
+sys.path.append('../../../../')
+os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
+django.setup()
+
+from events.models import Event as DSEvent, Organization, Scraper, Calendar
+import events.digitools as digitools
+from dateutil import relativedelta
+td = relativedelta.relativedelta(hours=5)
+
+
+venue, created = Organization.objects.get_or_create(
+     name="idioki",
+     city="Medellin",
+     website="https://idioki.com/",
+    )
+event_type = "Mu"
+
+try:
+    scraper, created = Scraper.objects.get_or_create(
+        name="idioki",
+        website="https://calendar.google.com/calendar/ical/46ae0446724b1b3ee83cbd7dbc0db6a235bf97509ad860ca91eada3c267b5e41%40group.calendar.google.com/public/basic.ics",
+        calendar = Calendar.objects.get(shortcode='mde'),
+        items = 0,
+        new_items = 0,
+        last_ran = datetime.now(),
+        )
+except Exception as e:
+    print(e)
+    scraper = Scraper.objects.get(name=venue.name)
+
+item_count_start = scraper.items
+
+objIcalData = requests.get(scraper.website)
+gcal = iCalendar.from_ical(objIcalData.text)
+tz = pytz.timezone("US/Central")
+
+def createEvent(day, date, event, scraper, venue, event_type):
+    days = [day-1, day+6, day+13]
+    for day in days:
+        event['dateStamp'] = date + timedelta(days=day)
+        event['dateStart'] = event['dateStamp']
+        digitools.createCleanIcalEvent(event, scraper, venue, event_type)
+    return
+
+def splitLocation(event):
+     loc_split = event['strLocation'].split(',')
+     venue_name = loc_split[0]
+     venue, created = Organization.objects.get_or_create(
+        name=venue_name,
+        city="Medellin",
+        website="https://idioki.com/",
+        )
+     event['venue'] = venue
+     return event
+
+counter = 0
+
+for component in gcal.walk():
+    event = {}
+    event['scraper'] = scraper
+    event['calendar'] = scraper.calendar
+    event['strSummary'] = f"{(component.get('SUMMARY'))}"
+    event['strDesc'] = component.get('DESCRIPTION')
+    event['strLocation'] = str(component.get('LOCATION'))
+    # startDate = component.get('DTSTART')
+    # startTime = startDate.time()
+    event['dateStart'] = component.get('DTSTART')
+    event['dateStamp'] = component.get('DTSTAMP')
+
+    if event['strSummary'] != 'None':
+        event['details'] = {
+            "description" : event['strDesc'], 
+            "Location" : event['strLocation'],
+        }
+    
+    if event['dateStamp'] != None:
+        event['dateStart'] = event['dateStart'].dt
+        event['dateStart'] = datetime.strptime(str(event['dateStart'])[:-6], '%Y-%m-%d %H:%M:%S')
+        rules = component.get('RRule')
+        if rules['FREQ'][0] == 'WEEKLY':
+            if datetime.today().weekday() != 0:
+                event = splitLocation(event)
+                date = datetime.today().date() - timedelta(days=datetime.today().weekday())
+                date = datetime.combine(date, event['dateStart'].time())
+                days = ["SU", "MO", "TU", "WE", "TH", "FR", "SA"]
+                for day in rules['BYDAY']:
+                    day = days.index(day)
+                    loc_split = event['strLocation'].split(",")
+                    venue, created = Organization.objects.get_or_create(
+                        name=loc_split[0],
+                        city="Medellin",
+                        website="https://idioki.com/",
+                        )
+                    createEvent(day, date, event, scraper, venue, "Ed")
+
+digitools.updateScraper(scraper, item_count_start)
+
+new_time = datetime.now() - timedelta(days=1)
+right_bound_time = datetime.now() + timedelta(days=45)
+events = DSEvent.objects.filter(show_date__lte=new_time)
+events1 = DSEvent.objects.filter(show_date__gte=right_bound_time)
+
+for e in events:
+    e.delete()
+
+for e in events1:
+    e.delete()

events/scrapers/Working/iCal/ical_run.KJHideaway.StPaul.py

@@ -11,6 +11,8 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
 django.setup()
 
 from events.models import Event as DSEvent, Organization, Scraper, Calendar
+import events.digitools as digitools
+
 from dateutil import relativedelta
 td = relativedelta.relativedelta(hours=5)
 
@@ -40,5 +42,5 @@ event_type = "Mu"
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 tz = pytz.timezone("US/Central")
-digitools.getiCalEvents(gcal, scraper)
+digitools.getiCalEvents(gcal, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_run.SocialableCider.mpls.py

@@ -44,5 +44,5 @@ item_count_start = scraper.items
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 tz = pytz.timezone("US/Central")
-digitools.getiCalEvents(gcal, scraper)
+digitools.getiCalEvents(gcal, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_run.bunkers.py

@@ -44,5 +44,5 @@ event_type = "Mu"
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 tz = pytz.timezone("US/Central")
-digitools.getiCalEvents(gcal, scraper)
+digitools.getiCalEvents(gcal, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_run.cfpa.py

@@ -44,5 +44,5 @@ event_type = "Ed"
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 tz = pytz.timezone("US/Central")
-digitools.getiCalEvents(gcal, scraper)
+digitools.getiCalEvents(gcal, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_run.eagles.py

@@ -42,5 +42,5 @@ event_type = "Mu"
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 tz = pytz.timezone("US/Central")
-digitools.getiCalEvents(gcal, scraper)
+digitools.getiCalEvents(gcal, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_run.terminalbar-mpls.py

@@ -46,5 +46,5 @@ event_type = "Mu"
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 tz = pytz.timezone("US/Central")
-digitools.getiCalEvents(gcal, scraper)
+digitools.getiCalEvents(gcal, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)