more scrapers

Working/iCal/col.mde.yoga.py

@@ -0,0 +1,53 @@
+import requests, os, sys
+from icalendar import Calendar as iCalendar, Event
+
+from datetime import datetime, timedelta
+from dateutil import relativedelta
+td = relativedelta.relativedelta(hours=5)
+
+from pprint import pprint as ppr
+import pytz
+
+sys.path.append('/var/www/digisnaxx.ado/scrapers')
+import dtss
+dtss.getReady()
+
+from events.models import Event as DSEvent, Organization, Scraper, Calendar
+import events.digitools as digitools
+
+venue, created = Organization.objects.get_or_create(
+     name="Yoga MDE",
+     city="Medellin",
+     website="https://digisnaxx.com/",
+    )
+
+try:
+    scraper, created = Scraper.objects.get_or_create(
+        name="Yoga MDE",
+        website="https://calendar.google.com/calendar/ical/721a61e06e9a96b7bd30cdb7439aa7791829c316bb8b77237ea93cf41d5396b4%40group.calendar.google.com/public/basic.ics",
+        calendar = Calendar.objects.get(shortcode='mde'),
+        items = 0,
+        new_items = 0,
+        last_ran = datetime.now(),
+        )
+except Exception as e:
+    print(e)
+    scraper = Scraper.objects.get(name=venue.name)
+
+item_count_start = scraper.items
+objIcalData = requests.get(scraper.website)
+gcal = iCalendar.from_ical(objIcalData.text.strip())
+counter = 0
+digitools.getiCalRepeateEvents(gcal, scraper, venue, "Ex", "mde")
+digitools.updateScraper(scraper, item_count_start)
+
+new_time = datetime.now() - timedelta(days=1)
+right_bound_time = datetime.now() + timedelta(days=45)
+events = DSEvent.objects.filter(show_date__lte=new_time)
+events1 = DSEvent.objects.filter(show_date__gte=right_bound_time)
+
+for e in events:
+    e.delete()
+
+for e in events1:
+    e.delete()

Working/iCal/col.mde.yoga.py~

@@ -0,0 +1,55 @@
+import requests, os, sys
+from icalendar import Calendar as iCalendar, Event
+
+from datetime import datetime, timedelta
+from dateutil import relativedelta
+td = relativedelta.relativedelta(hours=5)
+
+from pprint import pprint as ppr
+import pytz
+
+sys.path.append('/var/www/digisnaxx.ado/scrapers')
+import dtss
+dtss.getReady()
+
+from events.models import Event as DSEvent, Organization, Scraper, Calendar
+import events.digitools as digitools
+
+venue, created = Organization.objects.get_or_create(
+     name="idioki",
+     city="Medellin",
+     website="https://idioki.com/",
+    )
+
+try:
+    scraper, created = Scraper.objects.get_or_create(
+        name="Yoga MDE",
+        website="",
+        calendar = Calendar.objects.get(shortcode='mde'),
+        items = 0,
+        new_items = 0,
+        last_ran = datetime.now(),
+        )
+except Exception as e:
+    print(e)
+    scraper = Scraper.objects.get(name=venue.name)
+
+event_type = "Mu"
+item_count_start = scraper.items
+objIcalData = requests.get(scraper.website)
+gcal = iCalendar.from_ical(objIcalData.text)
+
+counter = 0
+digitools.getiCalRepeateEvents(gcal, scraper, venue, "Ed", "mde")
+digitools.updateScraper(scraper, item_count_start)
+
+new_time = datetime.now() - timedelta(days=1)
+right_bound_time = datetime.now() + timedelta(days=45)
+events = DSEvent.objects.filter(show_date__lte=new_time)
+events1 = DSEvent.objects.filter(show_date__gte=right_bound_time)
+
+for e in events:
+    e.delete()
+
+for e in events1:
+    e.delete()

Working/iCal/col.medellin.py

@@ -1,7 +1,7 @@
 import requests, os, sys
 from icalendar import Calendar as iCalendar, Event
 
-from datetime import datetime
+from datetime import datetime, timedelta
 from dateutil import relativedelta
 td = relativedelta.relativedelta(hours=5)
 
@@ -34,13 +34,11 @@ except Exception as e:
     print(e)
     scraper = Scraper.objects.get(name=venue.name)
 
-event_type = "Mu"
 item_count_start = scraper.items
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
-
 counter = 0
-digitools.getiCalRepeateEvents(gcal, scraper, venue, "Ed", cal)
+digitools.getiCalRepeateEvents(gcal, scraper, venue, "Ed", "mde")
 digitools.updateScraper(scraper, item_count_start)
 
 new_time = datetime.now() - timedelta(days=1)
@@ -52,4 +50,4 @@ for e in events:
     e.delete()
 
 for e in events1:
-    e.delete()
+    e.delete()

Working/iCal/df.online_events.py

@@ -1,7 +1,7 @@
 import requests, os, sys
 from icalendar import Calendar as iCalendar, Event
 
-from datetime import datetime
+from datetime import datetime, timedelta
 from dateutil import relativedelta
 td = relativedelta.relativedelta(hours=5)
 
@@ -40,17 +40,18 @@ objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 
 counter = 0
-
-digitools.getiCalRepeateEvents(gcal, scraper, venue, "Ed", cal)
+print("Got Gcal")
+digitools.getiCalRepeateEvents(gcal, scraper, venue, "Ed", "000")
+print("Got EVents")
 digitools.updateScraper(scraper, item_count_start)
+print("Updated")
+# new_time = datetime.now() - timedelta(days=1)
+# right_bound_time = datetime.now() + timedelta(days=45)
+# events = DSEvent.objects.filter(show_date__lte=new_time)
+# events1 = DSEvent.objects.filter(show_date__gte=right_bound_time)
 
-new_time = datetime.now() - timedelta(days=1)
-right_bound_time = datetime.now() + timedelta(days=45)
-events = DSEvent.objects.filter(show_date__lte=new_time)
-events1 = DSEvent.objects.filter(show_date__gte=right_bound_time)
+# for e in events:
+#     e.delete()
 
-for e in events:
-    e.delete()
-
-for e in events1:
-    e.delete()
+# for e in events1:
+#     e.delete()

Working/iCal/il.chicago.citycouncil.py

@@ -21,13 +21,19 @@ venue, created = Organization.objects.get_or_create(
      website="https://chicityclerkelms.chicago.gov/Meetings/",
     )
 
-website="https://calendar.google.com/calendar/ical/chicagolegislativereference%40gmail.com/public/basic.ics",
+website="https://calendar.google.com/calendar/ical/chicagolegislativereference%40gmail.com/public/basic.ics"
+# website="https://calendar.google.com/calendar/ical/chicagolegislativereference%40gmail.com/public/basic.ics"
 scraper,item_count_start,virtcal = digitools.getScraper(venue, website, 'chi')
-
+print("Got Scraper")
 event_type = "Gv"
 item_count_start = scraper.items
-objIcalData = requests.get(scraper.website)
+
+objIcalData = requests.get(website)
+print("Got Obj")
 gcal = iCalendar.from_ical(objIcalData.text)
+
+print("Got Gcal")
 events = digitools.getiCalEvents(gcal, scraper, venue, event_type)
-digitools.buildiCalEvents(events, event_type)
+print("Got Events")
+digitools.buildiCalEvents(events, event_type, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

Working/iCal/mn.mpls.bunkers.py

@@ -32,5 +32,5 @@ objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 
 events = digitools.getiCalEvents(gcal, scraper, venue, event_type)
-digitools.buildiCalEvents(events, event_type)
+digitools.buildiCalEvents(events, event_type, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

Working/iCal/mn.mpls.cafac.py

@@ -28,5 +28,5 @@ item_count_start = scraper.items
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 events = digitools.getiCalEvents(gcal, scraper, venue, event_type)
-digitools.buildiCalEvents(events, event_type)
+digitools.buildiCalEvents(events, event_type, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

Working/iCal/mn.mpls.eagles.py

@@ -28,5 +28,5 @@ item_count_start = scraper.items
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 events = digitools.getiCalEvents(gcal, scraper, venue, event_type)
-digitools.buildiCalEvents(events, event_type)
+digitools.buildiCalEvents(events, event_type, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

Working/iCal/mn.mpls.socialablecider.py

@@ -28,5 +28,5 @@ item_count_start = scraper.items
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 events = digitools.getiCalEvents(gcal, scraper, venue, "Mu")
-digitools.buildiCalEvents(events, event_type)
+digitools.buildiCalEvents(events, event_type, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

Working/iCal/mn.mpls.terminalbar.py

@@ -29,5 +29,5 @@ item_count_start = scraper.items
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 events = digitools.getiCalEvents(gcal, scraper, venue, event_type)
-digitools.buildiCalEvents(events, event_type)
+digitools.buildiCalEvents(events, event_type, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

Working/iCal/mn.stp.cfpa.py

@@ -28,5 +28,5 @@ item_count_start = scraper.items
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.text)
 events = digitools.getiCalEvents(gcal, scraper, venue, event_type)
-digitools.buildiCalEvents(events, event_type)
+digitools.buildiCalEvents(events, event_type, scraper, venue)
 digitools.updateScraper(scraper, item_count_start)

Working/iCal/mn.stp.whitesquirrel.py

@@ -28,6 +28,9 @@ scraper,item_count_start,virtcal = digitools.getScraper(venue, website, 'msp')
 item_count_start = scraper.items
 objIcalData = requests.get(scraper.website)
 gcal = iCalendar.from_ical(objIcalData.content)
+print("Got gcal")
 events = digitools.getiCalEvents(gcal, scraper, venue, event_type)
-digitools.buildiCalEvents(events, event_type)
+print("Got Events")
+digitools.buildiCalEvents(events, event_type, scraper, venue)
+print("Built Events")
 digitools.updateScraper(scraper, item_count_start)