updated to running

events/admin.py

@@ -1,6 +1,10 @@
 from django.contrib import admin
 from .models import *
 
+class CalendarAdmin(admin.ModelAdmin):
+#    prepopulated_fields = {"slug": ("shortname",)}
+    list_display = ("name", "shortcode")
+
 
 class ScraperAdmin(admin.ModelAdmin):
 #    prepopulated_fields = {"slug": ("shortname",)}
@@ -26,8 +30,8 @@ class PromoAdmin(admin.ModelAdmin):
 
 
 # Register your models here.
-admin.site.register(Scraper, ScraperAdmin)
-admin.site.register(Event, EventAdmin)
-admin.site.register(Organization, OrganizationAdmin)
 admin.site.register(Promo, PromoAdmin)
-admin.site.register(Calendar)
+admin.site.register(Event, EventAdmin)
+admin.site.register(Scraper, ScraperAdmin)
+admin.site.register(Calendar, CalendarAdmin)
+admin.site.register(Organization, OrganizationAdmin)

events/digitools.py

@@ -13,24 +13,18 @@ from selenium import webdriver as wd
 
 from events.models import Event as DSEvent, Organization, Promo, Scraper, Calendar
 
-tz = pytz.timezone("US/Central")
+# tz = pytz.timezone("US/Central")
-td = relativedelta.relativedelta(months=1)
+plus_one_month = relativedelta.relativedelta(months=1)
-odt = datetime.now() + td
+odt_next_month = datetime.now() + plus_one_month
 
 
-def getReady():
+# Get Scraper name, item count and online_calendar (virtcal)
-    sys.path.append('../../')
+def getScraper(venue, webite, cal):
-    os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-    django.setup()
-    return
-
-
-def getScraper(venue, cal):
     virtcal = Calendar.objects.get(shortcode='000')
     try:
         scraper, created = Scraper.objects.get_or_create(
             name=venue.name,
-            website=venue.website,
+            website=website,
             calendar = Calendar.objects.get(shortcode=cal),
             items = 0,
             new_items = 0,
@@ -43,9 +37,9 @@ def getScraper(venue, cal):
         scraper.items = len(num_of_events)
         scraper.save()
         print("Scraper: ", scraper)
-        pass
     return scraper, scraper.items, virtcal
 
+# Update item_count of the Scraper at the end of the scrape
 def updateScraper(scraper, item_count_start):
     num_of_events = DSEvent.objects.filter(scraper=scraper)
     scraper.items = len(num_of_events)
@@ -54,19 +48,18 @@ def updateScraper(scraper, item_count_start):
     scraper.save()
     return
 
+# Get site HTML content for XPATH travel
 def getSource(browser, link):
     browser.get(link)
     sleep(5)
     ps = html.fromstring(browser.page_source)
     return ps
 
+# Get Selenium Web Drive, with params for Chrome or Firefox
+# Or in production to run headless
 def getBrowser(run_env):
     if run_env == 'dev':
         print("Chrome is a go!")
-        # chromeOptions = wd.ChromeOptions()
-        # chromeOptions.binary_location = "/Application/Google\ Chrome.app"
-        # chromeDriver = "/opt/homebrew/bin/chromedriver"
-        # br = wd.Chrome(chromeDriver, options=chromeOptions)
         br = wd.Chrome()
         return br
     elif run_env == "def":
@@ -85,19 +78,28 @@ def getBrowser(run_env):
         print("Failed", sys.argv, arg1)
         quit()
 
+# Create Dated URL with zero-padded numbers
 def createBasicURL(site_url):
     month = datetime.now().month
-    next_month = odt.month
+    next_month = odt_next_month.month
     year = datetime.now().year
-    links = [
+    if next_month == 1:
+        next_year = year+1
+        links = [
         site_url + str(month) + "/" + str(year),
-        site_url + str(next_month) + "/" + str(year)
+        site_url + str(next_month) + "/" + str(next_year)
     ]
+    else: 
+        links = [
+            site_url + str(month) + "/" + str(year),
+            site_url + str(next_month) + "/" + str(year)
+        ]
     return links
 
+# Create Dated URL without zero-padded numbers
 def createURLNoZero(site_url):
     month = datetime.now().month
-    next_month = odt.month
+    next_month = odt_next_month.month
     year = datetime.now().year
     links = [
         site_url + str(year) + "/" + str(month),
@@ -108,13 +110,14 @@ def createURLNoZero(site_url):
         links.append(site_url + str(year) + "/" + str(next_month))
     return links
 
+# Create Dated URL Link with zero-padding
 def createURL(site_url):
     month = datetime.now().month
     if month < 10:
         month = "0" + str(month)
     else:
         month = str(month)
-    next_month = odt.month
+    next_month = odt_next_month.month
     if next_month < 10:
         next_month = "0" + str(next_month)
     else:
@@ -129,6 +132,7 @@ def createURL(site_url):
         links.append(site_url + str(year) + "/" + next_month)
     return links
 
+# Create Dated URL with dashes
 def createDashURL(site_url):
     month = datetime.now().month
     if month < 10:
@@ -148,16 +152,16 @@ def createDashURL(site_url):
     print(links)
     return links
 
+# Add Calendar to Event Object (maybe extraneous)
 def add_calendar(event, calendar):
-    print("\nNEW EVENT DIGITOOLS: ", event, type(event), calendar)
     if type(event) is tuple:
-        print("STS: ", event)
         event = event[0]
     cal = Calendar.objects.get(shortcode=calendar)
     event.calendar.add(cal)
     event.save()
     return event
 
+# Add Calendars to Event Object ??
 def add_calendars(event, data):
     if type(data['calendars']) is not list:
         event.calendar.add(data['calendars'])
@@ -167,6 +171,7 @@ def add_calendars(event, data):
         event.save()
     return event
 
+# Create Basic DigiSnaxx Event
 def createBasicEvent(event, event_type, venue):
     new_event, created = DSEvent.objects.update_or_create(
         event_type = event_type,
@@ -178,8 +183,10 @@ def createBasicEvent(event, event_type, venue):
         venue = venue
     )
     new_event = add_calendars(new_event, event)
+    print("\n+new event+\n")
     return new_event, created
 
+# Create iCal Event
 def createBasiciCalEvent(event, event_type, venue):
     new_event, created = DSEvent.objects.update_or_create(
         event_type = event_type,
@@ -191,8 +198,11 @@ def createBasiciCalEvent(event, event_type, venue):
         venue = venue
     )
     new_event = add_calendars(new_event, event)
+    print("\n+new event+\n")
     return new_event, created
 
+# Create Detailed Event with Details & Guests
+# Details in JSON Format
 def createDetailedEvent(event, event_type, venue, scraper):
     new_event, created = DSEvent.objects.update_or_create(
         event_type = event_type,
@@ -206,31 +216,10 @@ def createDetailedEvent(event, event_type, venue, scraper):
         venue = venue
     )
     new_event = add_calendars(new_event, event)
+    print("\n+new event+\n")
     return new_event, created
 
-def createBasicArticle(article, event_type, organization):
+# Create iCal event from DF_Online & Medellin
-    new_article, created = Promo.objects.update_or_create(
-        promo_type = 'Ja',
-        title = article['title'],
-        target_link = article['link'],
-        published = True,
-        organization = organization
-    )
-    return new_article, created
-
-def createIcalEvent(event, scraper, venue, event_type):
-    new_date = event['dateStart']
-    new_event = {}
-    new_event['scraper'] = scraper
-    new_event['calendar'] = scraper.calendar
-    new_event['title'] = event['strSummary'],
-    new_event['date'] = str(new_date)[:-6],
-    new_event['dateStamp'] = str(new_date)[:-6],
-    new_event['link'] = venue.website
-    print("NEW iCal Event")
-    ppr(new_event)
-    createBasiciCalEvent(new_event, event_type, venue)
-
 def createCleanIcalEvent(event, scraper, venue, event_type):
     new_date = event['dateStart']
     new_event = {}
@@ -242,6 +231,7 @@ def createCleanIcalEvent(event, scraper, venue, event_type):
     new_event['link'] = venue.website
     createBasiciCalEvent(new_event, event_type, venue)
 
+# Get events from iCal
 def getiCalEvents(gcal, scraper, venue, event_type):
     events = []
     for component in gcal.walk():
@@ -259,13 +249,8 @@ def getiCalEvents(gcal, scraper, venue, event_type):
             try:
                 event['dateStart'] = event['dateStart'].dt
             except Exception as e:
-                event['dateStart'] = event['dateStart'].dt
+                print("what? ", e)
         
-        event['dateEnd'] = (component.get('DTEND'))
-        if event['dateEnd'] is not None:
-                event['dateEnd'] = event['dateEnd'].dt
-        else:
-            event['dateEnd'] = event['dateStart']
         if event['strSummary'] != 'None':
             event['details'] = {
                 "description" : event['strDesc'], 
@@ -274,55 +259,110 @@ def getiCalEvents(gcal, scraper, venue, event_type):
         events.append(event)
     return events
 
-def returniCalEvents(gcal, scraper, venue, event_type):
+# Build iCal Events and Send to Create
-    events = []
+def buildiCalEvents(events, event_type, scraper):
+    for event in events:
+        e = {}
+        e['calendars'] = event['calendars']
+        try:
+            e['dateStamp'] = event['dateStart'][0]
+        except:
+            e['dateStamp'] = event['dateStart']
+        e['title'] = event['strSummary']
+        e['scraper'] = scraper 
+        e['link'] = venue.website
+        try:
+            createBasicEvent(e, event_type, venue)
+            scraper.items+=1
+        except Exception as e:
+            print("Error: ", e)
+    scraper.save()
+    return
+
+def getMDEVenue(venue):
+    if venue.name == "DANCEFREE":
+        venue.website = "https://www.instagram.com/dancefreeco"
+    if venue.name == "Vintrash":
+        venue.website = "https://www.instagram.com/vintrashbar"
+    if venue.name == "The Wandering Paisa":
+        venue.website = "https://wanderingpaisahostel.com"
+    if venue.name == "Dulce Posion":
+        venue.website = "https://www.instagram.com/dulceposionr"
+    if venue.name == "Blood Dance Company":
+        venue.website = "https://www.instagram.com/blooddancecompany"
+    if venue.name == "OLSA Certified Spanish School":
+        venue.website = "https://www.olsafoundation.org/"
+    if event['strSummary'] == "Merli Rooftop Language Exchange":
+        venue.website = "https://calendar.google.com/calendar/embed?src=46ae0446724b1b3ee83cbd7dbc0db6a235bf97509ad860ca91eada3c267b5e41%40group.calendar.google.com&ctz=America%2FBogota"
+    if "Concious Warrior" in event['strSummary']:
+        venue.website = "https://www.consciouscolombia.com/"
+    venue.save()
+    return
+
+# Get iCal events for Medellin & OnlineEvents
+def getiCalRepeateEvents(gcal, scraper, venue, event_type, cal):
     for component in gcal.walk():
         event = {}
         event['scraper'] = scraper
         event['calendars'] = [scraper.calendar]
         event['strSummary'] = f"{(component.get('SUMMARY'))}"
         event['strDesc'] = component.get('DESCRIPTION')
-        event['strLocation'] = component.get('LOCATION')
+        event['strLocation'] = str(component.get('LOCATION'))
         event['dateStart'] = component.get('DTSTART')
         event['dateStamp'] = component.get('DTSTAMP')
-        if event['dateStamp'] is not None:
-            event['dateStamp'] = event['dateStamp'].dt
-        if event['dateStart'] is not None:
-            try:
-                event['dateStart'] = event['dateStart'].dt
-            except Exception as e:
-                event['dateStart'] = event['dateStart'].dt
-        
-        event['dateEnd'] = (component.get('DTEND'))
-        if event['dateEnd'] is not None:
-                event['dateEnd'] = event['dateEnd'].dt
-        else:
-            event['dateEnd'] = event['dateStart']
         if event['strSummary'] != 'None':
             event['details'] = {
                 "description" : event['strDesc'], 
                 "Location" : event['strLocation'],
             }
-        events.append(event)
+        if event['dateStamp'] != None:
-        now_now = datetime.today().date()
+            event['dateStart'] = event['dateStart'].dt
-        try:
+            event['dateStart'] = datetime.strptime(str(event['dateStart'])[:-6], '%Y-%m-%d %H:%M:%S')
-            print("1Event: ", event['dateStart'])
+            rules = component.get('RRule')
-            if event['dateStart'] > now_now:
-                print("Check Check: ", event['dateStart'])
-                createIcalEvent(event, scraper, venue, event_type)
-            else:
-                print("WHAT?")
-                ppr(event)
-        except Exception as e:
             try:
-                event['dateStart'] = event['dateStart'].date()
+                if rules['FREQ'][0] == 'WEEKLY':
-                print("2Event: ", event['dateStart'])
+                    if datetime.today().weekday() != 0:
-                if event['dateStart'] > now_now:
+                        event = digitools.splitLocation(event, "Medellin")
-                    print("Check Check: ", event['dateStart'])
+                        date = datetime.today().date() - timedelta(days=datetime.today().weekday())
-                    createIcalEvent(event, scraper, venue, event_type)
+                        date = datetime.combine(date, event['dateStart'].time())
-                else:
+                        days = ["SU", "MO", "TU", "WE", "TH", "FR", "SA"]
-                    print("WHAT?")
+                        for day in rules['BYDAY']:
-                    ppr(event)
+                            day = days.index(day)
+                            if cal == 'mde':
+                                getVenue(event['venue'])
+                            iCalEventRepeatFilter(day, date, event, scraper, event['venue'], "Ed")
+                            
             except Exception as e:
-                print("The Error: ", e)
+                print("Error: ", e, "\n\n\n\n")
-                pass
+                pass
+
+def iCalEventRepeatFilterteEvent(day, date, event, scraper, venue, event_type):
+    days = [day-1, day+6, day+13]
+    for day in days:
+        event['dateStamp'] = date + timedelta(days=day)
+        event['dateStart'] = event['dateStamp']
+        digitools.createCleanIcalEvent(event, scraper, venue, event_type)
+    return
+
+def splitLocation(event, **kwargs):
+     loc_split = event['strLocation'].split(',')
+     venue_name = loc_split[0]
+     venue, created = Organization.objects.get_or_create(
+        name=venue_name,
+        )
+     event['venue'] = venue
+     if city:
+        venue.city = kwargs['city']
+        venue.save()
+     return event
+
+# ARCHIVED Methods
+def createBasicArticle(article, event_type, organization):
+    new_article, created = Promo.objects.update_or_create(
+        promo_type = 'Ja',
+        title = article['title'],
+        target_link = article['link'],
+        published = True,
+        organization = organization
+    )
+    return new_article, created

events/models.py

@@ -10,6 +10,10 @@ class Calendar(models.Model):
     shortcode = models.CharField(max_length=3, unique=True)
     desc = models.TextField(blank=True, null=True)
 
+    class Meta:
+        verbose_name_plural = "Calendars"
+        ordering = ['name',]
+
     def __unicode__(self):
         return "%s" % self.shortcode
 

events/scrapers/Templates/TemplateScraper.py

@@ -1,90 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-from events.digitools import getBrowser, createURL, createBasicEvent, getSource
-
-count = 0
-
-venue, created = Organization.objects.get_or_create(
-     name="Venue Name",
-     city="Minneapolis",
-     website="Event Website",
-    )
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name=venue.name,
-        website=venue.website,
-        items = 0,
-        last_ran = datetime.now(),
-    )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-    print("Scraper: ", scraper)
-
-event_type = ""
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%b %d %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/li[@class="event"]')
-    for c in contents:
-        try:
-            event = {}
-            day = c.xpath('.//*/span[@class="day"]/text()')[0]
-            month = c.xpath('.//*/span[@class="mth"]/text()')[0]
-            year = datetime.now().year
-            if month == "Jan":
-                year = int(year) + 1
-            event['calendar'] = venue.calendar
-            event['title'] = c.xpath('.//*/span[@class="event_title"]/a/text()')[0]
-            event['date'] = [month, day, str(year),  c.xpath('.//*/span[@class="event_time"]/text()')[0].strip()]
-            event['date'] = " ".join(event['date'])
-            event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
-            event['link'] = c.xpath('.//*/span[@class="event_title"]/a/@href')[0]
-            print("Event Dict Created")
-            createBasicEvent(event, event_type, venue)
-            scraper.items+=1
-        except Exception as e:
-            print(e)
-            ppr(event)
-            print("\n\n+++\n\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-ps = getSource(br, venue.website)
-get_events(ps, "Ed")
-sleep(3)
-
-
-scraper.save()
-
-# Get Event Page Link(s)
-# links = createURL("https://acmecomedycompany.com/the-club/calendar/")  
-
-# for link in links:
-#     ps = getSource(br, link)
-#     get_events(ps, "Ed")
-#     sleep(3)
-
-br.close()

events/scrapers/Templates/TemplateScraper.py.bak

@@ -1,67 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization
-from events.digitools import getBrowser, createURL, createBasicEvent, getSource
-
-venue, created = Organization.objects.get_or_create(
-     name="Venue Name",
-     city="Minneapolis",
-     website="Event Website",
-    )
-
-event_type = ""
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%b %d %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/li[@class="event"]')
-    for c in contents:
-        try:
-            event = {}
-            day = c.xpath('.//*/span[@class="day"]/text()')[0]
-            month = c.xpath('.//*/span[@class="mth"]/text()')[0]
-            year = datetime.now().year
-            if month == "Jan":
-                year = int(year) + 1
-            event['title'] = c.xpath('.//*/span[@class="event_title"]/a/text()')[0]
-            event['date'] = [month, day, str(year),  c.xpath('.//*/span[@class="event_time"]/text()')[0].strip()]
-            event['date'] = " ".join(event['date'])
-            event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
-            event['link'] = c.xpath('.//*/span[@class="event_title"]/a/@href')[0]
-            print("Event Dict Created")
-            createBasicEvent(event, event_type, venue)
-        except Exception as e:
-            print(e)
-            ppr(event)
-            print("\n\n+++\n\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-# Get Event Page Link(s)
-links = createURL("https://acmecomedycompany.com/the-club/calendar/")  
-
-for link in links:
-    ps = getSource(br, link)
-    get_events(ps, "Ed")
-    sleep(3)
-
-br.close()

events/scrapers/Templates/ical_run.py.template

@@ -1,84 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from events.models import Event as DSEvent, Organization
-
-venue, created = Organization.objects.get_or_create(
-     name="location",
-     city="Minneapolis",
-     website="",
-    )
-
-event_type = ""
-
-calendar_url = [
-    'https://calendar.google.com/calendar/ical/js94epu90r2et31aopons1ifm8%40group.calendar.google.com/public/basic.ics',
-    'https://calendar.google.com/calendar/ical/6rpooudjg01vc8bjek1snu2ro0%40group.calendar.google.com/public/basic.ics',
-    'https://calendar.google.com/calendar/ical/teflgutelllvla7r6vfcmjdjjo%40group.calendar.google.com/public/basic.ics'
-]
-
-objIcalData = requests.get(calendar_url[1])
-
-gcal = iCalendar.from_ical(objIcalData.text)
-
-cfpa_events = []
-tz = pytz.timezone("US/Central")
-
-for component in gcal.walk():
-    event = {}
-    event['strSummary'] = f"{(component.get('SUMMARY'))}"
-    event['strDesc'] = component.get('DESCRIPTION')
-    event['strLocation'] = component.get('LOCATION')
-    event['dateStart'] = component.get('DTSTART')
-    event['dateStamp'] = component.get('DTSTAMP')
-    if event['dateStamp'] is not None:
-        event['dateStamp'] = event['dateStamp'].dt
-    if event['dateStart'] is not None:
-        try:
-            event['dateStart'] = event['dateStart'].dt.astimezone(pytz.utc)
-        except Exception as e:
-             event['dateStart'] = event['dateStart'].dt
-        
-    event['dateEnd'] = (component.get('DTEND'))
-    if event['dateEnd'] is not None:
-            event['dateEnd'] = event['dateEnd'].dt
-    else:
-        event['dateEnd'] = event['dateStart']
-    if event['strSummary'] != 'None':
-        event['details'] = {
-             "description" : event['strDesc'], 
-             "DateTime" : event['dateStart'],
-             "Location" : event['strLocation'],
-        }
-        cfpa_events.append(event)
-        new_event = DSEvent.objects.update_or_create(
-            event_type = event_type,
-            show_title = event['strSummary'],
-            show_link = event['link'],
-            show_date = event['dateStart'],
-            show_day = event['dateStart'].date(),
-            more_details = event["details"],
-            venue = venue
-        )
-
-
-
-# {'dateEnd': datetime.datetime(2022, 10, 22, 18, 30, tzinfo=<UTC>),
-#  'dateStamp': datetime.datetime(2023, 3, 23, 1, 57, 45, tzinfo=<UTC>),
-#  'dateStart': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
-#  'details': {'DateTime': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
-#              'Location': vText('b'''),
-#              'description': None},
-#  'strDesc': None,
-#  'strLocation': vText('b'''),
-#  'strSummary': 'Nia Class with Beth Giles'}
-
-

events/scrapers/Templates/ical_template.py

@@ -1,84 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from events.models import Event as DSEvent, Organization
-
-venue, created = Organization.objects.get_or_create(
-     name="location",
-     city="Minneapolis",
-     website="",
-    )
-
-event_type = ""
-
-calendar_url = [
-    'https://calendar.google.com/calendar/ical/js94epu90r2et31aopons1ifm8%40group.calendar.google.com/public/basic.ics',
-    'https://calendar.google.com/calendar/ical/6rpooudjg01vc8bjek1snu2ro0%40group.calendar.google.com/public/basic.ics',
-    'https://calendar.google.com/calendar/ical/teflgutelllvla7r6vfcmjdjjo%40group.calendar.google.com/public/basic.ics'
-]
-
-objIcalData = requests.get(calendar_url[1])
-
-gcal = iCalendar.from_ical(objIcalData.text)
-
-cfpa_events = []
-tz = pytz.timezone("US/Central")
-
-for component in gcal.walk():
-    event = {}
-    event['strSummary'] = f"{(component.get('SUMMARY'))}"
-    event['strDesc'] = component.get('DESCRIPTION')
-    event['strLocation'] = component.get('LOCATION')
-    event['dateStart'] = component.get('DTSTART')
-    event['dateStamp'] = component.get('DTSTAMP')
-    if event['dateStamp'] is not None:
-        event['dateStamp'] = event['dateStamp'].dt
-    if event['dateStart'] is not None:
-        try:
-            event['dateStart'] = event['dateStart'].dt.astimezone(pytz.utc)
-        except Exception as e:
-             event['dateStart'] = event['dateStart'].dt
-        
-    event['dateEnd'] = (component.get('DTEND'))
-    if event['dateEnd'] is not None:
-            event['dateEnd'] = event['dateEnd'].dt
-    else:
-        event['dateEnd'] = event['dateStart']
-    if event['strSummary'] != 'None':
-        event['details'] = {
-             "description" : event['strDesc'], 
-             "DateTime" : event['dateStart'],
-             "Location" : event['strLocation'],
-        }
-        cfpa_events.append(event)
-        new_event = DSEvent.objects.update_or_create(
-            event_type = event_type,
-            show_title = event['strSummary'],
-            show_link = event['link'],
-            show_date = event['dateStart'],
-            show_day = event['dateStart'].date(),
-            more_details = event["details"],
-            venue = venue
-        )
-
-
-
-# {'dateEnd': datetime.datetime(2022, 10, 22, 18, 30, tzinfo=<UTC>),
-#  'dateStamp': datetime.datetime(2023, 3, 23, 1, 57, 45, tzinfo=<UTC>),
-#  'dateStart': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
-#  'details': {'DateTime': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
-#              'Location': vText('b'''),
-#              'description': None},
-#  'strDesc': None,
-#  'strLocation': vText('b'''),
-#  'strSummary': 'Nia Class with Beth Giles'}
-
-

events/scrapers/Working/cals/MplStpMag.mn.py

@@ -1,102 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization
-from events.digitools import getBrowser, createURL, createBasicEvent, getSource
-
-venue, created = Organization.objects.get_or_create(
-     name="Mpls Stp Mag",
-     city="Minneapolis",
-     website="https://calendar.mspmag.com/calendars/all-events/",
-    )
-
-event_type = ""
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-td = relativedelta.relativedelta(days=1)
-fortnight = relativedelta.relativedelta(days=14)
-odt = datetime.now() + fortnight
-
-# DATETIME_FORMAT = '%b %d %Y %I:%M %p'
-DATETIME_FORMAT = '%A, %B %d %Y %I:%M%p'
-DATETIME_FORMAT_ALT = '%A, %B %d %Y'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/div[@class="css-card js-card day-card type-smad expandable"]')
-    for c in contents:
-        try:
-            event = {}
-            event['calendar'] = venue.calendar
-            event_block = c.xpath('.//*/li[@class="card-listings-item event-element"]')
-            date = c.xpath('.//div[@class="day-card__header day-card__header--daily"]/text()')[0].replace("\n", "").strip()
-            if date == "Today":
-                date = datetime.today()
-            elif date == "Tomorrow":
-                date = datetime.today() + td
-            # month = c.xpath('.//*/span[@class="mth"]/text()')[0]
-            year = datetime.now().year
-            # if month == "Jan":
-            #     year = int(year) + 1
-            dateTime = datetime.strptime(date + " " + str(year), DATETIME_FORMAT_ALT)
-            if dateTime > odt:
-                print("DATE TIME ", dateTime)
-                break
-            for ev in event_block:
-                time = ev.xpath('.//*/span[@class="card-listing-item-time"]/text()')[0].replace("@", "").strip()
-                if time == "All day":
-                    time = "12:00pm"
-                    event['title'] = ev.xpath('.//*/div[@class="card-listing-item-title"]/text()')[0] + " (Check link for times.)"
-                elif "-" in time:
-                    time = time.split("-")[0]
-                    event['title'] = ev.xpath('.//*/div[@class="card-listing-item-title"]/text()')[0]
-                else:
-                    event['title'] = ev.xpath('.//*/div[@class="card-listing-item-title"]/text()')[0]
-                
-                event['location'] = ev.xpath('.//*/span[@class="card-listing-item-location"]/text()')[0]
-                if event['location'] == '7th St. Entry':
-                    event['location'] = '7th St Entry'
-                elif event['location'] == '7th Street Entry':
-                    event['location'] = '7th St Entry'
-                elif event['location'] == 'Amsterdam Bar and Hall':
-                    event['location'] = 'Amsterdam Bar & Hall'
-                new_venue, created = Organization.objects.get_or_create(name=event['location'])
-                print("V: ", new_venue, created)
-
-                event['dateTime'] = date + " " + str(year) + " " + time
-                event['link'] = venue.website + c.xpath('.//@data-event')[0]
-                event['dateStamp'] = datetime.strptime(event['dateTime'], DATETIME_FORMAT)
-
-                
-
-                createBasicEvent(event, event_type, new_venue)
-        except Exception as e:
-            print(e)
-            ppr(event)
-            print("\n\n+++\n\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-# Get Event Page Link(s)
-# links = createURL("https://acmecomedycompany.com/the-club/calendar/")  
-
-ps = getSource(br, venue.website)
-get_events(ps, "Ed")
-sleep(3)
-
-br.close()

events/scrapers/Working/cals/minnestar.py

@@ -1,105 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-from events.digitools import getBrowser, createURL, createBasicEvent, getSource
-
-count = 0
-
-venue, created = Organization.objects.get_or_create(
-     name="Minnestar",
-     city="Minneapolis",
-     website="https://minnestar.org/community/calendar",
-    )
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name=venue.name,
-        website=venue.website,
-        items = 0,
-        last_ran = datetime.now(),
-    )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-    print("Scraper: ", scraper)
-
-event_type = ""
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%B %d, %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%B %d %Y'
-
-def get_events(ps, event_type):
-    links = ps.xpath('.//*/div[@id="community-calendar-list-view-container"]/a/@href')
-    ppr(links)
-    for l in links:
-        pse = getSource(br, l)
-        sleep(1)
-        event = {}
-        event['calendar'] = venue.calendar
-        event['link'] = l
-        try:
-            starttime = pse.xpath('.//*/time/text()')[0]
-            endtime = pse.xpath('.//*/time/@datetime')[1]
-            event['dateStamp'] = datetime.strptime(starttime, DATETIME_FORMAT) 
-            event['title'] = pse.xpath('.//*/h1[@class="heading-2"]/text()')[0]
-            # event['detail-headers'] = pse.xpath('.//*/ul[@class="eo-event-meta"]/li/strong/text()')
-            # event['details'] = pse.xpath('.//*/ul[@class="eo-event-meta"]/li/text()')
-
-        except:
-            try:
-                event['title'] = pse.xpath('.//*/h1[@class="heading-2"]/text()')[0]
-                starttime = pse.xpath('.//*/time/text()')[0]
-                event['dateStamp'] = datetime.strptime(starttime, DATETIME_FORMAT)
-            except Exception as e:
-                try:
-                    print(e)
-                    print('failed event: ', event)
-                    starttime = pse.xpath('.//*/time/text()')[0]
-                    event['dateStamp'] = datetime.strptime(starttime + ' 2025', DATETIME_FORMAT_2)
-                except Exception as e:
-                    print(e)
-                    print("failed event: ", event)
-        ppr(event)
-        try:
-            createBasicEvent(event, "Ot", venue)
-            scraper.items+=1
-        except Exception as e:
-            print(e)
-            print('failed to create: ', event)
-        
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-ps = getSource(br, venue.website)
-get_events(ps, "Ot")
-sleep(3)
-
-scraper.save()
-
-# Get Event Page Link(s)
-# links = createURL("https://acmecomedycompany.com/the-club/calendar/")  
-
-# for link in links:
-#     ps = getSource(br, link)
-#     get_events(ps, "Ed")
-#     sleep(3)
-
-br.close()

events/scrapers/Working/govt/MNLeg.py

@@ -1,167 +0,0 @@
-# Install Chromedriver and Quarantine
-# xattr -d com.apple.quarantine <name-of-executable>
-
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-from selenium import webdriver as wd
-
-from xvfbwrapper import Xvfb
-from lxml import html
-import pytz
-
-from events.models import Event, Organization, Scraper, Calendar
-from events.digitools import getBrowser, createURL, createBasicEvent, getSource, add_calendar
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="MN Legislature",
-        website="https://www.leg.mn.gov/cal?type=all",
-        calendar = Calendar.objects.get(shortcode='msp'),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-    )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name="MN Legislature")
-    print("Scraper: ", scraper)
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p'
-
-# Set initial variables for City, etc
-calendar_url = 'https://www.leg.mn.gov/cal?type=all'
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-br.get(calendar_url)
-sleep(10)
-ps = html.fromstring(br.page_source)
-
-commEvents = ps.xpath('.//*/div[@class="card border-dark comm_item cal_item ml-lg-3"]')
-senateEvents = ps.xpath('.//*/div[@class="card border-dark senate_item cal_item ml-lg-3"]')
-houseEvents = ps.xpath('.//*/div[@class="card border-dark house_item cal_item ml-lg-3"]')
-meetings = []
-
-for hE in houseEvents:
-    details = {}
-    dateTime = hE.xpath('.//*/b/text()')[0]
-    try:
-        title = hE.xpath('.//*/h3/a/text()')[0]
-    except:
-        title = hE.xpath('.//*/h3/text()')[0]
-    try:
-        link = "https://www.leg.mn.gov/" + hE.xpath('.//*/div[@class="float-right text-center mr-2 d-print-none"]/a/@href')[0]
-    except:
-        link = hE.xpath('.//*/h3/a/@href')[0]
-    details['location'] = hE.xpath('.//*/div[@class=""]/text()')[0]
-    # print(dateTime, title, link, details['location'])
-    venue, created = Organization.objects.get_or_create(name="MN House", city="St. Paul")
-    new_event, created = Event.objects.update_or_create(
-        event_type = 'Gv',
-        show_title = title,
-        show_link = link,
-        show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
-        show_day = datetime.strptime(dateTime, DATETIME_FORMAT).date(),
-        more_details = details['location'],
-        venue = venue,
-        scraper = scraper
-    )
-
-    if type(new_event) is tuple:
-        print("STS: ", new_event)
-        add_calendar(new_event[0], 'msp')
-    else:
-        add_calendar(new_event, 'msp')
-    scraper.items+=1
-
-
-for sE in senateEvents:
-    details = {}
-    dateTime = sE.xpath('.//*/b/text()')[0]
-    try:
-        title = sE.xpath('.//*/h3/a/text()')[0]
-    except:
-        title = sE.xpath('.//*/h3/text()')[0]
-    try:
-        link = "https://www.leg.mn.gov/" + sE.xpath('.//*/div[@class="float-right text-center mr-2"]/a/@href')[0]
-    except:
-        link = sE.xpath('.//*/h3/a/@href')[0]
-    location_list = sE.xpath('.//*/text()')
-    if 'Location: ' in location_list:
-        iN = location_list.index("Location: ")
-        details['location'] = location_list[iN + 1]
-    elif 'Senate Floor Session' in location_list:
-        details['location'] = 'Senate Floor Session'
-    venue, created = Organization.objects.get_or_create(name="MN Senate", city="St. Paul")
-    new_event = Event.objects.update_or_create(
-        event_type = 'Gv',
-        show_title = title,
-        show_link = link,
-        show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
-        show_day = datetime.strptime(dateTime, DATETIME_FORMAT).date(),
-        more_details = details['location'],
-        venue = venue,
-        scraper = scraper
-    )
-    if type(new_event) is tuple:
-        print("STS: ", new_event)
-        add_calendar(new_event[0], 'msp')
-    else:
-        add_calendar(new_event, 'msp')
-    scraper.items+=1
-
-for cE in commEvents:
-    details = {}
-    dateTime = cE.xpath('.//*/b/text()')[0]
-    try:
-        title = cE.xpath('.//*/h3/a/text()')[0]
-    except:
-        title = cE.xpath('.//*/h3/text()')[0]
-    try:
-        link = "https://www.leg.mn.gov/" + cE.xpath('.//*/div[@class="float-right text-center mr-2"]/a/@href')[0]
-    except:
-        link = cE.xpath('.//*/h3/a/@href')[0]
-    location_list = cE.xpath('.//*/text()')
-    if 'Room: ' in location_list:
-        iN = location_list.index("Room: ")
-        details['location'] = location_list[iN + 1]
-    # print(dateTime, title, link, details['location'])
-    venue, created = Organization.objects.get_or_create(name="MN Legislature", city="St. Paul")
-    new_event = Event.objects.update_or_create(
-        event_type = 'Gv',
-        show_title = title,
-        show_link = link,
-        show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
-        show_day = datetime.strptime(dateTime, DATETIME_FORMAT).date(),
-        more_details = details['location'],
-        venue = venue,
-        scraper = scraper
-    )
-    if type(new_event) is tuple:
-        print("STS: ", new_event)
-        add_calendar(new_event[0], 'msp')
-    else:
-        add_calendar(new_event, 'msp')
-    scraper.items+=1
-
-
-br.close()
-scraper.save()
-
-# br.find_element_by_class_name('fc-btn_allCalendars-button').click()

events/scrapers/Working/govt/MplsCityCouncil.py

@@ -1,122 +0,0 @@
-import re, os, sys
-from datetime import datetime, timedelta
-import requests
-import json
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event, Organization, Scraper, Calendar
-
-from time import sleep
-from pprint import pprint as ppr
-from selenium import webdriver as wd
-from selenium.webdriver.support.ui import Select
-from selenium.webdriver.common.by import By
-
-from xvfbwrapper import Xvfb
-from lxml import html
-import pytz
-from events.digitools import getBrowser, createURL, createBasicEvent, getSource, add_calendar
-
-import events.digitools as digitools
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="Mpls City Council",
-        website="https://lims.minneapolismn.gov/Calendar/citycouncil/upcoming",
-        calendar = Calendar.objects.get(shortcode='msp'),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-    )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name="Mpls City Council")
-    print("Scraper: ", scraper)
-
-DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
-tz = pytz.timezone("US/Central")
-td = timedelta(days=2)
-odt = datetime.now() - td
-
-month = odt.strftime("%b")
-day = int(datetime.now().day)
-
-if day == 1:
-    day = 30
-
-year = int(datetime.now().year)
-
-cal_url = 'https://lims.minneapolismn.gov/Calendar/GetCalenderList?fromDate={}%20{},%20{}&toDate=null&meetingType=0&committeeId=null&pageCount=100&offsetStart=0&abbreviation=undefined&keywords=&sortOrder=1'.format(month, day, year)
-
-# print("URL: ", cal_url)
-
-# cal_url = 'https://lims.minneapolismn.gov/Calendar/GetCalenderList?fromDate=Nov%2015,%202025&toDate=null&meetingType=0&committeeId=null&pageCount=50&offsetStart=0&abbreviation=undefined&keywords=&sortOrder=1'
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-br.get(cal_url)
-sleep(2)
-ps = html.fromstring(br.page_source)
-events = ps.xpath('.//body/pre/text()')[0]
-new_events = json.loads(events)
-# ppr(dict(new_events[0]))
-event_list = []
-
-for event in new_events:
-    e = {}
-    e['title'] = event['CommitteeName']
-    e['link'] = scraper.website
-    e['dateStamp'] = datetime.strptime(event['MeetingTime'], DATETIME_FORMAT)
-    e['agendaStatus'] = event['AgendaStatus']
-    e['address'] = event['Address']
-    e['description'] = event['Description']
-    e['scraper'] = scraper
-    e['calendars'] = [scraper.calendar]
-    try:
-        # print(event['Location'].split(",")[1])
-        loc = event['Location'].split(",")[1]
-        if "City Hall" in loc:
-            e['location'] = "Mpls City Hall"
-            venue = Organization.objects.get(
-                name="Mpls City Hall",
-            )
-            digitools.createBasicEvent(e, 'Gv', venue)
-        elif "Public Service Building" in loc:
-            e['location'] = "Mpls Public Service Building"
-            venue = Organization.objects.get(
-                name="Mpls Public Service Building",
-            )
-            digitools.createBasicEvent(e, 'Gv', venue)
-    except:
-        # print(event['Location'].split("-")[0])
-        e['location'] = event['Location'].split("-")[0].strip()
-        try:
-            venue, created = Organization.objects.get_or_create(
-                    name=e['location'] ,
-                    city="Minneapolis",
-                    website=scraper.website,
-                    is_venue=False,
-                    address_complete=e['address']
-                )
-        except:
-            venue = Organization.objects.get(
-                    name=e['location'] ,
-                    city="Minneapolis",
-                )
-        digitools.createBasicEvent(e, 'Gv', venue)
-    event_list.append(e)
-    
-
-br.close()
-
-scraper.save()
-# br.find_element_by_class_name('fc-btn_allCalendars-button').click()

events/scrapers/Working/govt/StPaulCityCouncil.py

@@ -1,82 +0,0 @@
-import re, os, sys
-from datetime import datetime
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event, Organization, Scraper, Calendar
-
-from time import sleep
-from pprint import pprint as ppr
-from selenium import webdriver as wd
-from selenium.webdriver.common.by import By
-
-from xvfbwrapper import Xvfb
-from lxml import html
-import pytz
-from events.digitools import getBrowser, createURL, createBasicEvent, getSource, add_calendar
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="St Paul City Council",
-        website="https://www.stpaul.gov/calendar",
-        calendar = Calendar.objects.get(shortcode='msp'),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-    )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name="St Paul City Council")
-    print("Scraper: ", scraper)
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%B %d, %Y at %I:%M %p'
-
-calendar_url = 'https://www.stpaul.gov/calendar'
-city_site = "https://www.stpaul.gov"
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-br.get(calendar_url)
-sleep(3)
-
-
-def getEvents(br):
-    ps = html.fromstring(br.page_source)
-    eventBlocks = ps.xpath('.//*/div[@class="calendar__item views-row"]')
-
-    for eB in eventBlocks:
-        title = eB.xpath('.//div/h3[@class="field-content calendar__title"]/text()')[0]
-        link = city_site + eB.xpath('.//div/span[@class="field-content calendar__link"]/a/@href')[0]
-        dateTime = eB.xpath('.//div[@class="views-field views-field-field-calendar-date-value"]/span/text()')[0]
-        print(dateTime, title, link)
-        print('\n\n++++\n\n')
-        venue, created = Organization.objects.get_or_create(name="Somewhere in St Paul", city="St. Paul")
-        new_event = Event.objects.update_or_create(
-            event_type = 'Gv',
-            show_title = title,
-            show_link = link,
-            show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
-            show_day = datetime.strptime(dateTime, DATETIME_FORMAT),
-            venue = venue,
-            scraper = scraper
-        )
-        add_calendar(new_event, 'msp')
-
-getEvents(br)
-sleep(5)
-br.get("https://www.stpaul.gov/calendar?page=1")
-getEvents(br)
-
-br.close()
-
-# br.find_element_by_class_name('fc-btn_allCalendars-button').click()

events/scrapers/Working/govt/mngov.py

@@ -1,120 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-from datetime import datetime
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-
-import events.digitools as digitools
-
-td = relativedelta.relativedelta(hours=5)
-odt = datetime.now() + td
-
-venue, created = Organization.objects.get_or_create(
-     name="MN Launch",
-     city="Minneapolis",
-     website="https://mn.gov/launchmn/calendar",
-    )
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name=venue.name,
-        website=venue.website,
-        calendar = Calendar.objects.get(shortcode='msp'),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-    )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-    print("Scraper: ", scraper)
-
-event_type = "Ed"
-
-cal_url = "https://timelyapp.time.ly/api/calendars/54705514/export?format=ics&target=copy&start_date=2024-12-13"
-calendar_url = 'https://calendar.google.com/calendar/ical/uvkshlggh1h4ck08emab22btkum9hl94%40import.calendar.google.com/public/basic.ics'
-
-objIcalData = requests.get(cal_url)
-
-gcal = iCalendar.from_ical(objIcalData.text)
-
-cfpa_events = []
-tz = pytz.timezone("US/Central")
-
-for component in gcal.walk():
-    event = {}
-    event['strSummary'] = f"{(component.get('SUMMARY'))}"
-    event['strDesc'] = component.get('DESCRIPTION')
-    event['strLocation'] = component.get('LOCATION')
-    event['dateStart'] = component.get('DTSTART')
-    event['dateStamp'] = component.get('DTSTAMP')
-    if event['dateStamp'] is not None:
-        event['dateStamp'] = event['dateStart'].dt
-    if event['dateStart'] is not None:
-        try:
-            event['dateStart'] = event['dateStart'].dt
-        except Exception as e:
-             event['dateStart'] = event['dateStart'].dt
-        
-    event['dateEnd'] = (component.get('DTEND'))
-    if event['dateEnd'] is not None:
-            event['dateEnd'] = event['dateEnd'].dt
-    else:
-        event['dateEnd'] = event['dateStart']
-    if event['strSummary'] != 'None':
-        event['details'] = {
-             "description" : event['strDesc'], 
-             "Location" : event['strLocation'],
-        }
-        cfpa_events.append(event)
-        now_now = datetime.now().astimezone(tz)
-        try:
-            if event['dateStart'] > now_now:
-                print(event['strSummary'])
-                new_event, created = DSEvent.objects.update_or_create(
-                    event_type = event_type,
-                    show_title = event['strSummary'],
-                    show_link = venue.website,
-                    show_date = event['dateStart']-td,
-                    show_day = event['dateStart']-td,
-                    more_details = event["details"],
-                    venue = venue
-                )
-                digitools.add_calendar(new_event, 'msp')
-                scraper.items+=1
-                if event['strLocation'] != None and event['strLocation'] != 'MN' and event['strLocation'] != 'Online':
-                    loc = event['strLocation'].split('@')
-                    new_venue_name = loc[0]
-                    if len(loc) > 1:
-                        address = loc[1].split(",")
-                        city = address[1].strip()
-                        new_venue, created = Organization.objects.get_or_create(
-                            name=new_venue_name,
-                            city=city,
-                            website="https://mn.gov/launchmn/calendar",
-                        )
-                        new_event.venue = new_venue
-                        new_event.save()
-                else:
-                    new_event.venue = venue
-                    new_event.save()
-        except Exception as e:
-             print(e)
-             print("Event: ", event['dateStart'], event['strSummary'])
-             print("Clock: ", now_now)                  
-    else:
-        print("Failed: ", component.get('DESCRIPTION'))
-        
-scraper.save()
-
-

events/scrapers/Working/iCal/ical.CAFAC.mpls.py

@@ -1,68 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-
-from datetime import datetime
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-import events.digitools as digitools
-
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-venue, created = Organization.objects.get_or_create(
-     name="Chicago Ave Fire Arts Center",
-     city="Minneapolis",
-     website="https://www.cafac.org/classes",
-    )
-event_type = "Ed"
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="Chicago Ave Fire Arts Center",
-        website="https://calendar.google.com/calendar/ical/9qj2426rukra3jv933nslsf3r8%40group.calendar.google.com/public/basic.ics",
-        calendar = Calendar.objects.get(id=1),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-        )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-
-item_count_start = scraper.items
-
-event_type = "Ed"
-
-objIcalData = requests.get(scraper.website)
-gcal = iCalendar.from_ical(objIcalData.text)
-tz = pytz.timezone("US/Central")
-
-events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
-
-for event in events:
-    # ppr(event)
-    e = {}
-    e['calendars'] = event['calendars']
-    try:
-        e['dateStamp'] = event['dateStart'][0]
-    except:
-        e['dateStamp'] = event['dateStart']
-    e['title'] = event['strSummary']
-    e['scraper'] = scraper 
-    e['link'] = venue.website
-    try:
-        digitools.createBasicEvent(e, 'Ed', venue)
-    except Exception as e:
-        print("Error: ", e)
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_ingles.py

@@ -1,146 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-
-from datetime import datetime, timedelta
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from config.env import env 
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-import events.digitools as digitools
-
-td = relativedelta.relativedelta(hours=5)
-
-
-venue, created = Organization.objects.get_or_create(
-     name="idioki",
-     city="Medellin",
-     website="https://idioki.com/",
-    )
-event_type = "Mu"
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="idioki",
-        website="https://calendar.google.com/calendar/ical/46ae0446724b1b3ee83cbd7dbc0db6a235bf97509ad860ca91eada3c267b5e41%40group.calendar.google.com/public/basic.ics",
-        calendar = Calendar.objects.get(shortcode='mde'),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-        )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-
-item_count_start = scraper.items
-objIcalData = requests.get(scraper.website)
-gcal = iCalendar.from_ical(objIcalData.text)
-tz = pytz.timezone("US/Central")
-
-def createEvent(day, date, event, scraper, venue, event_type):
-    ppr(event)
-    print("NAME: ", venue.name)
-    print('\n\n')
-    
-    if venue.name == "DANCEFREE":
-        venue.website = "https://www.instagram.com/dancefreeco"
-    if venue.name == "Vintrash":
-        venue.website = "https://www.instagram.com/vintrashbar"
-    if venue.name == "The Wandering Paisa":
-        venue.website = "https://wanderingpaisahostel.com"
-    if venue.name == "Dulce Posion":
-        venue.website = "https://www.instagram.com/dulceposionr"
-    if venue.name == "Blood Dance Company":
-        venue.website = "https://www.instagram.com/blooddancecompany"
-    if venue.name == "OLSA Certified Spanish School":
-        venue.website = "https://www.olsafoundation.org/"
-    if event['strSummary'] == "Merli Rooftop Language Exchange":
-        venue.website = "https://calendar.google.com/calendar/embed?src=46ae0446724b1b3ee83cbd7dbc0db6a235bf97509ad860ca91eada3c267b5e41%40group.calendar.google.com&ctz=America%2FBogota"
-    if "Concious Warrior" in event['strSummary']:
-        venue.website = "https://www.consciouscolombia.com/"
-    # if venue.name == "":
-    #     venue.website = "https://www.consciouscolombia.com/"
-    # if venue.name == "":
-    #     venue.website = "https://www.consciouscolombia.com/"
-    # if venue.name == "":
-    #     venue.website = "https://www.consciouscolombia.com/"
-    venue.save()
-
-    days = [day-1, day+6, day+13]
-    for day in days:
-        event['dateStamp'] = date + timedelta(days=day)
-        event['dateStart'] = event['dateStamp']
-        print("sending")
-        digitools.createCleanIcalEvent(event, scraper, venue, event_type)
-    return
-
-def splitLocation(event):
-     loc_split = event['strLocation'].split(',')
-     venue_name = loc_split[0]
-     venue, created = Organization.objects.get_or_create(
-        name=venue_name,
-        city="Medellin",
-        )
-     event['venue'] = venue
-     return event
-
-counter = 0
-
-for component in gcal.walk():
-    event = {}
-    event['scraper'] = scraper
-    event['calendars'] = [scraper.calendar]
-    event['strSummary'] = f"{(component.get('SUMMARY'))}"
-    event['strDesc'] = component.get('DESCRIPTION')
-    event['strLocation'] = str(component.get('LOCATION'))
-    # startDate = component.get('DTSTART')
-    # startTime = startDate.time()
-    event['dateStart'] = component.get('DTSTART')
-    event['dateStamp'] = component.get('DTSTAMP')
-
-    if event['strSummary'] != 'None':
-        event['details'] = {
-            "description" : event['strDesc'], 
-            "Location" : event['strLocation'],
-        }
-    
-    if event['dateStamp'] != None:
-        event['dateStart'] = event['dateStart'].dt
-        event['dateStart'] = datetime.strptime(str(event['dateStart'])[:-6], '%Y-%m-%d %H:%M:%S')
-        rules = component.get('RRule')
-        try:
-            if rules['FREQ'][0] == 'WEEKLY':
-                if datetime.today().weekday() != 0:
-                    event = splitLocation(event)
-                    date = datetime.today().date() - timedelta(days=datetime.today().weekday())
-                    date = datetime.combine(date, event['dateStart'].time())
-                    days = ["SU", "MO", "TU", "WE", "TH", "FR", "SA"]
-                    for day in rules['BYDAY']:
-                        day = days.index(day)
-                        createEvent(day, date, event, scraper, event['venue'], "Ed")
-        except Exception as e:
-            print("Error: ", e, "\n\n\n\n")
-            pass
-
-digitools.updateScraper(scraper, item_count_start)
-
-new_time = datetime.now() - timedelta(days=1)
-right_bound_time = datetime.now() + timedelta(days=45)
-events = DSEvent.objects.filter(show_date__lte=new_time)
-events1 = DSEvent.objects.filter(show_date__gte=right_bound_time)
-
-for e in events:
-    e.delete()
-
-for e in events1:
-    e.delete()

events/scrapers/Working/iCal/ical_online_events.py

@@ -1,118 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-
-from datetime import datetime, timedelta
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from config.env import env 
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-import events.digitools as digitools
-
-td = relativedelta.relativedelta(hours=5)
-
-
-venue, created = Organization.objects.get_or_create(
-     name="Online Events",
-     city="Online",
-     website="https://dreamfreely.org/",
-    )
-event_type = "Mu"
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="Online Events",
-        website="https://calendar.google.com/calendar/ical/p1a4r9glkjpu4u6iv3fkmu8qtc%40group.calendar.google.com/public/basic.ics",
-        calendar = Calendar.objects.get(shortcode='000'),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-        )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-
-item_count_start = scraper.items
-objIcalData = requests.get(scraper.website)
-gcal = iCalendar.from_ical(objIcalData.text)
-tz = pytz.timezone("US/Central")
-
-def createEvent(day, date, event, scraper, venue, event_type):
-    days = [day-1, day+6, day+13]
-    for day in days:
-        event['dateStamp'] = date + timedelta(days=day)
-        event['dateStart'] = event['dateStamp']
-        digitools.createCleanIcalEvent(event, scraper, venue, event_type)
-    return
-
-def splitLocation(event):
-     loc_split = event['strLocation'].split(',')
-     venue_name = loc_split[0]
-     venue, created = Organization.objects.get_or_create(
-        name=venue_name,
-        )
-     event['venue'] = venue
-     return event
-
-counter = 0
-
-for component in gcal.walk():
-    event = {}
-    event['scraper'] = scraper
-    event['calendars'] = [scraper.calendar]
-    event['strSummary'] = f"{(component.get('SUMMARY'))}"
-    event['strDesc'] = component.get('DESCRIPTION')
-    event['strLocation'] = str(component.get('LOCATION'))
-    # startDate = component.get('DTSTART')
-    # startTime = startDate.time()
-    event['dateStart'] = component.get('DTSTART')
-    event['dateStamp'] = component.get('DTSTAMP')
-
-    if event['strSummary'] != 'None':
-        event['details'] = {
-            "description" : event['strDesc'], 
-            "Location" : event['strLocation'],
-        }
-    if 'Mikel' in event['strSummary']:
-        print('JELLOO \n\n\n JELOOO')
-        pass
-    elif event['dateStamp'] != None:
-        event['dateStart'] = event['dateStart'].dt
-        event['dateStart'] = datetime.strptime(str(event['dateStart'])[:-6], '%Y-%m-%d %H:%M:%S')
-        rules = component.get('RRule')
-        try:
-            if rules['FREQ'][0] == 'WEEKLY':
-                if datetime.today().weekday() != 0:
-                    event = splitLocation(event)
-                    date = datetime.today().date() - timedelta(days=datetime.today().weekday())
-                    date = datetime.combine(date, event['dateStart'].time())
-                    days = ["SU", "MO", "TU", "WE", "TH", "FR", "SA"]
-                    for day in rules['BYDAY']:
-                        day = days.index(day)
-                        createEvent(day, date, event, scraper, event['venue'], "Ed")
-        except Exception as e:
-            print("Error (no repeat): ", e)
-            pass
-
-digitools.updateScraper(scraper, item_count_start)
-
-new_time = datetime.now() - timedelta(days=1)
-right_bound_time = datetime.now() + timedelta(days=45)
-events = DSEvent.objects.filter(show_date__lte=new_time)
-events1 = DSEvent.objects.filter(show_date__gte=right_bound_time)
-
-for e in events:
-    e.delete()
-
-for e in events1:
-    e.delete()

events/scrapers/Working/iCal/ical_run.SocialableCider.mpls.py

@@ -1,65 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-
-from datetime import datetime
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-import events.digitools as digitools
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-
-venue, created = Organization.objects.get_or_create(
-     name="Sociable Ciderwerks",
-     city="Minneapolis",
-     website="https://sociablecider.com/events",
-    )
-event_type = "Mu"
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="Sociable Ciderwerks",
-        website="https://calendar.google.com/calendar/ical/c_oa7uitvkn871o1ojl5e1os4ve8%40group.calendar.google.com/public/basic.ics",
-        calendar = Calendar.objects.get(id=1),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-        )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-
-item_count_start = scraper.items
-
-objIcalData = requests.get(scraper.website)
-gcal = iCalendar.from_ical(objIcalData.text)
-tz = pytz.timezone("US/Central")
-events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
-
-for event in events:
-    # ppr(event)
-    e = {}
-    e['calendars'] = event['calendars']
-    try:
-        e['dateStamp'] = event['dateStart'][0]
-    except:
-        e['dateStamp'] = event['dateStart']
-    e['title'] = event['strSummary']
-    e['scraper'] = scraper 
-    e['link'] = venue.website
-    try:
-        digitools.createBasicEvent(e, 'Mu', venue)
-    except Exception as e:
-        print("Error: ", e)
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_run.bunkers.py

@@ -1,66 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-
-import events.digitools as digitools
-
-from datetime import datetime
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-venue, created = Organization.objects.get_or_create(
-     name="Bunkers",
-     city="Minneapolis",
-     website="https://bunkersmusic.com/calendar/",
-     is_venue = True
-    )
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="Bunkers",
-        website="https://calendar.google.com/calendar/ical/js94epu90r2et31aopons1ifm8%40group.calendar.google.com/public/basic.ics",
-        calendar = Calendar.objects.get(id=1),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-        )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-
-item_count_start = scraper.items
-
-event_type = "Mu"
-
-objIcalData = requests.get(scraper.website)
-gcal = iCalendar.from_ical(objIcalData.text)
-tz = pytz.timezone("US/Central")
-events = digitools.getiCalEvents(gcal, scraper, venue, "Mu")
-
-for event in events:
-    # ppr(event)
-    e = {}
-    e['calendars'] = event['calendars']
-    try:
-        e['dateStamp'] = event['dateStart'][0]
-    except:
-        e['dateStamp'] = event['dateStart']
-    e['title'] = event['strSummary']
-    e['scraper'] = scraper 
-    e['link'] = venue.website
-    try:
-        digitools.createBasicEvent(e, 'Mu', venue)
-    except Exception as e:
-        print("Error: ", e)
-
-# digitools.returniCalEvents(gcal, scraper, venue, "Mu")
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_run.cfpa.py

@@ -1,93 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-
-from datetime import datetime
-from dateutil import relativedelta
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-import events.digitools as digitools
-
-tz = pytz.timezone("US/Central")
-td = relativedelta.relativedelta(hours=5)
-
-venue, created = Organization.objects.get_or_create(
-     name="Center for Performing Arts",
-     city="Minneapolis",
-     website="https://www.cfpampls.com/events",
-    )
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="Center for Performing Arts",
-        website="https://calendar.google.com/calendar/ical/6rpooudjg01vc8bjek1snu2ro0%40group.calendar.google.com/public/basic.ics",
-        calendar = Calendar.objects.get(id=1),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-        )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-
-item_count_start = scraper.items
-
-event_type = "Ed"
-
-objIcalData = requests.get(scraper.website)
-gcal = iCalendar.from_ical(objIcalData.text)
-tz = pytz.timezone("US/Central")
-
-events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
-
-for event in events:
-    ppr(event)
-    e = {}
-    e['calendars'] = event['calendars']
-    try:
-        e['dateStamp'] = event['dateStart'][0]
-    except:
-        e['dateStamp'] = event['dateStart']
-    e['title'] = event['strSummary']
-    e['scraper'] = scraper 
-    e['link'] = venue.website
-    try:
-        digitools.createBasicEvent(e, 'Ed', venue)
-    except Exception as e:
-        print("Error: ", e)
-
-
-    # now_now = datetime.today().date()
-    # try:
-    #     print("1Event: ", event['dateStart'])
-    #     if event['dateStart'] > now_now:
-    #         print("Check Check: ", event['dateStart'])
-    #         ppr(event)
-    #         # createIcalEvent(event, scraper, venue, event_type)
-    #     else:
-    #         print("WHAT?")
-    #         ppr(event)
-    # except Exception as e:
-    #     try:
-    #         event['dateStart'] = event['dateStart'].date()
-    #         print("2Event: ", event['dateStart'])
-    #         if event['dateStart'] > now_now:
-    #             print("Check Check: ", event['dateStart'])
-    #             ppr(event)
-    #             # createIcalEvent(event, scraper, venue, event_type)
-    #         else:
-    #             print("WHAT?")
-    #             ppr(event)
-    #     except Exception as e:
-    #         print("The Error: ", e)
-    #         pass
-
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_run.eagles.py

@@ -1,63 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-from datetime import datetime
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-import events.digitools as digitools
-
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-venue, created = Organization.objects.get_or_create(
-     name="Eagles #34",
-     city="Minneapolis",
-     website="https://www.minneapoliseagles34.org/events-entertainment.html",
-    )
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="Eagles #34",
-        website="https://calendar.google.com/calendar/ical/teflgutelllvla7r6vfcmjdjjo%40group.calendar.google.com/public/basic.ics",
-        calendar = Calendar.objects.get(id=1),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-        )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-
-item_count_start = scraper.items
-
-event_type = "Mu"
-
-objIcalData = requests.get(scraper.website)
-gcal = iCalendar.from_ical(objIcalData.text)
-tz = pytz.timezone("US/Central")
-events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
-
-for event in events:
-    ppr(event)
-    e = {}
-    e['calendars'] = event['calendars']
-    try:
-        e['dateStamp'] = event['dateStart'][0]
-    except:
-        e['dateStamp'] = event['dateStart']
-    e['title'] = event['strSummary']
-    e['scraper'] = scraper 
-    e['link'] = venue.website
-    try:
-        digitools.createBasicEvent(e, 'Mu', venue)
-    except Exception as e:
-        print("Error: ", e)
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_run.terminalbar-mpls.py

@@ -1,67 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-from datetime import datetime
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-
-import events.digitools as digitools
-
-td = relativedelta.relativedelta(hours=5)
-odt = datetime.now() + td
-
-venue, created = Organization.objects.get_or_create(
-     name="Terminal Bar",
-     city="Minneapolis",
-     website="https://terminalbarmn.com",
-    )
-event_type = "Mu"
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="Terminal Bar",
-        website="https://calendar.google.com/calendar/ical/terminalbar32%40gmail.com/public/basic.ics",
-        calendar = Calendar.objects.get(id=1),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-        )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-
-item_count_start = scraper.items
-
-event_type = "Mu"
-
-objIcalData = requests.get(scraper.website)
-gcal = iCalendar.from_ical(objIcalData.text)
-tz = pytz.timezone("US/Central")
-events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
-
-for event in events:
-    # ppr(event)
-    e = {}
-    e['calendars'] = event['calendars']
-    try:
-        e['dateStamp'] = event['dateStart'][0]
-    except:
-        e['dateStamp'] = event['dateStart']
-    e['title'] = event['strSummary']
-    e['scraper'] = scraper 
-    e['link'] = venue.website
-    try:
-        digitools.createBasicEvent(e, 'Mu', venue)
-    except Exception as e:
-        print("Error: ", e)
-        
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ical_run.whitesquirrel.py

@@ -1,69 +0,0 @@
-import requests, os, sys
-# from icalendar import Calendar as iCalendar, Event
-from icalendar import Calendar as iCalendar
-from datetime import datetime
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-import events.digitools as digitools
-
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%Y-%m-%d %I:%M %p'
-
-venue, created = Organization.objects.get_or_create(
-     name="White Squirrel",
-     city="St. Paul",
-     website="https://whitesquirrelbar.com",
-     is_venue = True
-    )
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="White Squirrel",
-        website="https://calendar.google.com/calendar/ical/vh5sr9h59nmrs2op5lmptu2fsa344cig%40import.calendar.google.com/public/basic.ics",
-        calendar = Calendar.objects.get(shortcode='msp'),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-        )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-
-item_count_start = scraper.items
-event_type = "Mu"
-
-objIcalData = requests.get(scraper.website)
-gcal = iCalendar.from_ical(objIcalData.content)
-tz = pytz.timezone("US/Central")
-events = digitools.getiCalEvents(gcal, scraper, venue, event_type)
-
-for event in events:
-    ppr(event)
-    e = {}
-    e['calendars'] = event['calendars']
-    try:
-        e['dateStamp'] = event['dateStart'][0]
-    except:
-        e['dateStamp'] = event['dateStart']
-    e['title'] = event['strSummary']
-    e['scraper'] = scraper 
-    e['link'] = venue.website
-    try:
-        digitools.createBasicEvent(e, event_type, venue)
-    except Exception as e:
-        print("Error: ", e)
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/iCal/ws.ics

@@ -1,498 +0,0 @@
-BEGIN:VCALENDAR
-PRODID:-//Google Inc//Google Calendar 70.9054//EN
-VERSION:2.0
-CALSCALE:GREGORIAN
-METHOD:PUBLISH
-X-WR-CALNAME:The White Squirrel Bar
-X-WR-TIMEZONE:UTC
-X-WR-CALDESC:Events for The White Squirrel Bar
-BEGIN:VEVENT
-DTSTART:20251118T030000Z
-DTEND:20251118T060000Z
-DTSTAMP:20251116T033816Z
-UID:10806-1763413200-1763424000@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/the-deeper-kind-w-23-watts/
-CREATED:20251104T233617Z
-LAST-MODIFIED:20251105T053618Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:The Deeper Kind w. 23 Watts
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251129T000000Z
-DTEND:20251129T020000Z
-DTSTAMP:20251116T033816Z
-UID:10852-1764352800-1764360000@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/the-mary-cutrufello-band-3/
-CREATED:20251115T173617Z
-DESCRIPTION:A rock and roll band! Killer harmonies\, rippin’ solos\, cool s
- ongs\, funny\nshit in between. \nHailed as a country guitarist\, Mary Cutru
- fello grew up on East Coast\nclassic rock. (Yes\, think Springsteen!) This 
- tight\, taut trio explores\nher songs old and new with the fearless guitars
- \, stories\, and\nbetween-song banter she’s known for.
-LAST-MODIFIED:20251115T233617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:The Mary Cutrufello Band
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251117T030000Z
-DTEND:20251117T060000Z
-DTSTAMP:20251116T033816Z
-UID:10802-1763326800-1763337600@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/karaoke-with-ally/
-CREATED:20251103T163617Z
-LAST-MODIFIED:20251103T223619Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Karaoke With Ally!
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251127T000000Z
-DTEND:20251127T020000Z
-DTSTAMP:20251116T033816Z
-UID:10846-1764180000-1764187200@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/family-dinner-w-doug-sarah-4/
-CREATED:20251115T013617Z
-LAST-MODIFIED:20251115T073617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Family Dinner w. Doug & Sarah
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251116T190000Z
-DTEND:20251116T220000Z
-DTSTAMP:20251116T033816Z
-UID:10147-1763298000-1763308800@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/the-friend-ship/
-CREATED:20251103T003616Z
-DESCRIPTION:These popular singer/songwriters first got together to perform 
- on Joel Sax’s debut\nalbum\, ‘Billions of Stars.’ They had so much fun and 
- got such a great response at their\nsold-out release show that they have jo
- ined together to form a new band\, The Friend\nShip! They will bring their 
- best songs\, sweetest harmonies\, and funniest stories to the\nWhite Squirr
- el for a night of beautiful music. \nNikki Lemire is an incredibly in deman
- d artist\, having earned accolades from fans and critics\nalike. Natalia To
- ledo\, from The Current describes ‘Mend’ It as “a dreamy\, simple\, and\nbe
- autiful song led by the sounds of her harp.”\nJoel Sax writes heartfelt son
- gs about love\, regret\, and hope for a bigger\, brighter future.\nHis debu
- t album title track\, Billions of Stars\, is “… a wonderful mix for a conte
- mporary\nfolk song” – Minnesota Sound and ‘A Lovely New Album\,’ by Mostly 
- Minnesota\nMusic. \nMarc Severin has been a mainstay of the music scene for
-  many years. His music is\ntrue Americana and his storytelling is midwester
- n country at its best. The Friend Ship\nwill be previewing songs from his w
- ell anticipated new album coming out this Winter. \nDave Mehling is a highl
- y sought-after musician\, producer and songwriter. He has\nproduced albums 
- for many artists\, including Sarah Morris\, Haley E Rydell\, Joel Sax and\n
- Emily Haavik. “You are drawn in with his lyrics. They speak of very univers
- al issues of\nlove which we all have\, but few of us have the talent to giv
- e them a voice. Dave Mehling\npossesses this rare gift.” – Becca Martin
-LAST-MODIFIED:20251103T063617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:The Friend Ship
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251123T000000Z
-DTEND:20251123T020000Z
-DTSTAMP:20251116T033816Z
-UID:10827-1763834400-1763841600@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/giant-valley-string-band/
-CREATED:20251109T183617Z
-DESCRIPTION:Giant Valley String Band is a Twin Cities acoustic group playin
- g an\neclectic mix of covers and originals in Bluegrass and Americana. The\
- nband members are Noelle Haland on guitar\, Ted Haland on dobro and\nbass\,
-  Nate Hess on fiddle and guitar\, Erika Janik on banjo\, Matt\nJensen on ma
- ndolin\, and Debra Pflipsen on guitar\, harmonica\,\naccordion\, and bass. 
- More info at giantvalleystringband.com. \n\n\n\n\nBlue Groove is a multiple
-  award-winning bluegrass band based in the\nTwin Cities that plays both con
- temporary and classic bluegrass tunes\nwith a creative approach that drives
-  while still maintaining a sense\nof laid-back\, Midwestern chill. That’s w
- hat they mean by “groove”. The\nbands inviting sound combined with their op
- enness warmth on and\noff-stage have made them crowd favorites at festivals
-  and concert\nvenues all around Minnesota and the Upper Midwest. \nThe band
-  covers a number of past and current contemporary bluegrass\nartists. A Blu
- e Groove show may feature songs from Alison Krauss or\nRhonda Vincent as we
- ll as traditional bluegrass (Bill Monroe\, Flatt &\nScruggs) with all of it
- s drive\, improvisation and harmony singing!\nBlue Groove also likes to put
-  a bluegrass spin on more\ncontemporary/pop artists such as U2\, Passenger\
- , and the Doobie\nBrothers! \nMembers of Blue Groove bluegrass band include
-  Adelle Hyrkas\, lead\nvocals and rhythm guitar\; David Smith\, banjo and v
- ocals\; Bob Doe\, lead\nguitar\, dobro\, and vocals\; Pete Mathison bass an
- d vocals\, Tom\nWadzinski mandolin and vocals. Ocassionally we may have a g
- uest\nfiddler- past guests have included fiddlers Richard Kriehn\, Michael\
- nPrewitt\, and AJ Srubas. \nYou can find out more about Blue Groove at www.
- bluegroovebluegrass.com
-LAST-MODIFIED:20251110T003617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Giant Valley String Band w. Blue Groove
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251121T220000Z
-DTEND:20251122T040000Z
-DTSTAMP:20251116T033816Z
-UID:11255-1763740800-1763762400@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/the-angry-line-cook-4/
-CREATED:20251114T133616Z
-DESCRIPTION:The Best Smash Burgers in town!
-LAST-MODIFIED:20251114T193617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:The Angry Line Cook
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251127T030000Z
-DTEND:20251127T060000Z
-DTSTAMP:20251116T033816Z
-UID:10848-1764190800-1764201600@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/nights-with-tim-2/
-CREATED:20251115T013617Z
-DESCRIPTION:Dig into the deeper\, progressive side of disco\, funk\, and R&
- B from the ’70s and ’80s\, with the occasional turn into house and left-fie
- ld gems. An all vinyl night courtesy of local selector Tim. He knows how to
-  keep the vibes at White Squirrel curious and moving.
-LAST-MODIFIED:20251115T073617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Nights with Tim
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251122T190000Z
-DTEND:20251122T220000Z
-DTSTAMP:20251116T033816Z
-UID:10825-1763816400-1763827200@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/caitlin-robertson/
-CREATED:20251109T023617Z
-LAST-MODIFIED:20251109T083618Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Caitlin Robertson
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251128T220000Z
-DTEND:20251129T040000Z
-DTSTAMP:20251116T033816Z
-UID:11257-1764345600-1764367200@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/the-angry-line-cook-5/
-CREATED:20251115T013617Z
-DESCRIPTION:The Best Smash Burgers in town!
-LAST-MODIFIED:20251115T073617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:The Angry Line Cook
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251122T000000Z
-DTEND:20251122T020000Z
-DTSTAMP:20251116T033816Z
-UID:10821-1763748000-1763755200@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/chris-holm-friends-2/
-CREATED:20251108T203618Z
-LAST-MODIFIED:20251109T023619Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Chris Holm & Friends
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251123T030000Z
-DTEND:20251123T060000Z
-DTSTAMP:20251116T033816Z
-UID:10829-1763845200-1763856000@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/13-howell/
-CREATED:20251110T003617Z
-LAST-MODIFIED:20251110T063617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:13 Howell w. Steph Was & The Secret Izz\, TH3
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251120T000000Z
-DTEND:20251120T020000Z
-DTSTAMP:20251116T033816Z
-UID:10813-1763575200-1763582400@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/st-paul-mudsteppers-4/
-CREATED:20251107T033617Z
-LAST-MODIFIED:20251107T093617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:St Paul Mudsteppers
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251116T000000Z
-DTEND:20251116T020000Z
-DTSTAMP:20251116T033816Z
-UID:10819-1763229600-1763236800@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/in-circles-sunny-day-real-estate-tri
- bute-w-120-minutes/
-CREATED:20251102T013616Z
-DESCRIPTION:It’s the 30th anniversary of Sunny Day Real Estates LP2 record!
-  In Circles with be performing the album in its entirety. \n 
-LAST-MODIFIED:20251102T063617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:In Circles (Sunny Day Real Estate Tribute) w. 120 Minutes
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251124T000000Z
-DTEND:20251124T020000Z
-DTSTAMP:20251116T033816Z
-UID:10833-1763920800-1763928000@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/the-jeff-becker-band/
-CREATED:20251110T223617Z
-LAST-MODIFIED:20251111T043617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:The Jeff Becker Band
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251122T030000Z
-DTEND:20251122T060000Z
-DTSTAMP:20251116T033816Z
-UID:10823-1763758800-1763769600@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/ancient-waves-2/
-CREATED:20251108T203618Z
-LAST-MODIFIED:20251109T023619Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Ancient Waves w. Muun Batos & Jarad Miles
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251121T000000Z
-DTEND:20251121T020000Z
-DTSTAMP:20251116T033816Z
-UID:10815-1763661600-1763668800@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/devaney-friends-2/
-CREATED:20251107T223617Z
-LAST-MODIFIED:20251108T043617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Devaney & Friends w. Aaron James
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251121T030000Z
-DTEND:20251121T060000Z
-DTSTAMP:20251116T033816Z
-UID:10817-1763672400-1763683200@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/country-oke-3/
-CREATED:20251108T043618Z
-LAST-MODIFIED:20251108T103619Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Country-oke
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251124T030000Z
-DTEND:20251124T060000Z
-DTSTAMP:20251116T033816Z
-UID:10835-1763931600-1763942400@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/bryan-the-haggards/
-CREATED:20251111T043617Z
-LAST-MODIFIED:20251111T103617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Bryan & The Haggards w. Jump Loop\, The American Songbook
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251125T030000Z
-DTEND:20251125T060000Z
-DTSTAMP:20251116T033816Z
-UID:10839-1764018000-1764028800@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/seven-seasons/
-CREATED:20251112T013616Z
-LAST-MODIFIED:20251112T073617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Slasher Film
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251126T030000Z
-DTEND:20251126T060000Z
-DTSTAMP:20251116T033816Z
-UID:10841-1764104400-1764115200@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/third-date-tuesday-night-residency-2
- /
-CREATED:20251113T053616Z
-LAST-MODIFIED:20251113T113617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Third Date Tuesday Night Residency
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251117T000000Z
-DTEND:20251117T020000Z
-DTSTAMP:20251116T033816Z
-UID:10800-1763316000-1763323200@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/bingo-with-pete-3/
-CREATED:20251103T003616Z
-LAST-MODIFIED:20251103T063617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Bingo With Pete!
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251125T000000Z
-DTEND:20251125T020000Z
-DTSTAMP:20251116T033816Z
-UID:10837-1764007200-1764014400@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/devil-dodger/
-CREATED:20251111T203617Z
-LAST-MODIFIED:20251112T023617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Devil Dodger
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251126T000000Z
-DTEND:20251126T020000Z
-DTSTAMP:20251116T033816Z
-UID:10843-1764093600-1764100800@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/ali-grays-country-jamboree-2/
-CREATED:20251112T233616Z
-DESCRIPTION:Come on out for a tribute to the Darlings of Country music\, fe
- aturing Ali Gray’s Country Jamboree! Hear songs from the catalogs of Lorett
- a Lynn\, Tammy Wynette\, Tanya Tucker\, Dolly Parton\, Linda Ronstadt\, Pam
-  Tillis\, Patty Loveless and many more. We won’t leave out the Kings of Cou
- ntry\, sung by guitar legend Dan Neale and drummer extraordinaire Scott Wen
- um\, including favorites like Roger Miller\, Hank Williams\, Johnny Cash\, 
- Buck Owens\, Waylon Jennings and more!  Also featuring Dan Lowinger on bass
- ! 6-8.
-LAST-MODIFIED:20251113T053616Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Ali Gray's Country Jamboree
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251116T030000Z
-DTEND:20251116T060000Z
-DTSTAMP:20251116T033816Z
-UID:10798-1763240400-1763251200@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/lost-island-society-w-woolly-mack-po
- ison-ivy-the-people/
-CREATED:20251102T183617Z
-LAST-MODIFIED:20251103T003617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Lost Island Society w. Woolly Mack\, Poison Ivy & The People
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251123T190000Z
-DTEND:20251123T220000Z
-DTSTAMP:20251116T033816Z
-UID:10831-1763902800-1763913600@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/the-stress-of-her-regard/
-CREATED:20251110T003617Z
-LAST-MODIFIED:20251110T063617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:The Stress Of Her Regard
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251119T030000Z
-DTEND:20251119T060000Z
-DTSTAMP:20251116T033816Z
-UID:10811-1763499600-1763510400@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/third-date-tuesday-night-residency/
-CREATED:20251106T213618Z
-LAST-MODIFIED:20251107T033618Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Third Date Tuesday Night Residency
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251119T000000Z
-DTEND:20251119T020000Z
-DTSTAMP:20251116T033816Z
-UID:10809-1763488800-1763496000@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/the-new-havoline-supremes/
-CREATED:20251105T053616Z
-DESCRIPTION:The New Havoline Supremes are a Twin Cities based band\,  featu
- ring the legendary Mary Cutrufello on guitar and vocals\, and Dan Lowinger 
- on guitar.  Formed in October of 2023\, the band plays a mix of classic cou
- ntry and Mary’s original tunes\, and features Greg Schutte (Ryan Bingham\, 
- Mickey Hart\, Chasity Brown) on drums\, and Erik Lillestol (Cafe Accordion)
-  on bass.\n\nMary Cutrufello\n\nWith 30 years of experience in the music bu
- siness\, Mary Cutrufello is everything from Texas honky-tonk heroine and fi
- ery Midwestern roots-rocker to a powerhouse acoustic performer. Connecticut
- -raised and Yale-educated\, Mary’s musical journey has taken her from the E
- ast Coast to Houston and now to Minnesota.  Career highlights include: the 
- major label release of    ”When the Night is Through\,” on Mercury Records 
- in 1998\, appearances on national tv shows such as Tonight Show with Jay Le
- no\, appearances at major festivals including Farm Aid\, and tours with Jim
- mie Dale Gilmore and Tish Hinojosa.  Mary has seven self-released albums\, 
- and has played guitar locally for Annie Mack\, Michael Perry and the Long B
- eds\, and many others.\n\n\n\nDan Lowinger\, a transplant from Portland\, O
- R\, is a guitarist steeped in the tradition of Western swing and classic co
- untry. Since moving to the Twin Cities in 2017\, Dan has become a fixture i
- n the local country and Americana scene\, playing on a regular basis with a
- ll star Western swing band Honky Tonk Jump\, and country rockers Cole Diamo
- nd\, while also sharing the stage with local greats\, Martin Zellar (Gear D
- addies)\, Tony Andreason (The Trashmen)\, Pat Donohue (Prairie Home Compani
- on) and many others! He has performed regionally and toured nationally with
-  many different country and swing bands from the Pacific Northwest includin
- g: The Barn Door Slammers\, Ralph Carney’s Pepper Grinders\, Vince Mira\, a
- nd Western Centuries.
-LAST-MODIFIED:20251105T113617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:The New Havoline Supremes
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251120T030000Z
-DTEND:20251120T060000Z
-DTSTAMP:20251116T033816Z
-UID:10782-1763586000-1763596800@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/venturer-w-peeler-mild-manner/
-CREATED:20251107T223617Z
-DESCRIPTION:Venturer is a rock based band with many other genre influences 
- such as funk\, folk\, prog\, punk\, and jazz. Venturer has played around th
- e United States\, mainly in Milwaukee\, Madison\, Chicago\, and Minneapolis
- \, but also on the East Coast and greater Midwest.
-LAST-MODIFIED:20251108T043617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Venturer w. Peeler & Mild Manner
-TRANSP:OPAQUE
-END:VEVENT
-BEGIN:VEVENT
-DTSTART:20251118T000000Z
-DTEND:20251118T020000Z
-DTSTAMP:20251116T033816Z
-UID:10804-1763402400-1763409600@whitesquirrelbar.com
-URL:https://whitesquirrelbar.com/event/clovers-daughter/
-CREATED:20251103T223616Z
-LAST-MODIFIED:20251104T043617Z
-SEQUENCE:0
-STATUS:CONFIRMED
-SUMMARY:Clovers Daughter
-TRANSP:OPAQUE
-END:VEVENT
-END:VCALENDAR

events/scrapers/Working/news/minnpost.mn.py

@@ -1,78 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-from events.digitools import getBrowser, createURL, createBasicArticle, getSource
-
-
-org, created = Organization.objects.get_or_create(
-     name="MinnPost",
-     city="Minneapolis",
-     website="https://www.minnpost.com/",
-     is_venue=False,
-    )
-
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name=org.name,
-        website=org.website,
-        items = 0,
-        last_ran = datetime.now(),
-    )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=org.name)
-    print("Scraper: ", scraper)
-
-event_type = "Ja"
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%b %d %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/article')
-    count = 0
-    ppr(contents)
-    for c in contents:
-        try:
-            if count > 10:
-                br.close()
-                quit()
-            article = {}
-            article['title'] = c.xpath('.//*/h2[@class="entry-title"]/a/text()')[0]
-            article['link'] = c.xpath('.//*/h2[@class="entry-title"]/a/@href')[0]
-            createBasicArticle(article, event_type, org)
-            ppr(article)
-            print("Success")
-            count+=1
-        except Exception as e:
-            print(e)
-            ppr(article)
-            print("\n\n+++\n\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-
-ps = getSource(br, org.website)
-get_events(ps, "Ed")
-sleep(3)
-
-br.close()
-scraper.save()

events/scrapers/Working/news/racket.mn.py

@@ -1,68 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-from events.digitools import getBrowser, createURL, createBasicArticle, getSource
-
-scraper, created = Scraper.objects.get_or_create(
-     name="Racket MN",
-     website="https://racketmn.com",
-     last_ran = datetime.now(),
-    )
-
-org, created = Organization.objects.get_or_create(
-     name="Racket MN",
-     city="Minneapolis",
-     website="https://racketmn.com",
-     is_venue=False,
-    )
-
-event_type = "Ja"
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%b %d %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps, event_type):
-    count = 0
-    contents = ps.xpath('.//*/div[@class="PostCard_stackedWrapper__S21Fy"]') + ps.xpath('.//*/div[@class="PostCard_wrapper__uteO3"]')
-    for c in contents:
-        if count > 10:
-                br.close()
-                quit()
-        try:
-            article = {}
-            article['title'] = c.xpath('.//div/a/h3/text()')[0]
-            article['link'] = org.website + c.xpath('.//div/a/@href')[1]
-            createBasicArticle(article, event_type, org)
-            count+=1
-        except Exception as e:
-            print(e)
-            ppr(article)
-            print("\n+++\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-
-ps = getSource(br, org.website)
-get_events(ps, "Ed")
-sleep(3)
-
-br.close()
-scraper.save()

events/scrapers/Working/news/sahan.mn.py

@@ -1,68 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-from events.digitools import getBrowser, createURL, createBasicArticle, getSource
-
-scraper, created = Scraper.objects.get_or_create(
-     name="Sahan Journal",
-     website="https://sahanjournal.com/",
-     last_ran = datetime.now(),
-    )
-
-org, created = Organization.objects.get_or_create(
-     name="Sahan Journal",
-     city="Minneapolis",
-     website="https://sahanjournal.com/",
-     is_venue=False,
-    )
-
-event_type = "Ja"
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%b %d %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/article')
-    count = 0
-    for c in contents:
-        try:
-            if count > 10:
-                br.close()
-                quit()
-            article = {}
-            article['title'] = c.xpath('.//*/h2[@class="entry-title"]/a/text()')[0]
-            article['link'] = c.xpath('.//*/h2[@class="entry-title"]/a/@href')[0]
-            createBasicArticle(article, event_type, org)
-            count+=1
-        except Exception as e:
-            print(e)
-            ppr(article)
-            print("\n+++\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-
-ps = getSource(br, org.website)
-get_events(ps, "Ed")
-sleep(3)
-
-br.close()
-scraper.save()

events/scrapers/Working/news/unicornriot.py

@@ -1,63 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-from events.digitools import getBrowser, createURL, createBasicArticle, getSource
-
-scraper, created = Scraper.objects.get_or_create(
-     name="Uniocorn Riot",
-     website="https://unicornriot.ninja/",
-     last_ran = datetime.now(),
-    )
-
-org, created = Organization.objects.get_or_create(
-     name="Uniocorn Riot",
-     city="Minneapolis",
-     website="https://unicornriot.ninja/",
-     is_venue=False,
-    )
-
-event_type = "Ja"
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%b %d %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/article')
-    for c in contents[:10]:
-        try:
-            article = {}
-            article['title'] = c.xpath('.//*/h3[@class="title entry-title is-3"]/a/text()')[0]
-            article['link'] = c.xpath('.//*/h3[@class="title entry-title is-3"]/a/@href')[0]
-            createBasicArticle(article, event_type, org)
-        except Exception as e:
-            print(e)
-            ppr(article)
-            print("\n+++\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-
-ps = getSource(br, org.website)
-get_events(ps, "Ed")
-sleep(3)
-
-br.close()
-scraper.save()

events/scrapers/Working/smedia/bluesky.py

@@ -1,132 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-from atproto import Client
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from socials.models import SocialLink, SocialPost
-# from digitools import getBrowser, createURL, createBasicEvent, getSource
-
-tz = pytz.timezone("US/Central")
-
-USERNAME = "dreamfreely.org"
-PASSWORD = "gU):3-BA]DaK[_K$DHmI"
-
-client = Client()
-client.login(USERNAME, PASSWORD)
-feed = client.get_author_feed(USERNAME, limit = 100)
-
-def createSocialLink(post):
-    new_post, created = SocialLink.objects.update_or_create(
-        uri = post['uri'],
-        text = post['text'],
-        link = post['link'],
-        handle = post['handle'],
-        likes = post['likes'],
-        reposts = post['reposts'],
-        quotes = post['quotes'],
-        replies = post['replies'],
-        created_at = post['created_at'],
-        platform = 'bluesky',
-        rt_uri = post['rt_uri'],
-        rt_text = post['rt_text'],
-        rt_link = post['rt_link'],
-        rt_handle = post['rt_handle'],
-    )
-    # print(created, new_post)
-    print("completed write")
-
-tweets = []
-
-print(len(feed.feed))
-
-for post in feed.feed:
-    post = post.post
-    print("\n\nNEW POST\n\n")
-    # try:
-    #     ppr(post.embed.record.record.author.handle)
-    #     ppr(post.embed.record.record.value.text.split("\n")[:2])
-    #     ppr(post.embed.record.record.value.embed.external.uri.split("?")[0])
-    #     ppr(post.embed.record.record.uri.split("feed.post/")[1])
-    # except:
-    #     pass
-
-    if hasattr(post.record.embed, 'external'):
-        p = {}
-        try:
-            p['link'] = post.record.embed.external.uri.split("?")[0]
-        except:
-            pass
-        p['text'] = " ".join(post.record.text.split("\n")[:2])
-        p['handle'] = post.author.handle
-        p['uri'] = post.uri.split("feed.post/")[1]
-        p['likes'] = post.like_count
-        p['quotes'] = post.quote_count
-        p['replies'] = post.reply_count
-        p['reposts'] = post.repost_count
-        p['created_at'] = post.record.created_at
-
-        p['rt_handle'] = "blank"
-        p['rt_text'] = "blank"
-        p['rt_uri'] = "blank"
-        p['rt_link'] = "blank"
-        
-    elif hasattr(post.embed, 'record'):
-        p = {}
-        p['text'] = " ".join(post.record.text.split("\n")[:2])
-        p['handle'] = post.author.handle
-        p['uri'] = post.uri.split("feed.post/")[1]
-        p['likes'] = post.like_count
-        p['quotes'] = post.quote_count
-        p['replies'] = post.reply_count
-        p['reposts'] = post.repost_count
-        p['created_at'] = post.record.created_at
-        p['link'] = "blank"
-
-        try:
-            p['rt_handle'] = post.embed.record.record.author.handle
-            p['rt_text'] = " ".join(post.embed.record.record.value.text.split("\n")[:2])
-            p['rt_uri'] = post.embed.record.record.uri.split("feed.post/")[1]
-            p['rt_link'] = post.embed.record.record.value.embed.external.uri.split("?")[0]
-        except:
-            p['rt_handle'] = "blank"
-            p['rt_text'] = "blank"
-            p['rt_uri'] = "blank"
-            p['rt_link'] = "blank"
-        
-
-    else:
-        p = {}
-        p['text'] = " ".join(post.record.text.split("\n")[:2])
-        p['handle'] = post.author.handle
-        p['uri'] = post.uri.split("feed.post/")[1]
-        p['likes'] = post.like_count
-        p['quotes'] = post.quote_count
-        p['replies'] = post.reply_count
-        p['reposts'] = post.repost_count
-        p['created_at'] = post.record.created_at
-
-        p['rt_handle'] = "blank"
-        p['rt_text'] = "blank"
-        p['rt_uri'] = "blank"
-        p['rt_link'] = "blank"
-        p['link'] = "blank"
-
-    # ppr(p)
-    # tweets.append(p)
-
-    try:
-        print('writing file')
-        createSocialLink(p)
-    except Exception as e:
-        ppr(post.record.embed)
-        print(e, "\nthis\n\n")

events/scrapers/Working/smedia/bluesky_media.py

@@ -1,72 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-from atproto import Client
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from socials.models import SocialImg
-# from digitools import getBrowser, createURL, createBasicEvent, getSource
-
-tz = pytz.timezone("US/Central")
-
-USERNAME = "dreamfreely.org"
-PASSWORD = "gU):3-BA]DaK[_K$DHmI"
-
-client = Client()
-client.login(USERNAME, PASSWORD)
-feed = client.get_author_feed(USERNAME, limit = 100)
-
-def createSocialImg(post):
-    new_post, created = SocialImg.objects.update_or_create(
-        uri = post['uri'],
-        text = post['text'],
-        img_link = post['img_link'],
-        handle = post['handle'],
-        created_at = post['created_at'],
-        platform = 'bluesky',
-    )
-    print(created, new_post)
-
-tweets = []
-
-print(len(feed.feed))
-
-for post in feed.feed:
-    post = post.post
-
-    # print(post, "\n\n")
-    
-    # try:
-    #     ppr(post.embed.images[0].fullsize)
-    #     # ppr(post.embed.record.record.value.text.split("\n")[:2])
-    #     # ppr(post.embed.record.record.value.embed.external.uri.split("?")[0])
-    #     # ppr(post.embed.record.record.uri.split("feed.post/")[1])
-    # except Exception as e:
-    #     print("failed:", e)
-
-    if hasattr(post.embed, 'images'):
-        p = {}
-        p['img_link'] = post.embed.images[0].fullsize
-        p['text'] = " ".join(post.record.text.split("\n")[:2])
-        p['handle'] = post.author.handle
-        p['uri'] = post.uri.split("feed.post/")[1]
-        p['created_at'] = post.record.created_at
-
-        # ppr(p)
-        tweets.append(p)
-
-        try:
-            print('writing file')
-            createSocialImg(p)
-        except Exception as e:
-            ppr(post.embed)
-            print(e, "\nthis\n\n")

events/scrapers/Working/smedia/redsky.py

@@ -1,72 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-import praw
-
-from socials.models import SocialLink, SocialPost
-# from digitools import getBrowser, createURL, createBasicEvent, getSource
-
-tz = pytz.timezone("US/Central")
-
-
-# timestamp = 1729322547223
-# dt_object = datetime.datetime.fromtimestamp(timestamp)
-# print(dt_object)
-
-reddit = praw.Reddit(
-    client_id="rxW3Ywqke6FZDP7pIhYYuw",
-    client_secret="cg1VNl0I-RTuYUwgz16ryKh2wWKEcA",
-    password="7CTu4sGFi9E0",
-    user_agent="CultureClap",
-    username="cultureclap",
-)
-
-
-def createSocialLink(post):
-    new_post, created = SocialLink.objects.update_or_create(
-        text = post['text'],
-        link = post['link'],
-        handle = post['handle'],
-        likes = post['likes'],
-        replies = post['replies'],
-        platform = post['platform'],
-        created_at = post['created_at'],
-        rt_uri = 'blank',
-        rt_text = 'blank',
-        rt_link = 'blank',
-        rt_handle = 'blank',
-    )
-    print(created, new_post)
-
-count = 0
-
-for item in reddit.user.me().upvoted():
-    rdt = {}
-    rdt['text'] = item.title + " | " + item.selftext
-    rdt['handle'] = item.author.name
-    rdt['link'] = item.url
-    rdt['likes'] = item.ups
-    rdt['replies'] = len(item.comments.list())
-    rdt['created_at'] = datetime.fromtimestamp(item.created_utc)
-    rdt['platform'] = 'reddit'
-    
-    try:
-        print('writing file')
-        createSocialLink(rdt)
-        count +=1
-        if count > 50:
-            quit()
-    except Exception as e:
-        ppr(item)
-        print(e, "\nthis\n\n")
-    # ppr(item)

events/scrapers/Working/venues/AcmeComedy.Mpls.py

@@ -1,72 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-import events.digitools as digitools
-
-from events.models import Organization, Scraper, Calendar, Event
-
-venue, created = Organization.objects.get_or_create(
-     name="Acme Comedy Club",
-     city="Minneapolis",
-     website="https://acmecomedycompany.com/the-club/calendar/",
-     is_venue = True
-    )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
-
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%b %d %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/li[@class="event"]')
-    for c in contents:
-        try:
-            event = {}
-            day = c.xpath('.//*/span[@class="day"]/text()')[0]
-            month = c.xpath('.//*/span[@class="mth"]/text()')[0]
-            year = datetime.now().year
-            if month == "Jan":
-                year = int(year) + 1
-            event['scraper'] = scraper
-            event['calendars'] = [scraper.calendar]
-            event['title'] = c.xpath('.//*/span[@class="event_title"]/a/text()')[0]
-            event['date'] = [month, day, str(year),  c.xpath('.//*/span[@class="event_time"]/text()')[0].strip()]
-            event['date'] = " ".join(event['date'])
-            event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
-            event['link'] = c.xpath('.//*/span[@class="event_title"]/a/@href')[0]
-            digitools.createBasicEvent(event, "Co", venue)
-            scraper.items+=1
-        except Exception as e:
-            print(e)
-            ppr(event)
-            print("\n\n+++\n\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    br.close()
-    quit()
-
-links = digitools.createURL("https://acmecomedycompany.com/the-club/calendar/")  
-
-for link in links:
-    ps = digitools.getSource(br, link)
-    get_events(ps, "Co")
-
-digitools.updateScraper(scraper, item_count_start)
-br.close()

events/scrapers/Working/venues/Amsterdam.StPaul.py

@@ -1,67 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper, Calendar
-import events.digitools as digitools
-
-venue, created = Organization.objects.get_or_create(
-     name="Amsterdam Bar & Hall",
-     city="St. Paul",
-     website="https://www.amsterdambarandhall.com/events-new/",
-     is_venue=True
-    )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
-
-DATETIME_FORMAT = '%B %d  %Y %I:%M%p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps):
-    contents = ps.xpath('.//*/ul[@class="events-list"]/li')
-    for c in contents:
-        try:
-            event = {}
-            day = c.xpath('.//*/div[@class="date-day"]/text()')[0]
-            month = c.xpath('.//*/div[@class="date-month"]/text()')[0]
-            year = datetime.now().year
-            event['scraper'] = scraper
-            event['calendars'] = [scraper.calendar]
-            event['title'] = c.xpath('.//div/h4/a/text()')[0]
-            event['date'] = [month, day, str(year),  c.xpath('.//div[@class="event-info"]/p/text()')[0].split(" ")[0]]
-            event['date']  = " ".join(event['date'])
-            event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT)
-            event['link'] = c.xpath('.//div[@class="event-info"]/h4/a/@href')[0]
-            if " presents" in event['title']:
-                event['title'] = event['title'].split("presents")[1][1:].strip()
-            if event['title'].startswith('.'):
-                print("BLAHH\n")
-                event['title'] = event['title'][1:].strip()
-            digitools.createBasicEvent(event, "Mu", venue)
-            scraper.items+=1
-        except Exception as e:
-            print(e)
-
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-ps = digitools.getSource(br, venue.website)
-get_events(ps)
-sleep(3)
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/FirstAveScrape.py

@@ -1,178 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-
-from lxml import html
-import pytz
-
-from events.models import Organization, Scraper, Event
-import events.digitools as digitools
-
-venue, created = Organization.objects.get_or_create(
-     name="First Avenue",
-     city="Minneapolis",
-     website="https://first-avenue.com",
-     is_venue = True
-    )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%b %d %Y %I%p'
-DATETIME_FORMAT_2 = '%b %d %Y %I:%M%p'
-DATETIME_FORMAT_3 = '%b %d %Y'
-
-# Set initial variables for City, etc
-month = int(datetime.now().month)
-day = int(datetime.now().day)
-
-if month == 12:
-    next_month = "01"
-else:
-    next_month = month + 1
-    if next_month < 10:
-        next_month = "0" + str(next_month)
-
-if month < 10:
-    month = "0" + str(month)
-
-year = int(datetime.now().year)
-
-calendar_url = 'https://first-avenue.com/shows/?start_date=' + str(year) + str(month) + str(day)
-
-next_month_string = str(next_month) + "01"
-
-if next_month == 1:
-    calendar_url_2 = 'https://first-avenue.com/shows/?start_date=' + str(year + 1) + next_month_string
-else:
-    if int(next_month) == 1:
-        calendar_url_2 = 'https://first-avenue.com/shows/?start_date=' + str(year + 1) + next_month_string
-    else:
-        calendar_url_2 = 'https://first-avenue.com/shows/?start_date=' + str(year) + next_month_string
-
-
-print("\n\n", calendar_url, calendar_url_2, "\n\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-
-if datetime.now().day < 8:
-    ps = digitools.getSource(br, calendar_url)
-    shows = ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')[:63]
-elif 7 < datetime.now().day < 15:
-    ps = digitools.getSource(br, calendar_url)
-    shows = ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')
-elif 14 < datetime.now().day < 21:
-    ps = digitools.getSource(br, calendar_url)
-    shows = ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')[:95]
-    ps = digitools.getSource(br, calendar_url_2)
-    shows = shows +  ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')[:31]
-else:
-    ps = digitools.getSource(br, calendar_url)
-    shows = ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')
-
-    ps = digitools.getSource(br, calendar_url_2)
-    shows = shows +  ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')[:63]
-
-events = []
-
-def get_info(pse):
-    event = {}
-    event['scraper'] = scraper
-    event['calendars'] = [scraper.calendar]
-    event["venue"] = pse.xpath('.//*/div[@class="content"]/div/div[@class="venue_name"]/text()')[0].replace('\t', '').replace('\n', '').strip()
-    event["show_title"] = pse.xpath('.//*/span[@class="show_title"]/text()')[0].replace('\t', '').replace('\n', '')
-    if event["show_title"] == "":
-        event["show_title"] = pse.xpath('.//*/span[@class="show_title"]/text()')[2].replace('\t', '').replace('\n', '')
-    event["guests"] = pse.xpath('.//*/div[@class="feature_details_main d-flex align-items-center"]/div/h4/text()')
-    event["flyer"] = pse.xpath('.//*/img[@class="gig_poster lazy loaded"]/@src')
-    try:
-        event = get_date(pse, event)
-    except Exception as e:
-        print("date issue: ", e)
-    try:
-        event = get_details(pse, event)
-    except Exception as e:
-        print("details issue: ", e)
-    try:
-        event["date_time"] = datetime.strptime(" ".join(event["date"]) + " " + event["details"]["Doors Open"], DATETIME_FORMAT)
-    except Exception as e:
-        print("Using alt date format 2: ", e)
-        try:
-            event["date_time"] = datetime.strptime(" ".join(event["date"]) + " " + event["details"]["Doors Open"], DATETIME_FORMAT_2)
-            ppr(event)
-        except Exception as e:
-            print("Using alt date format 3: ", e)
-            print(event['date'])
-            event["date_time"] = datetime.strptime(" ".join(event["date"]), DATETIME_FORMAT_3)
-    return event
-
-def get_date(pse, event):
-    month = pse.xpath('.//*/div[@class="date_container"]/div/div[@class="month"]/text()')[0].replace('\t', '').replace('\n', '')
-    day = pse.xpath('.//*/div[@class="date_container"]/div/div[@class="day"]/text()')[0].replace('\t', '').replace('\n', '')
-    year = pse.xpath('.//*/div[@class="date_container"]/div/div[@class="year"]/text()')[0].replace('\t', '').replace('\n', '')
-    event["date"] = [month, day, year]
-    return event
-
-def get_details(pse, event):
-    try:
-        details = pse.xpath('.//*/div[@class="show_details text-center"]/div/div/h6/text()')
-        info = pse.xpath('.//*/div[@class="show_details text-center"]/div/div/h2/text()')
-        di = zip(details, info)
-        details = {}
-        for d,i in di:
-            details[d] = i
-        event["details"] = details 
-        return event
-    except Exception as e:
-        print("details issue: ", e)
-
-for show in shows:
-    br.get(show)
-    sleep(2)
-    try:
-        pse = html.fromstring(br.page_source)
-    except Exception as e:
-        print(show)
-        pass
-    try:
-        event = get_info(pse)
-    except Exception as e:
-        print("get_info error: ", e)
-    try:
-        event["link"] = show
-        if event["venue"] in ["Palace Theater", "Turf Club", "The Fitzgerald Theater", "Amsterdam Bar & Hall"]:
-            venue, created = Organization.objects.get_or_create(name=event["venue"], is_venue=True, city="St. Paul")
-        else:
-            venue, created = Organization.objects.get_or_create(name=event["venue"], is_venue=True, city="Minneapolis")
-    except Exception as e:
-        print("Venue creation error: ", e, "\n", event, "\n", event["venue"])
-    try:
-        event['dateStamp'] = event['date_time']
-        event['scraper'] = scraper
-        new_event, created = digitools.createDetailedEvent(event, "Mu", venue, scraper)
-        scraper.items+=1
-    except Exception as e:
-        print("event creation error: ", e, "\n\n", event, "\n\n", created) 
-        quit()
-
-ppr(events)
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)
-
-# br.find_element_by_class_name('fc-btn_allCalendars-button').click()

events/scrapers/Working/venues/GinkgoCoffee.stp.py

@@ -1,68 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-import events.digitools as digitools
-
-venue, created = Organization.objects.get_or_create(
-     name="Ginkgo Coffee",
-     city="Saint Paul",
-     website="https://ginkgocoffee.com/events/",
-     is_venue = True
-    )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
-
-event_type = ""
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT_2 = '%b %d %Y %I:%M %p'
-DATETIME_FORMAT = '%B %d @ %I:%M %p %Y'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/article')
-    for c in contents:
-        try:
-            event = {}
-            dateTime = c.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0]
-            month = c.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].split(' ')[0]
-            year = datetime.now().year
-            if month == "January":
-                year = int(year) + 1
-            event['scraper'] = scraper
-            event['calendars'] = [scraper.calendar]
-            event['title'] = c.xpath('.//*/h3/a/text()')[0].replace("\n", "").replace("\t", "")
-            event['date'] = " ".join([ dateTime, str(year)])
-            event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
-            event['link'] = c.xpath('.//*/h3/a/@href')[0]
-            digitools.createBasicEvent(event, event_type, venue)
-        except Exception as e:
-            print(e)
-            ppr(event)
-            print("\n\n+++\n\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-ps = digitools.getSource(br, venue.website)
-get_events(ps, "Mu")
-sleep(3)
-
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/GreenRoom.Mpls.py

@@ -1,71 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-import events.digitools as digitools
-
-
-venue, created = Organization.objects.get_or_create(
-     name="Green Room",
-     city="Minneapolis",
-     website="https://www.greenroommn.com/events",
-     is_venue = True
-    )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
-
-event_type = "Mu"
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%a %b %d %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/div[@class="vp-event-card vp-venue-greenroom vp-col"]')
-    for c in contents:
-        try:
-            event = {}
-            time = c.xpath('.//*/span[@class="vp-time"]/text()')[0].strip()
-            date = c.xpath('.//*/span[@class="vp-date"]/text()')[0].strip()
-            month = date.split(" ")[1]
-            year = datetime.now().year
-            # if month == "Jan":
-            #     year = int(year) + 1
-            event['scraper'] = scraper
-            event['calendars'] = [scraper.calendar]
-            event['title'] = c.xpath('.//*/div[@class="vp-event-name"]/text()')[0]
-            event['datetime'] = date + " " + str(year) + " " + time
-            event['dateStamp'] = datetime.strptime(event['datetime'], DATETIME_FORMAT)
-            event['link'] = venue.website + c.xpath('.//a[@class="vp-event-link"]/@href')[0]
-            digitools.createBasicEvent(event, event_type, venue)
-            scraper.items+=1
-        except Exception as e:
-            print(e)
-            ppr(event)
-            print("\n+++\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-ps = digitools.getSource(br, venue.website)
-get_events(ps, event_type)
-sleep(3)
-
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/HookLadderScrape.py

@@ -1,96 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper, Event
-import events.digitools as digitools
-
-
-from lxml import html
-
-count = 0
-
-venue, created = Organization.objects.get_or_create(
-            name="Hook & Ladder",
-            city="Minneapolis",
-            website="https://thehookmpls.com",
-            is_venue=True,
-            )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
-ppr(scraper)
-
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%a, %b %d, %Y @ %I:%M %p'
-
-# Set initial variables for City, etc
-calendar_url = [
-    "https://thehookmpls.com/events/list/page/1",
-    "https://thehookmpls.com/events/list/page/2",
-    "https://thehookmpls.com/events/list/page/3"
-]
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-
-def get_listings(pse, events):
-    nevents = pse.xpath('.//*/article')
-    for event in nevents:
-        e = {} 
-        e['datetime'] = event.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0]
-        e['show_title'] = event.xpath('.//*/header/h2/a/@title')[0]
-        e['link'] = event.xpath('.//*/header/h2/a/@href')[0]
-        try:
-            e['subtitle'] = event.xpath('.//*/header/div[@class="eventSubHead"]/text()')[0]
-        except:
-            continue
-        try:
-            e['price'] = event.xpath('.//*/span[@class="tribe-events-c-small-cta__price"]/strong/text()')[0].replace("Tickets ", "")
-        except:
-            e['price'] = "See Link"
-        e['image'] = event.xpath('.//*/img/@data-src')[0]
-        e["date_time"] = datetime.strptime(e['datetime'], DATETIME_FORMAT)
-        e['scraper'] = scraper
-        e['calendar'] = [scraper.calendar]
-        events.append(e)
-        
-events = []
-
-for cal in calendar_url:
-    br.get(cal)
-    sleep(3)
-    pse = html.fromstring(br.page_source)
-    get_listings(pse, events)
-
-for event in events:
-    try:
-        new_event = Event.objects.update_or_create(
-            scraper = event['scraper'],
-            event_type = 'Mu',
-            show_title = event["show_title"],
-            show_link = event["link"],
-            show_date = event["date_time"],
-            show_day = event["date_time"],
-            guests = " ".join(event["subtitle"]),
-            venue = venue
-        )
-        digitools.add_calendar(new_event, 'msp')
-    except Exception as e:
-        print("oops ", e, "\n\n", "Scraper:", scraper) 
-
-br.close()
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/MagersQuinn.py

@@ -1,70 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-import events.digitools as digitools
-
-venue, created = Organization.objects.get_or_create(
-     name="Magers & Quinn",
-     city="Minneapolis",
-     website="https://www.magersandquinn.com/events",
-     is_venue=False
-    )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
-
-DATETIME_FORMAT = '%A, %B %d , %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/div[@class="day  has-event"]')
-    for c in contents:
-        try:
-            event = {}
-            day = c.xpath('.//*/div[@class="dd"]/text()')[0]
-            month = c.xpath('.//*/div[@class="month"]/text()')[0]
-            year = c.xpath('.//*/div[@class="year"]/text()')[0]
-            event['scraper'] = scraper
-            event['calendars'] = [scraper.calendar]
-            event['title'] = c.xpath('.//*/h3/text()')[0]
-            event['date'] = [month, day, year,  c.xpath('.//*/p[@class="time"]/text()')[0]]
-            event['date']  = " ".join(event['date'])
-            event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT)
-            event['link'] = "https://www.magersandquinn.com" + c.xpath('.//a[@class="event in-store"]/@href')[0]
-            digitools.createBasicEvent(event, "Ed", venue)
-            scraper.items+=1
-        except Exception as e:
-            event['link'] = "https://www.magersandquinn.com" + c.xpath('.//a[@class="event off-site"]/@href')[0]
-            print("Error: ", e)
-            ppr(event)
-            digitools.createBasicEvent(event, "Ed", venue)
-            print("\n\n+++\n\n")
-
-
-links = digitools.createBasicURL("https://www.magersandquinn.com/events/")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-for link in links:
-    ps = digitools.getSource(br, link)
-    get_events(ps, "Ed")
-    sleep(3)
-# ppr(events)
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/MplsVFW.py

@@ -1,80 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper, Event
-import events.digitools as digitools
-
-
-from selenium.webdriver.common.by import By
-from lxml import html
-
-venue, created = Organization.objects.get_or_create(
-     name="Uptown VFW",
-     city="Minneapolis",
-     website="https://noboolpresents.com/venues/uptown-vfw/",
-     is_venue = True
-    )
-
-scraper,item_count_start, virtcal = digitools.getScraper(venue, 'msp')
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%a  %B %d @ %I:%M %p %Y'
-DATETIME_FORMAT_2 = '%b %d %I:%M%p %Y'
-DATETIME_FORMAT_3 = '%b %d %Y'
-# Set initial variables for City, etc
-calendar_url = 'https://noboolpresents.com/venues/uptown-vfw/'
-current_year = str(datetime.now().year)
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-    
-br.get(calendar_url)
-sleep(20)
-           
-def getEvents(br):
-    ps = html.fromstring(br.page_source)
-    events = ps.xpath('.//*/article')
-    for event in events:
-        deets = {}
-        dateTime = event.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].replace("•", "").strip() + " " + current_year
-        title = event.xpath('.//*/h2[@class="alt-font"]/a/text()')[0].replace("\n", "").replace("\t", "")
-        link = event.xpath('.//*/h2[@class="alt-font"]/a/@href')[0]
-        # deets["tickets"] = event.xpath('.//*/span[@class="tribe-events-c-small-cta__price"]/strong/text()')[0]
-        try:
-            new_event, created = Event.objects.update_or_create(
-                scraper = scraper,
-                event_type = 'Mu',
-                show_title = title,
-                show_link = link,
-                show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
-                show_day = datetime.strptime(dateTime, DATETIME_FORMAT),
-                # more_details = deets["tickets"],
-                venue = venue
-            )
-            digitools.add_calendar(new_event, 'msp')
-            scraper.items+=1
-        except Exception as e:
-            print("oops", e) 
-
-getEvents(br)
-br.find_element(By.XPATH, './/*/li[@class="tribe-events-c-nav__list-item tribe-events-c-nav__list-item--next"]/a').click()
-sleep(5)
-getEvents(br)
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/ParkwayTheater.py

@@ -1,106 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper, Event as DSEvent
-import events.digitools as digitools
-
-
-try:
-    venue, created = Organization.objects.get_or_create(
-        name="Parkway Theater",
-        city="Minneapolis",
-        website="https://theparkwaytheater.com",
-        is_venue = True
-        )
-except Exception as e:
-    venue = Organization.objects.get(name="Parkway Theater")
-
-scraper,item_count_start, virtcal = digitools.getScraper(venue, 'msp')
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%b %d, %Y %I:%M %p'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/div[@class="summary-content sqs-gallery-meta-container"]')
-    img_etc = ps.xpath('.//*/div[@class="summary-thumbnail-outer-container"]/a/div/img/@src')
-    ps.xpath('.//*/span[@class="event-time-12hr"]/text()')
-    for c,i in zip(contents,img_etc):
-        try:
-            event = {}
-            event['calendars'] = [scraper.calendar]
-            event['title'] = c.xpath('.//*/a[@class="summary-title-link"]/text()')[0]
-            event['link'] = "https://theparkwaytheater.com" + c.xpath('.//*/a[@class="summary-title-link"]/@href')[0]
-            event['date'] = c.xpath('.//div/div/time/text()')[0] + " " + c.xpath('.//*/span[@class="event-time-12hr"]/text()')[0].split("–")[0].strip()
-            event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
-            event['desc'] = c.xpath('.//*/p/text()')[0]
-            event['img_link'] = i
-            event['details'] = {
-                'description': event['desc'],
-                'img_link': event['img_link'],
-            }
-
-            try:
-                new_event, created = DSEvent.objects.update_or_create(
-                    scraper = scraper,
-                    event_type = event_type,
-                    show_title = event['title'],
-                    show_link = event['link'],
-                    show_date = datetime.strptime(event['date'], DATETIME_FORMAT),
-                    show_day = datetime.strptime(event['date'], DATETIME_FORMAT),
-                    more_details = event["details"],
-                    venue = venue
-                )
-                digitools.add_calendar(new_event, 'msp')
-                scraper.items+=1
-            except Exception as e:
-                try:
-                    event['date'] = c.xpath('.//div/div/time/text()')[0].split("–")[0] + " " + c.xpath('.//*/span[@class="event-time-12hr"]/text()')[0].split("–")[0].strip()
-                    event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
-                    new_event, created = DSEvent.objects.update_or_create(
-                        scraper = scraper,
-                        event_type = event_type,
-                        show_title = event['title'],
-                        show_link = event['link'],
-                        show_date = datetime.strptime(event['date'], DATETIME_FORMAT),
-                        show_day = datetime.strptime(event['date'], DATETIME_FORMAT),
-                        more_details = event["details"],
-                        venue = venue
-                    )
-                    digitools.add_calendar(new_event, 'msp')
-                    scraper.items+=1
-                except Exception as e:
-                    print(e)
-                    print("\n\n+++\n\n")
-        except Exception as e:
-            continue
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-calendar_url = 'https://theparkwaytheater.com/live-events'
-ps = digitools.getSource(br, calendar_url)
-get_events(ps, "Mu")
-
-calendar_url = "https://theparkwaytheater.com/movies"
-ps = digitools.getSource(br, calendar_url)
-get_events(ps, "Th")
-
-# ppr(events)
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/SPCO.stp.py

@@ -1,97 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-
-import events.digitools as digitools
-
-venue, created = Organization.objects.get_or_create(
-     name="St Paul Chamber Orchestra",
-     city="St Paul",
-     website="https://thespco.org",
-     is_venue = False
-    )
-
-scraper,item_count_start, virtcal = digitools.getScraper(venue, 'msp')
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%A, %B %d, %Y – %I:%M %p'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/div[@class="event-title"]/a/@href')
-    for c in set(contents):
-        try:
-            link = 'https://content.thespco.org' + c
-            ps = digitools.getSource(br, link)
-            ntitle = ps.xpath('.//*/article/h1/text()')
-            subtitle = ps.xpath('.//*/article/h1/em/text()')
-            event = {}
-            event['scraper'] = scraper
-            event['calendars'] = [scraper.calendar]
-            if len(subtitle) == 1:
-                if len(ntitle) == 2:
-                    title = ntitle[0] + subtitle[0] + ntitle[1]
-                elif ntitle[0].startswith(" "):
-                    title = subtitle[0] + ntitle[0]
-                else:
-                    title = ntitle[0] + subtitle[0]
-            else:
-                title = ntitle[0]
-
-            events = ps.xpath('.//*/div[@class="day"]')
-            for e in events:
-                new_venue = e.xpath('.//*/strong[@class="venue"]/text()')[0].strip()
-                location = e.xpath('.//*/span[@class="location"]/text()')[0].strip()
-                if 'Minneapolis' in location:
-                    location = 'Minneapolis'
-                elif 'St. Paul' in location:
-                    location = 'St. Paul'
-                else:
-                    location = location
-
-                venue, created = Organization.objects.get_or_create(
-                    name=new_venue,
-                    city=location,
-                    is_venue = True
-                    )
-
-                dateTime = e.xpath('.//*/h3[@class="date"]/text()')[0].replace("\n", "").replace("\t", "").strip()
-                event['dateStamp'] = datetime.strptime(dateTime, DATETIME_FORMAT)
-                event['venue'] = venue
-                event['location'] = location
-                event['title'] = "SPCO: " + title
-                event['link'] = link
-                event_type = "Mu"
-                digitools.createBasicEvent(event, event_type, venue)
-                scraper.items+=1
-        except Exception as e:
-            print("ERROR: ", e)
-            print("\n\n+++\n\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-# Get Event Page Link(s)
-links = digitools.createURLNoZero("https://content.thespco.org/events/calendar/")  
-
-for link in links:
-    ps = digitools.getSource(br, link)
-    get_events(ps, "Mu")
-    sleep(3)
-
-br.close()
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/WhiteSquirrelScrape.py

@@ -1,73 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-import events.digitools as digitools
-
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%Y-%m-%d %I:%M %p'
-
-venue, created = Organization.objects.get_or_create(
-     name="White Squirrel",
-     city="St. Paul",
-     website="https://whitesquirrelbar.com",
-     is_venue = True
-    )
-
-scraper,item_count_start, virtcal = digitools.getScraper(venue, 'msp')
-
-
-# Set initial variables for City, etc
-calendar_url = [
-    'https://whitesquirrelbar.com/calendar/list/page/1/',
-    'https://whitesquirrelbar.com/calendar/list/page/2/',
-    'https://whitesquirrelbar.com/calendar/list/page/3/'
-]
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-def get_listings(pse, events):
-    listings = pse.xpath('.//*/li[@class="tribe-common-g-row tribe-events-calendar-list__event-row"]')
-    print("get listings", listings)
-    for l in listings:
-        event = {}
-        event['scraper'] = scraper
-        event['calendars'] = [scraper.calendar]
-        print("1: ", event)
-        try:
-            event["image"] = l.xpath('.//*/img/@src')[0]
-        except:
-            event["image"] = "none"
-        event["date"] = l.xpath('.//time/@datetime')[0]
-        event["time"] = l.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].split("@")[1]
-        event["title"] = l.xpath('.//*/h3/a/text()')[0].replace("\t", "").replace("\n", "")
-        event["link"]  = l.xpath('.//*/h3/a/@href')[0]
-        event['datetime'] = event['date'] + " " + event['time']
-        event["dateStamp"] = datetime.strptime(event['datetime'] , DATETIME_FORMAT)
-        events.append(event)
-        digitools.createBasicEvent(event, "Mu", venue)
-        scraper.items+=1
-        
-events = []
-
-for cal in calendar_url:
-    ps = digitools.getSource(br, cal)
-    get_listings(ps, events)
-
-br.close()
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/cedar.mpls.py

@@ -1,74 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-import events.digitools as digitools
-
-venue, created = Organization.objects.get_or_create(
-     name="Cedar Cultural Center",
-     city="Minneapolis",
-     website="https://www.thecedar.org",
-     is_venue=True
-    )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-DATETIME_FORMAT_3 = '%A, %B %d at %I:%M%p %Y'
-DATETIME_FORMAT_4 = '%A, %B %d at %I%p %Y'
-DATETIME_FORMAT_5 = '%A, %B %d @%I%p %Y'
-
-def get_events(ps):
-    links = ps.xpath('.//*/div[@class="summary-title"]/a/@href')
-    for l in links:
-        if "cedar-news-blog" in l:
-            continue
-        pse = digitools.getSource(br, "https://www.thecedar.org" + l)
-        event = {}
-        event['scraper'] = scraper
-        event['calendars'] = [scraper.calendar]
-        event['link'] = "https://www.thecedar.org" + l
-        try:
-            time = pse.xpath('.//*/time[@class="event-time-localized-start"]/text()')[0]
-            date = pse.xpath('.//*/time[@class="event-date"]/text()')[0]
-            event['title'] = pse.xpath('.//*/h1[@class="eventitem-title"]/text()')[0]
-        except:
-            try:
-                time = pse.xpath('.//*/time[@class="event-time-localized"]/text()')[0]
-                date = pse.xpath('.//*/time[@class="event-date"]/text()')[0]
-                event['title'] = pse.xpath('.//*/h1[@class="eventitem-title"]/text()')[0]
-            except Exception as e:
-                print(e)
-                print("failed event: ", event)
-        dateStamp = date + " " + time 
-        event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
-        digitools.createBasicEvent(event, "Mu", venue)
-        scraper.items+=1
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-
-calendar_url = 'https://www.thecedar.org/listing'
-ps = digitools.getSource(br, calendar_url)
-get_events(ps)
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/club331Scrape.py

@@ -1,105 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from selenium.webdriver.common.by import By
-from lxml import html
-
-from events.models import Organization, Scraper, Event
-import events.digitools as digitools
-
-venue, created = Organization.objects.get_or_create(
-        name="Club 331",
-        city="Minneapolis",
-        website="https://331club.com",
-        is_venue=True,
-    )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%b %d %I%p %Y'
-DATETIME_FORMAT_2 = '%b %d %I:%M%p %Y'
-DATETIME_FORMAT_3 = '%b %d %Y'
-# Set initial variables for City, etc
-calendar_url = 'https://331club.com/#calendar'
-current_year = str(datetime.now().year)
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-br.get(calendar_url)
-sleep(3)
-
-br.find_element(By.CLASS_NAME, 'more_events').click()
-sleep(2)
-ps = html.fromstring(br.page_source)
-sleep(3)
-
-dates = ps.xpath('.//*/div[@class="event"]')
-dates = dates + ps.xpath('.//*/div[@class="event hidden"]')
-
-def process_times(times):
-    # print("Times: ", times)
-    time = []
-    for t in times:
-        t = t.replace("\n", "").replace("TBA", "")
-        if len(t) > 0 and t.endswith("pm"):
-            if "-" in t:
-                t = t.split("-")[0] + "pm"
-            time.append(t)
-    return time
-
-events = []           
-
-for d in dates:
-  event_date = d.xpath('.//div[@class="event-date"]/span/text()')[:2]
-  cols = d.xpath('.//div[@class="column"]')
-  for c in cols:
-    bands = c.xpath('.//p/a/text()')
-    links = c.xpath('.//p/a/@href')
-    time = process_times(c.xpath('.//p/text()'))
-    event = {}
-    event["datetime"] = event_date + time + [current_year] 
-    try:
-        event["date_time"] = datetime.strptime(" ".join(event["datetime"]), DATETIME_FORMAT)
-    except:
-        try:
-            event["date_time"] = datetime.strptime(" ".join(event["datetime"]), DATETIME_FORMAT_2)
-        except:
-            try:
-                event["date_time"] = datetime.strptime(" ".join(event["datetime"]), DATETIME_FORMAT_3)
-            except:
-                event["date_time"] = "Invalid"
-    event["bands"]  = (", ").join(bands)
-    if len(bands) > 0:
-        event['scraper'] = scraper
-        event['calendars'] = [scraper.calendar]
-        event['title'] = event["bands"]
-        event['date'] = event["date_time"]
-        event['dateStamp'] = event["date_time"]
-        event['link'] = "https://331club.com/#calendar"
-
-        try:
-            digitools.createBasicEvent(event, "Mu", venue)
-        except Exception as e:
-            print('oops', e)
-        events.append(event)   
-
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/venues/manual/eventario.py

@@ -1,169 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-import json
-
-from selenium.webdriver.common.by import By
-from lxml import html
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper, Calendar
-import events.digitools as digitools
-
-venue, created = Organization.objects.get_or_create(
-     name="Events Medellin",
-     city="Medellin",
-     website="https://eventario.co/events-category/social/",
-     is_venue=True
-    )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'mde')
-
-DATETIME_FORMAT = '%B %d  %Y %I:%M%p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-# with open('data.json') as f:
-#     totalLinks = json.load(f)
-
-def getLinks(br, url, links):
-    br.get(url)
-    sleep(2)
-    br.execute_script("window.scrollTo(0, window.scrollY + 1500)")
-    sleep(2)
-    x = 1
-    while x == 1:
-        try:
-            div = br.find_element(By.XPATH, ".//*/div[@class='feed-pagination flexify']/a")
-            div.click()
-            sleep(2)
-            br.execute_script("window.scrollTo(0, window.scrollY + 1100)")
-            sleep(2)
-        except:
-            x = 0
-    ps = html.fromstring(br.page_source)
-    newLinks = []
-    # newlinks = ps.xpath('.//*/div[@class="e-con-inner"]/*/a/@href')
-    events = ps.xpath('.//*/div[@class="e-con-inner"]')
-    for event in events:
-        e = {}
-        try:
-            e['link'] = event.xpath('.//*/a/@href')[0]
-            e['title'] = event.xpath('.//*/h3/a/text()')[0]
-            e['venue'] = event.xpath('.//*/ul/li/a/text()')[-1:][0].replace('\n', '').replace('\t', '')
-            # e['venue'] = event.xpath('.//*/ul/li/a/text()')[-1:][0]
-            e['venueLink'] = event.xpath('.//*/ul/li/a/@href')[1]
-            label= event.xpath('.//*/li[@class="elementor-repeater-item-46edd7d flexify ts-action"]/div/text()')
-            e['label'] = ''.join([x.replace('\t', '').replace('\n', '') for x in label]).strip()
-            newLinks.append(e)
-        except Exception as e:
-            print("Error: ", e)
-
-    links = links + newLinks
-    return links
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-urls = ["https://eventario.co/events-category/social/", "https://eventario.co/events-category/musica/", "https://eventario.co/events-category/cultura/"]
-
-allLinks = []
-
-for url in urls:
-    allLinks = getLinks(br, url, allLinks)
-
-totalLinks = list({v['title']:v for v in allLinks}.values())
-ppr(len(totalLinks))
-# sortedlinks = allLinks.sort()
-# ppr(sortedlinks)
-
-for event in totalLinks:
-    br.get(event['link'])
-    sleep(2)
-    ps = html.fromstring(br.page_source)
-    dateTime= ps.xpath('.//*/div[@class="elementor-element elementor-element-d9beb21 elementor-widget elementor-widget-heading"]/span[@class="elementor-heading-title elementor-size-default"]/text()')
-    event['dateTime'] = [x[3:].split('-')[0].strip() for x in dateTime]
-    locations = ps.xpath('.//*/div[@class="elementor-element elementor-element-f04aae3 elementor-widget__width-initial elementor-widget-mobile__width-initial elementor-widget elementor-widget-ts-advanced-list"]/*/li[@class="elementor-repeater-item-138dbed flexify ts-action"]/a/text()')
-    location = [x.replace('\t', '').replace('\n', '') for x in locations]
-    if len(location) == 2: 
-        event['location'] = "{0}, {1}".format(location[1], location[0])
-    else:
-        try:
-            event['location'] = location[0]
-        except:
-            event['location'] = 'n/a'
-    address= ps.xpath('.//*/ul[@class="flexify simplify-ul ts-advanced-list"]/li[@class="elementor-repeater-item-842568d flexify ts-action"]/div/text()')
-    try:
-        event['address'] = [x for x in address if 'Capacidad' not in x and '$' not in x][0]
-    except:
-        event['address'] = address
-    # ppr(event)
-
-sleep(2)
-br.close()
-
-data = totalLinks
-print("Set:", len(totalLinks))
-
-paisa = []
-
-for d in data:
-    if len(d['dateTime']) != 0:
-        if 'Familia' in d['label']:
-            d['category'] = 'Ot'
-        elif 'Comedia' in d['label']:
-            d['category'] = 'Co'
-        elif ('Magic' in d['title']) or ('Juegos' in d['label']):
-            d['category'] = 'Ot'
-        elif ('Conferencias' in d['label']) or ('Intercambio' in d['label']):
-            d['category'] = 'Ed'
-        else:
-            d['category'] = 'Mu'
-        if "Antioquia" in d['location']:
-            try:
-                d['city'] = d['location'].split(',')[0]
-                paisa.append(d)
-            except:
-                continue
-
-cal = Calendar.objects.get(shortcode='mde')
-
-for d in paisa:
-    d['dateStamp'] =datetime.strptime(d['dateTime'][0], DATETIME_FORMAT)
-    try:
-        nvenue, created = Organization.objects.get_or_create(
-            name=d['venue'],
-            city=d['city'],
-            website=d['venueLink'],
-            address_complete = d['address'], 
-            is_venue=True
-            )
-    except:
-        nvenue = Organization.objects.get(name=d['venue'])
-        nvenue.address_complete = d['address']
-        nvenue.save()
-    new_event, created = Event.objects.update_or_create(
-        event_type = d['category'],
-        show_title = d['title'],
-        show_link = d['link'],
-        show_date = d['dateStamp'],
-        show_day = d['dateStamp'],
-        scraper = scraper,
-        venue = nvenue
-    )
-    new_event.calendar.add(cal)
-    new_event.save()
-    print(new_event)
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/workshop/ComedyUnderground.py

@@ -1,64 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-
-from events.models import Event as DSEvent, Organization
-from digitools import getBrowser, createDashURL, createBasicEvent, getSource
-
-venue, created = Organization.objects.get_or_create(
-     name="Comedy Corner",
-     city="Minneapolis",
-     website="https://comedycornerunderground.com/calendar",
-    )
-
-calendar_url = "https://comedycornerunderground.com/calendar"
-
-DATETIME_FORMAT = '%m %d %I%p %Y'
-
-def get_events(ps, link):
-    contents = ps.xpath('.//*/td')
-    for c in contents:
-        try:
-            day_num = c.xpath('.//*/div[@class="marker-daynum"]/text()')[0]
-            events = c.xpath('.//*/li')
-            # print(events)
-            for e in events:
-                event = {}
-                print(link)
-                month = link.split("month=")[1].split("-")[0]
-                year = link.split("month=")[1].split("-")[1]
-                event['title'] = e.xpath('.//*/span[@class="item-title"]/text()')[0]
-                event['time'] = e.xpath('.//*/span[@class="item-time item-time--12hr"]/text()')[0].replace("\xa0", "")
-                event['link'] = "https://comedycornerunderground.com" + e.xpath('.//a/@href')[0]
-                event['date'] = str(day_num) + ' ' + event['time'] + 'm'
-                dateStamp = month + ' ' + event['date'] + ' ' + year
-                event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
-                createBasicEvent(event, 'Co')
-                print('\n\n++\n\n')
-        except Exception as e:
-            continue
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-links = createDashURL("https://comedycornerunderground.com/calendar?view=calendar&month=")
-
-for link in links:
-    ps = getSource(br, link)
-    get_events(ps, link)
-    sleep(5)
-
-# ppr(events)
-br.close()

events/scrapers/Working/workshop/cabooze.mn.py

@@ -1,74 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-import events.digitools as digitools
-
-venue, created = Organization.objects.get_or_create(
-     name="Cabooze",
-     city="Minneapolis",
-     website="https://www.cabooze.com/#/events",
-     is_venue=True
-    )
-
-scraper,item_count_start = digitools.getScraper(venue)
-
-event_type = ""
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%b %d %I:%M %p %Y'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-
-def get_events(ps, event_type):
-    print("Getting events ...")
-    contents = ps.xpath('.//*/div[@class="vp-event-row vp-widget-reset vp-venue-thecabooze"]')
-    ppr(contents)
-    for c in contents:
-        try:
-            event = {}
-            date = c.xpath('.//*/span[@class="vp-month-n-day"]/text()')[0]
-            print(date)
-            month = date.split(" ")[0]
-            time = c.xpath('.//*/span[@class="vp-time"]/text()')[0]
-            year = datetime.now().year
-            if month == "Jan":
-                year = int(year) + 1
-            event['calendar'] = scraper.calendar
-            event['title'] = c.xpath('.//*/div[@class="vp-event-name"]/text()')[0]
-            event['date'] = [date, time, str(year)]
-            event['date'] = " ".join(event['date'])
-            event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
-            event['link'] = "https://www.cabooze.com/" + c.xpath('.//a[@class="vp-event-link"]/@href')[0]
-            print("Event Dict Created")
-            ppr(event)
-            digitools.createBasicEvent(event, event_type, venue)
-            scraper.items+=1
-        except Exception as e:
-            print(e)
-            ppr(event)
-            print("\n\n+++\n\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-ps = digitools.getSource(br, venue.website)
-get_events(ps, "Mu")
-sleep(3)
-
-digitools.updateScraper(scraper, item_count_start)
-br.close()

events/scrapers/Working/workshop/dakota.mpls.py

@@ -1,82 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-import events.digitools as digitools
-
-count = 0
-
-venue, created = Organization.objects.get_or_create(
-     name="Dakota",
-     city="Minneapolis",
-     website="https://www.dakotacooks.com/events/",
-     is_venue=True
-    )
-
-scraper,item_count_start = digitools.getScraper(venue)
-
-event_type = ""
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%a %b %d, %Y • %I:%M%p'
-DATETIME_FORMAT_2 = '%a %b %d, %Y • %I%p'
-
-def get_events(ps, event_type):
-    links = ps.xpath('.//*/div[@class="wicked-event-title tooltipstered"]/a/@href')
-    links = set(links)
-    for l in links:
-        pse = getSource(br, l)
-        event = {}
-        event['scraper'] = scraper
-        event['calendar'] = scraper.calendar
-        event['link'] = l
-        try:
-            event['time'] = pse.xpath('.//*/span[@class="text-uppercase"]/text()')[0].strip()
-            event['dateStamp'] = datetime.strptime(event['time'], DATETIME_FORMAT) 
-            event['title'] = pse.xpath('.//*/div[@class="sidebar-group"]/h1/text()')[0]
-            # event['detail-headers'] = pse.xpath('.//*/ul[@class="eo-event-meta"]/li/strong/text()')
-            # event['details'] = pse.xpath('.//*/ul[@class="eo-event-meta"]/li/text()')
-
-        except:
-            try:
-                event['title'] = pse.xpath('.//*/div[@class="sidebar-group"]/h1/text()')[0]
-                event['dateStamp'] = datetime.strptime(event['time'], DATETIME_FORMAT_2)
-                event['time'] = pse.xpath('.//*/span[@class="text-uppercase"]/text()')[0]
-            except Exception as e:
-                print(e)
-                print("failed event: ", event)
-        ppr(event)
-        try:
-            digitools.createBasicEvent(event, "Mu", venue)
-            scraper.items+=1
-        except Exception as e:
-            print(e)
-            print('failed to create: ', event)
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-
-ps = digitools.getSource(br, venue.website + '?wicked_month=04&wicked_year=2025&wicked_view=month')
-get_events(ps, "Mu")
-sleep(1)
-
-
-digitools.updateScraper(scraper, item_count_start)
-
-br.close()

events/scrapers/Working/workshop/icehouse.mpls.py

@@ -1,78 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-import events.digitools as digitools
-
-count = 0
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%a, %b %d %Y %I:%M %p SHOW'
-DATETIME_FORMAT_2 = '%a, %b %d %Y %I:%M %p SHOW'
-
-venue, created = Organization.objects.get_or_create(
-     name="Icehouse",
-     city="Minneapolis",
-     website = "https://icehouse.turntabletickets.com",
-     is_venue = True
-    )
-
-scraper,item_count_start = digitools.getScraper(venue)
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/div[@class="performances whitespace-pre-line w-full md:w-3/4"]')
-    for c in contents:
-        try:
-            event = {}
-            event['scraper'] = scraper
-            event['calendar'] = scraper.calendar
-            event['title'] = c.xpath('.//*/h3[@class="text-3xl font-semibold font-heading mr-auto"]/text()')[0]
-            event['link'] = venue.website + c.xpath('.//*/a[@class="show-link"]/@href')[0]
-            event['date'] = c.xpath('.//*/h4[@class="day-of-week"]/text()')[0]
-            year = datetime.now().year
-            if "Brunch" in event['title']:
-                event['time'] = "11:00 AM SHOW"
-            else:
-                event['time'] = c.xpath('.//*/div[@class="performance-btn"]/button/text()')[0]
-
-            event['datetime'] = event['date'] + " " + str(year) + " " + event['time']
-            try:
-                event['dateStamp'] =datetime.strptime(event['datetime'], DATETIME_FORMAT)
-            except:
-                event['datetime'] = event['date'] + " " + str(year) + " " + "07:00 PM SHOW"
-                event['dateStamp'] =datetime.strptime(event['datetime'], DATETIME_FORMAT)
-                event['title'] = event['title'] + " (Time Estimated)"
-            try:
-                digitools.createBasicEvent(event, event_type, venue)
-                scraper.items+=1
-            except Exception as e:
-                print(e)
-                quit()
-                
-        except Exception as e:
-            ppr(event)
-            print(e)
-            quit()
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-ps = digitools.getSource(br, venue.website)
-get_events(ps, "Mu")
-
-# ppr(events)
-br.close()
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/Working/workshop/pillarforum.mpls.py

@@ -1,86 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-import events.digitools as digitools
-
-
-current_year = str(datetime.now().year)
-
-venue, created = Organization.objects.get_or_create(
-     name="Piller Forum",
-     city="Minneapolis",
-     website="https://www.pilllar.com/pages/events",
-     is_venue = True
-    )
-
-scraper,item_count_start = digitools.getScraper(venue)
-
-event_type = "Mu"
-
-# Time Signatures
-tz = pytz.timezone("US/Central")
-DATETIME_FORMAT = '%b. %d %Y %I:%M %p'
-DATETIME_FORMAT_night = '%b. %d %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%b. %d %Y %I:%Mam'
-
-def get_events(ps, event_type):
-    contents = ps.xpath('.//*/div[@class="sse-column sse-half sse-center"]')
-    for c in contents:
-        try:
-            event = {}
-            event['scraper'] = scraper
-            event['calendar'] = scraper.calendar
-            event['link'] = venue.website
-            # time = c.xpath('.//*/span[@class="vp-time"]/text()')[0].strip()
-            date = c.xpath('.//h1[@class="sse-size-64"]/text()')[0]
-            if len(date) > 1:
-                print(date)
-                year = datetime.now().year
-                event_date = date + " " + str(year)
-                event['title'] = c.xpath('.//p/span/b/text()')[0]
-                details = c.xpath('.//p/text()')
-                if 'Music' in details[-1]:
-                    event_time = c.xpath('.//p/text()')[-1].split("Music")[1].strip()
-                    event_type = "Mu"
-                    event_dt = event_date + " " + event_time + " PM"
-                    event['dateStamp'] = datetime.strptime(event_dt, DATETIME_FORMAT_night)
-                elif len(details) == 1:
-                    try:
-                        event_time = details[0].split("-")[0].strip()
-                        event_dt = event_date + " " + event_time + ' PM'
-                        event['dateStamp'] = datetime.strptime(event_dt, DATETIME_FORMAT_night)
-                        event_type = "Ot"
-                    except Exception as e:
-                        event_time = details[0].split("-")[0].strip()
-                        event_dt = event_date + " " + event_time
-                        event['dateStamp'] = datetime.strptime(event_dt, DATETIME_FORMAT_2)
-                        event_type = "Ot"
-            digitools.createBasicEvent(event, event_type, venue)
-        except Exception as e:
-            print(e)
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-ps = digitools.getSource(br, venue.website)
-get_events(ps, event_type)
-sleep(3)
-
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/run_scrapers.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-
-DIGIDIR=/var/www/digisnaxx.ado
-DJANGODIR=/var/www/digisnaxx.ado/api.digisnaxx
-EVENTSDIR=/var/www/digisnaxx.ado/api.digisnaxx/events
-FIXTURESDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/fixtures
-SCRAPEDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers
-SCRIPTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/scripts
-VENUESDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/venues
-ICALDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/iCal
-GOVTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/govt
-
-export DJANGO_SUPERUSER_EMAIL=canin@dreamfreely.org
-export DJANGO_SUPERUSER_USERNAME=canin
-export DJANGO_SUPERUSER_PASSWORD='hello123'
-
-cd $DIGIDIR
-source venv/bin/activate
-
-cd $DJANGODIR
-mv *.bak baks/
-cp db.sqlite3 $(date -d "today" +"%Y%m%d%H%M").sqlite.bak
-
-# python manage.py migrate
-
-# cd $FIXTURESDIR
-# django-admin loaddata orgs.json
-# django-admin loaddata promo.json
-
-cd $SCRIPTDIR
-python start_up.py
-
-bash run_venues.sh $1
-bash run_govt.sh $1
-bash run_ical.sh 
-# # bash run_media_update.sh
-# # bash run_news.sh $1
-
-# python clean_up.py
-# cd ../
-# deactivate
-
-# rm -rf ../*/__pycache__
-# rm -rf ../*/*/__pycache__

events/scrapers/scripts/clean_up.py

@@ -1,33 +0,0 @@
-import re, os, sys
-from datetime import datetime, timedelta
-
-from django.db.models import Count
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event, Organization
-
-new_time = datetime.now() - timedelta(days=1)
-right_bound_time = datetime.now() + timedelta(days=45)
-events = Event.objects.filter(show_date__lte=new_time)
-events1 = Event.objects.filter(show_date__gte=right_bound_time)
-
-for e in events:
-    e.delete()
-
-for e in events1:
-    e.delete()
-
-org_sin_events = Organization.objects.annotate(num_events = Count('event')).filter(num_events__lt=1)
-
-for o in org_sin_events:
-    print(o)
-
-# for org in org_sin_events:
-#     print(org)
-#     org.delete()
-
-print("completed and cleaned scrapes")

events/scrapers/scripts/run_govt.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-BASEDIR=/var/www/digisnaxx.ado/api.digisnaxx
-EVENTSDIR=/var/www/digisnaxx.ado/api.digisnaxx/events
-SCRIPTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/scripts
-GOVTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/govt
-
-cd $GOVTDIR
-for file in * 
-do
-  python "$file" $1
-  echo "SCRIPT COMPLETE"
-done
-
-cd $SCRIPTDIR
-pwd
-python clean_up.py

events/scrapers/scripts/run_ical.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-BASEDIR=/var/www/digisnaxx.ado/api.digisnaxx
-DJANGODIR=/var/www/digisnaxx.ado/api.digisnaxx/events
-SCRIPTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/scripts
-ICALDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/iCal
-
-cd $ICALDIR
-for file in * 
-do
-  python "$file"
-  echo "SCRIPT COMPLETE"
-done
-
-cd $SCRIPTDIR
-python clean_up.py

events/scrapers/scripts/run_media_update.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-ENVDIR=/var/www/digisnaxx.ado/
-DJANGODIR=/var/www/digisnaxx.ado/api.digisnaxx/events
-EVENTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers
-WORKMEDIADIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/smedia
-
-# ENVDIR=/home/canin/Downloads/DigiSnaxxEvents
-# DJANGODIR=/home/canin/Documents/repos/digisnaxx/events
-# WORKMEDIADIR=/home/canin/Documents/repos/digisnaxx/events/scrapers/Working/smedia
-
-cd $ENVDIR
-pwd
-source venv/bin/activate
-
-cd $WORKMEDIADIR
-
-python bluesky.py
-python bluesky_media.py
-python redsky.py
-
-deactivate

events/scrapers/scripts/run_news.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-BASEDIR=/var/www/digisnaxx.com/api.digisnaxx
-DJANGODIR=/var/www/digisnaxx.com/api.digisnaxx/events
-SCRIPTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/scripts
-NEWSDIR=/var/www/digisnaxx.com/api.digisnaxx/events/scrapers/Working/news
-
-cd $NEWSDIR
-for file in * 
-do
-  python "$file" $1
-  echo "SCRIPT COMPLETE"
-done
-
-cd $SCRIPTDIR
-python clean_up.py

events/scrapers/scripts/run_venues.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-BASEDIR=/var/www/digisnaxx.ado/api.digisnaxx
-DJANGODIR=/var/www/digisnaxx.ado/api.digisnaxx/events
-SCRIPTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/scripts
-VENUESDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/venues
-
-cd $VENUESDIR
-for file in * 
-do
-  python "$file" $1
-  echo "SCRIPT COMPLETE"
-done
-
-cd $SCRIPTDIR
-python clean_up.py

events/scrapers/scripts/start_up.py

@@ -1,29 +0,0 @@
-import re, os, sys
-from datetime import datetime, timedelta
-
-import django
-sys.path.append('../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event,Organization, Promo, Calendar
-
-# venue, created = Organization.objects.get_or_create(name="DreamFreely",
-#                                                     website="https://www.dreamfreely.org",
-#                                                     city="Minneapolis",
-#                                                     contact_name="Canin Carlos",
-#                                                     contact_email="canin@dreamfreely.org",
-#                                                     phone_number="6124054535")
-
-# print("Created DreamFreely:", created, venue)
-
-# calendar, created = Calendar.objects.get_or_create(name='Mpls-StP', shortcode='msp', desc='none')
-# calendar, created = Calendar.objects.get_or_create(name='Medellin', shortcode='mde', desc='none')
-# calendar, created = Calendar.objects.get_or_create(name='Online', shortcode='000', desc='none')
-
-events = Event.objects.all()
-
-for e in events:
-    e.delete()
-
-print("Events are cleared. Ready to scrape.")

events/scrapers/zArchive/FaceBook/BillPull.py

@@ -1,30 +0,0 @@
-for sE in senateEvents[:5]:
-    bills = sE.xpath('.//*/div[@class="mb-1"]/a/text()')
-    bill_link = sE.xpath('.//*/div[@class="mb-1"]/a/@href')
-    bill_items = zip(bills, bill_link)
-    print(bills)
-    for b,i in bill_items:
-        if b.startswith("S.F."):
-            print(b, i, "\n\n")
-
-
-
-
-import os
-from twilio.rest import Client
-
-
-# Find your Account SID and Auth Token at twilio.com/console
-# and set the environment variables. See http://twil.io/secure
-account_sid = os.environ['ACb416a0b2ed0a1be44c107b8bc1f683c5']
-auth_token = os.environ['33cae777f215a003deea6d4a0d5027c2']
-client = Client(account_sid, auth_token)
-
-message = client.messages \
-                .create(
-                     body="Join Earth's mightiest heroes. Like Kevin Bacon.",
-                     from_='+15017122661',
-                     to='+15558675310'
-                 )
-
-print(message.sid)

events/scrapers/zArchive/FaceBook/Mortimers.mpls.py

@@ -1,63 +0,0 @@
-import re, os, sys
-from datetime import datetime
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from events.models import Event, Organization
-
-from pprint import pprint as ppr
-
-from time import sleep
-from pprint import pprint as ppr
-from selenium import webdriver as wd
-from selenium.webdriver.common.by import By
-
-from xvfbwrapper import Xvfb
-from lxml import html
-import pytz
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%a, %b %d %Y'
-calendar_url = "https://www.facebook.com/mortimersmpls/events/"
-current_year = str(datetime.now().year)
-
-# Initiate and start the Browser
-br = wd.Firefox()
-
-
-
-br.get(calendar_url)
-sleep(10)
-br.find_element(By.XPATH, '//*/div[@class="x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x1ypdohk xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1o1ewxj x3x9cwd x1e5q0jg x13rtm0m x87ps6o x1lku1pv x1a2a7pz x9f619 x3nfvp2 xdt5ytf xl56j7k x1n2onr6 xh8yej3"]').click()
-print("Input Login Info")
-sleep(30)
-
-ps = html.fromstring(br.page_source)
-
-listings = ps.xpath('.//*/div[@class="x9f619 x1n2onr6 x1ja2u2z x78zum5 x2lah0s x1qughib x6s0dn4 xozqiw3 x1q0g3np x1pi30zi x1swvt13 xsag5q8 xz9dl7a x1n0m28w xp7jhwk x1wsgfga x9otpla"]')
-
-for l in listings:
-    gT = l.xpath('.//*/span/text()')
-    dateTime = gT[0]
-    show_title = gT[1]
-    link = l.xpath('.//*/a/@href')[0].split("?")[0] + " " + current_year
-    print(show_title, dateTime, link)
-    venue, created = Organization.objects.get_or_create(name="Mortimer's")
-    try:
-        new_event = Event.objects.update_or_create(
-            event_type = 'Mu',
-            show_title = show_title,
-            show_link = link,
-            show_date = datetime.strptime(dateTime.split(" AT")[0].strip(), DATETIME_FORMAT),
-            venue = venue
-        )
-    except Exception as e:
-        print(e, "\n\n++++\n\n")
-        continue
-
-
-br.close()

events/scrapers/zArchive/FaceBook/Mortimers.py

@@ -1,69 +0,0 @@
-import re, os, sys
-from datetime import datetime
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from events.models import Event, Organization
-
-from pprint import pprint as ppr
-
-from time import sleep
-from pprint import pprint as ppr
-from selenium import webdriver as wd
-from selenium.common.exceptions import TimeoutException
-from selenium.webdriver.support.ui import Select
-from selenium.webdriver.common.by import By
-
-from xvfbwrapper import Xvfb
-
-import requests
-from lxml import html
-
-import pytz
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%a, %b %d %Y'
-# Set initial variables for City, etc
-calendar_url = "https://www.facebook.com/mortimersmpls/events/"
-current_year = str(datetime.now().year)
-
-# Initiate and start the Browser
-br = wd.Firefox()
-
-br.get(calendar_url)
-sleep(10)
-br.find_element(By.XPATH, '//*/div[@class="x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x1ypdohk xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1o1ewxj x3x9cwd x1e5q0jg x13rtm0m x87ps6o x1lku1pv x1a2a7pz x9f619 x3nfvp2 xdt5ytf xl56j7k x1n2onr6 xh8yej3"]').click()
-print("Input Login Info")
-sleep(30)
-
-ps = html.fromstring(br.page_source)
-
-listings = ps.xpath('.//*/div[@class="x9f619 x1n2onr6 x1ja2u2z x78zum5 x2lah0s x1qughib x6s0dn4 xozqiw3 x1q0g3np x1pi30zi x1swvt13 xsag5q8 xz9dl7a x1n0m28w xp7jhwk x1wsgfga x9otpla"]')
-
-for l in listings:
-    gT = l.xpath('.//*/span/text()')
-    dateTime = gT[0]
-    show_title = gT[1]
-    link = l.xpath('.//*/a/@href')[0].split("?")[0] + " " + current_year
-    print(show_title, dateTime, link)
-    venue, created = Organization.objects.get_or_create(name="Mortimer's")
-    try:
-        new_event = Event.objects.update_or_create(
-            event_type = 'Mu',
-            show_title = show_title,
-            show_link = link,
-            show_date = datetime.strptime(dateTime.split(" AT")[0].strip(), DATETIME_FORMAT),
-            venue = venue
-        )
-    except Exception as e:
-        print(e, "\n\n++++\n\n")
-        continue
-
-
-br.close()
-
-

events/scrapers/zArchive/FaceBook/pillarforum.mpls.py

@@ -1,85 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-from selenium import webdriver as wd
-from selenium.webdriver.common.by import By
-
-from xvfbwrapper import Xvfb
-from lxml import html
-import pytz
-
-from events.models import Event as DSEvent, Organization
-from digitools import getBrowser, createURL, createBasicEvent, getSource
-
-
-exit()
-
-current_year = str(datetime.now().year)
-venue, created = Organization.objects.get_or_create(
-     name="Piller Forum",
-     city="Minneapolis",
-     website="https://www.pilllar.com/pages/events",
-    )
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%B %d %I:%M%p %Y'
-DATETIME_FORMAT = '%B %A %d %I:%M-%I:%M%p'
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-    br.get(venue.website)
-else:
-    print("No run_env")
-    quit()
-
-try:
-    br.find_element(By.XPATH, '//*[@class="privy-dismiss-content"]').click()
-except Exception as e:
-    print(e)
-    pass
-
-months = br.find_elements(By.XPATH, '//*[@class="sse-display"]')
-
-for month in months:
-    month_name = month.find_element(By.XPATH, './/*[@class="sse-size-28"]/u').text.capitalize()
-    events = month.find_elements(By.XPATH, './/p')
-    for event in events:
-        e = {}
-        eventTitle = event.text
-        try:
-            e['title'] = " ".join(eventTitle.split("-")[1].split(" ")[1:-2])
-            if 'Music' in eventTitle:
-                e['event_type'] = "Mu"
-            elif 'The Growth Arc' in eventTitle:
-                e['event_type'] = "Ot"
-                e['dateTime'] = " ".join([month_name, date, "7:00pm", current_year])
-                e['dateStamp'] = datetime.strptime(e['dateTime'], DATETIME_FORMAT)
-                e['title'] = "The Growth Arc - Relationship Support Space"
-                e['link'] = venue.website
-            elif 'Event' in eventTitle:
-                e['event_type'] = "Mu"
-            else:
-                e['event_type'] = "Ot"
-            date = eventTitle.split(":")[0].split(" ")[1].replace("th", "").replace("nd", "").replace("rd", "").replace("st", "")
-            time = eventTitle.split("-")[1].split(" ")[-2:][0]
-            e['dateTime'] = " ".join([month_name, date, time, current_year])
-            e['dateStamp'] = datetime.strptime(e['dateTime'], DATETIME_FORMAT)
-            e['link'] = venue.website
-            createBasicEvent(e, venue)
-        except Exception as e:
-            print("error ", eventTitle)
-            print(e)
-
-sleep(3)
-# ppr(events)
-br.close()

events/scrapers/zArchive/bluesky_scrape_old.py

@@ -1,61 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-from atproto import Client
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from socials.models import SocialLink
-# from digitools import getBrowser, createURL, createBasicEvent, getSource
-
-tz = pytz.timezone("US/Central")
-
-USERNAME = "dreamfreely.org"
-PASSWORD = "Futbol21!@"
-
-client = Client()
-client.login(USERNAME, PASSWORD)
-feed = client.get_author_feed(USERNAME, limit = 100)
-
-def createSocialLink(post):
-    new_post, created = SocialLink.objects.update_or_create(
-        cid = post['link_id'],
-        uri = post['uri'],
-        text = post['text'],
-        link = post['link'],
-        handle = post['handle'],
-        likes = post['likes'],
-        reposts = post['reposts'],
-        quotes = post['quotes'],
-        replies = post['replies'],
-        created_at = post['created_at']
-    )
-    print(created, new_post)
-
-for post in feed.feed:
-    post = post.post
-    if hasattr(post.record.embed, 'external'):
-        p = {}
-        p['link'] = post.record.embed.external.uri.split("?")[0]
-        p['text'] = " ".join(post.record.text.split("\n")[:2])
-        p['handle'] = post.author.handle
-        p['link_id'] = post.uri.split("feed.post/")[-1]
-        p['uri'] = post.uri
-        p['likes'] = post.like_count
-        p['quotes'] = post.quote_count
-        p['replies'] = post.reply_count
-        p['reposts'] = post.repost_count
-        p['created_at'] = post.record.created_at
-        
-        try:
-            createSocialLink(p)
-        except Exception as e:
-            print(e)

events/scrapers/zArchive/broken/BirchbarkBooks.py

@@ -1,82 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-
-from events.models import Event as DSEvent, Organization
-from digitools import getBrowser, createURL, createBasicEvent, getSource
-
-current_year = str(datetime.now().year)
-
-venue, created = Organization.objects.get_or_create(
-     name="Birchbark Books",
-     city="Minneapolis",
-     website="https://birchbarkbooks.com/pages/events",
-    )
-
-DATETIME_FORMAT = '%A, %B %d @ %I:%M%p %Y'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-DATETIME_FORMAT_3 = '%A, %B %d at %I:%M%p %Y'
-DATETIME_FORMAT_4 = '%A, %B %d at %I%p %Y'
-DATETIME_FORMAT_5 = '%A, %B %d @%I%p %Y'
-
-def get_events(ps):
-    contents = ps.xpath('.//*/div[@class="feature-row"]')
-    # ppr("contents:", contents)
-    for c in contents:
-        try:
-            event = {}
-            event['title'] = c.xpath('.//*/p[@class="h3"]/text()')[0].strip()
-            event['link'] = "https://birchbarkbooks.com/pages/events"
-            event['date'] = c.xpath('.//*/p[@class="accent-subtitle"]/text()')[0].replace("Central", "") + " " + current_year
-            event['date_num'] = event['date'].split(" ")[2].replace("th", "").replace("st", "").replace("rd", "").replace("nd", "")
-            event['date'] = event['date'].split(" ")
-            event['date'][2] = event['date_num']
-            event['date']  = " ".join(event['date']) 
-            event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT)
-            createBasicEvent(event, "Ed", venue)
-        except Exception as e:
-            try:
-                print(e)
-                event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_2)
-                createBasicEvent(event, "Ed", venue)
-                print("\n\n+++\n\n")
-            except Exception as e:
-                try:
-                    print(e)
-                    event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_3)
-                    createBasicEvent(event, "Ed", venue)
-                    print("\n\n+++\n\n")
-                except Exception as e:
-                    try:
-                        print(e)
-                        event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_4)
-                        createBasicEvent(event, "Ed", venue)
-                        print("\n\n+++\n\n")
-                    except Exception as e:
-                        print(e)
-                        event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_5)
-                        createBasicEvent(event, "Ed", venue)
-                        print("\n\n+++\n\n")
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-calendar_url = 'https://birchbarkbooks.com/pages/events'
-
-ps = getSource(br, calendar_url)
-get_events(ps)
-
-# ppr(events)
-br.close()

events/scrapers/zArchive/broken/EastsideLibrary.py

@@ -1,70 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper
-import events.digitools as digitools
-
-current_year = str(datetime.now().year)
-
-venue, created = Organization.objects.get_or_create(
-     name="Eastside Freedom Library",
-     city="Minneapolis",
-     website="https://eastsidefreedomlibrary.org/events/",
-     is_venue=True
-    )
-
-scraper,item_count_start = digitools.getScraper(venue)
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%B %d @ %I:%M %p %Y'
-
-def get_events(ps):
-    contents = ps.xpath('.//*/article')
-    # ppr("contents:", contents)
-    for c in contents:
-        try:
-            event = {}
-            event['scraper'] = scraper
-            event['calendar'] = scraper.calendar
-            event['title'] = c.xpath('.//*/h3[@class="tribe-events-calendar-list__event-title tribe-common-h6 tribe-common-h4--min-medium"]/a/text()')[0].strip()
-            event['link'] = c.xpath('.//*/h3[@class="tribe-events-calendar-list__event-title tribe-common-h6 tribe-common-h4--min-medium"]/a/@href')[0]
-            event['date'] = c.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].strip() + " " + current_year
-            event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT)
-            try:
-                new_event = digitools.createBasicEvent(event, "Ed", venue)
-                scraper.items+=1
-            except Exception as e:
-                print(e)
-                ppr(event)
-                print("\n+++\n")
-        except Exception as e:
-            print(e)
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = digitools.getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-calendar_url = 'https://eastsidefreedomlibrary.org/events/'
-
-ps = digitools.getSource(br, calendar_url)
-
-get_events(ps)
-
-# ppr(events)
-br.close()
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/zArchive/broken/FaceBook/BillPull.py

@@ -1,30 +0,0 @@
-for sE in senateEvents[:5]:
-    bills = sE.xpath('.//*/div[@class="mb-1"]/a/text()')
-    bill_link = sE.xpath('.//*/div[@class="mb-1"]/a/@href')
-    bill_items = zip(bills, bill_link)
-    print(bills)
-    for b,i in bill_items:
-        if b.startswith("S.F."):
-            print(b, i, "\n\n")
-
-
-
-
-import os
-from twilio.rest import Client
-
-
-# Find your Account SID and Auth Token at twilio.com/console
-# and set the environment variables. See http://twil.io/secure
-account_sid = os.environ['ACb416a0b2ed0a1be44c107b8bc1f683c5']
-auth_token = os.environ['33cae777f215a003deea6d4a0d5027c2']
-client = Client(account_sid, auth_token)
-
-message = client.messages \
-                .create(
-                     body="Join Earth's mightiest heroes. Like Kevin Bacon.",
-                     from_='+15017122661',
-                     to='+15558675310'
-                 )
-
-print(message.sid)

events/scrapers/zArchive/broken/FaceBook/Mortimers.mpls.py

@@ -1,63 +0,0 @@
-import re, os, sys
-from datetime import datetime
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from events.models import Event, Organization
-
-from pprint import pprint as ppr
-
-from time import sleep
-from pprint import pprint as ppr
-from selenium import webdriver as wd
-from selenium.webdriver.common.by import By
-
-from xvfbwrapper import Xvfb
-from lxml import html
-import pytz
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%a, %b %d %Y'
-calendar_url = "https://www.facebook.com/mortimersmpls/events/"
-current_year = str(datetime.now().year)
-
-# Initiate and start the Browser
-br = wd.Firefox()
-
-
-
-br.get(calendar_url)
-sleep(10)
-br.find_element(By.XPATH, '//*/div[@class="x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x1ypdohk xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1o1ewxj x3x9cwd x1e5q0jg x13rtm0m x87ps6o x1lku1pv x1a2a7pz x9f619 x3nfvp2 xdt5ytf xl56j7k x1n2onr6 xh8yej3"]').click()
-print("Input Login Info")
-sleep(30)
-
-ps = html.fromstring(br.page_source)
-
-listings = ps.xpath('.//*/div[@class="x9f619 x1n2onr6 x1ja2u2z x78zum5 x2lah0s x1qughib x6s0dn4 xozqiw3 x1q0g3np x1pi30zi x1swvt13 xsag5q8 xz9dl7a x1n0m28w xp7jhwk x1wsgfga x9otpla"]')
-
-for l in listings:
-    gT = l.xpath('.//*/span/text()')
-    dateTime = gT[0]
-    show_title = gT[1]
-    link = l.xpath('.//*/a/@href')[0].split("?")[0] + " " + current_year
-    print(show_title, dateTime, link)
-    venue, created = Organization.objects.get_or_create(name="Mortimer's")
-    try:
-        new_event = Event.objects.update_or_create(
-            event_type = 'Mu',
-            show_title = show_title,
-            show_link = link,
-            show_date = datetime.strptime(dateTime.split(" AT")[0].strip(), DATETIME_FORMAT),
-            venue = venue
-        )
-    except Exception as e:
-        print(e, "\n\n++++\n\n")
-        continue
-
-
-br.close()

events/scrapers/zArchive/broken/FaceBook/Mortimers.py

@@ -1,69 +0,0 @@
-import re, os, sys
-from datetime import datetime
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from events.models import Event, Organization
-
-from pprint import pprint as ppr
-
-from time import sleep
-from pprint import pprint as ppr
-from selenium import webdriver as wd
-from selenium.common.exceptions import TimeoutException
-from selenium.webdriver.support.ui import Select
-from selenium.webdriver.common.by import By
-
-from xvfbwrapper import Xvfb
-
-import requests
-from lxml import html
-
-import pytz
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%a, %b %d %Y'
-# Set initial variables for City, etc
-calendar_url = "https://www.facebook.com/mortimersmpls/events/"
-current_year = str(datetime.now().year)
-
-# Initiate and start the Browser
-br = wd.Firefox()
-
-br.get(calendar_url)
-sleep(10)
-br.find_element(By.XPATH, '//*/div[@class="x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x1ypdohk xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1o1ewxj x3x9cwd x1e5q0jg x13rtm0m x87ps6o x1lku1pv x1a2a7pz x9f619 x3nfvp2 xdt5ytf xl56j7k x1n2onr6 xh8yej3"]').click()
-print("Input Login Info")
-sleep(30)
-
-ps = html.fromstring(br.page_source)
-
-listings = ps.xpath('.//*/div[@class="x9f619 x1n2onr6 x1ja2u2z x78zum5 x2lah0s x1qughib x6s0dn4 xozqiw3 x1q0g3np x1pi30zi x1swvt13 xsag5q8 xz9dl7a x1n0m28w xp7jhwk x1wsgfga x9otpla"]')
-
-for l in listings:
-    gT = l.xpath('.//*/span/text()')
-    dateTime = gT[0]
-    show_title = gT[1]
-    link = l.xpath('.//*/a/@href')[0].split("?")[0] + " " + current_year
-    print(show_title, dateTime, link)
-    venue, created = Organization.objects.get_or_create(name="Mortimer's")
-    try:
-        new_event = Event.objects.update_or_create(
-            event_type = 'Mu',
-            show_title = show_title,
-            show_link = link,
-            show_date = datetime.strptime(dateTime.split(" AT")[0].strip(), DATETIME_FORMAT),
-            venue = venue
-        )
-    except Exception as e:
-        print(e, "\n\n++++\n\n")
-        continue
-
-
-br.close()
-
-

events/scrapers/zArchive/broken/acadia.py

@@ -1,72 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-
-from events.models import Event as DSEvent, Organization
-from digitools import getBrowser, createDashURL, createBasicEvent, getSource
-
-venue, created = Organization.objects.get_or_create(
-     name="Acadia Cafe",
-     city="Minneapolis",
-     website="https://acadiacafe.com",
-    )
-
-calendar_url = "https://www.acadiacafe.com/events"
-
-DATETIME_FORMAT = '%d %m %Y %I:%M%p'
-
-def get_events(ps, link):
-    contents = ps.xpath('.//*/div[@class="cl-view-month__day__event__title"]')
-    print(contents)
-    quit()
-
-    for c in contents:
-        try:
-            day_num = c.xpath('.//*/div[@class="marker-daynum"]/text()')[0]
-            events = c.xpath('.//*/li')
-            # print(events)
-            for e in events:
-                event = {}
-                event['month'] = link.split("month=")[1].split("-")[0]
-                event['year'] = link.split("month=")[1].split("-")[1]
-                event['title'] = e.xpath('.//h1/a[@class="flyoutitem-link"]/text()')
-                event['time'] = e.xpath('.//div[@class="flyoutitem-datetime flyoutitem-datetime--12hr"]/text()')
-                event['link'] = e.xpath('.//a/@href')[0]
-                event['date'] = str(day_num) + ' ' + 'time'
-                # event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
-                if len(event['title']):
-                    nevent = {}
-                    nevent['title'] = "".join(event['title']).strip()
-                    event['time'] = event['time'][0].strip().split(" –")[0]
-                    nevent['link'] = "https://palmers-bar.com" + e.xpath('.//a/@href')[0]
-                    event['dateStamp'] = str(day_num) + ' ' + event['month'] + ' ' + event['year'] + ' ' + event['time']
-                    nevent['dateStamp'] = datetime.strptime(event['dateStamp'], DATETIME_FORMAT)
-                    createBasicEvent(nevent, 'Mu', venue)
-        except Exception as e:
-            continue
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-
-
-
-ps = getSource(br, calendar_url)
-sleep(5)
-get_events(ps, calendar_url)
-sleep(5)
-
-# ppr(events)
-br.close()

events/scrapers/zArchive/broken/cedar.mpls.py

@@ -1,64 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-from selenium import webdriver as wd
-
-from xvfbwrapper import Xvfb
-from lxml import html
-import pytz
-
-from events.models import Event as DSEvent, Organization
-from digitools import getBrowser, createBasicEvent, getSource
-
-venue, created = Organization.objects.get_or_create(
-     name="Cedar Cultural Center",
-     city="Minneapolis",
-     website="https://www.thecedar.org/listing",
-    )
-
-tz = pytz.timezone("US/Central")
-
-DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p'
-DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
-DATETIME_FORMAT_3 = '%A, %B %d at %I:%M%p %Y'
-DATETIME_FORMAT_4 = '%A, %B %d at %I%p %Y'
-DATETIME_FORMAT_5 = '%A, %B %d @%I%p %Y'
-
-def get_events(ps):
-    links = ps.xpath('.//*/div[@class="summary-title"]/a/@href')
-    # ppr("contents:", contents)
-    for l in links:
-        br.get("https://www.thecedar.org" + l)
-        sleep(3)
-        pse = html.fromstring(br.page_source)
-        event = {}
-        time = pse.xpath('.//*/time[@class="event-time-12hr-start"]/text()')[0]
-        date = pse.xpath('.//*/time[@class="event-date"]/text()')[0]
-        event['title'] = pse.xpath('.//*/h1[@class="eventitem-title"]/text()')[0]
-        dateStamp = date + " " + time 
-        event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
-        event['link'] = "https://www.thecedar.org" + l
-        createBasicEvent(event, "Mu", venue)
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-
-
-calendar_url = 'https://www.thecedar.org/listing'
-ps = getSource(br, calendar_url)
-get_events(ps)
-# ppr(events)
-br.close()

events/scrapers/zArchive/broken/digitools.py

@@ -1,117 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-from time import sleep
-import pytz
-from lxml import html
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from xvfbwrapper import Xvfb
-from selenium import webdriver as wd
-
-from events.models import Event as DSEvent, Organization
-
-tz = pytz.timezone("US/Central")
-td = relativedelta.relativedelta(months=1)
-odt = datetime.now() + td
-
-
-def getSource(browser, link):
-    browser.get(link)
-    sleep(5)
-    ps = html.fromstring(browser.page_source)
-    return ps
-
-def getBrowser(run_env):
-    if run_env == 'dev':
-        print("Chrome is a go!")
-        # chromeOptions = wd.ChromeOptions()
-        # chromeOptions.binary_location = "/Application/Google\ Chrome.app"
-        # chromeDriver = "/opt/homebrew/bin/chromedriver"
-        # br = wd.Chrome(chromeDriver, options=chromeOptions)
-        br = wd.Chrome()
-        return br
-    elif run_env == "def":
-        print("Firefox go vroom")
-        br = wd.Firefox()
-        return br
-    elif run_env == "prod":
-        start_cmd = "Xvfb :91 && export DISPLAY=:91 &"
-        xvfb = Xvfb()
-        os.system(start_cmd)
-        xvfb.start()
-        print("started Xvfb")
-        br = wd.Firefox()
-        return br
-    else:
-        print("Failed", sys.argv, arg1)
-        quit()
-
-def createBasicURL(site_url):
-    month = datetime.now().month
-    next_month = odt.month
-    year = datetime.now().year
-    print(month, next_month, year)
-    links = [
-        site_url + str(month) + "/" + str(year),
-        site_url + str(next_month) + "/" + str(year)
-    ]
-    print(links)
-    return links
-
-def createURL(site_url):
-    month = datetime.now().month
-    if month < 10:
-        month = "0" + str(month)
-    else:
-        month = str(month)
-    next_month = odt.month
-    if next_month < 10:
-        next_month = "0" + str(next_month)
-    else:
-        next_month = str(next_month)
-    year = datetime.now().year
-    links = [
-        site_url + str(year) + "/" + month,
-    ]
-    if next_month == "01":
-        links.append(site_url + str(int(year)+1) + "/" + next_month)
-    else:
-        links.append(site_url + str(year) + "/" + next_month)
-    print(links)
-    return links
-
-def createDashURL(site_url):
-    month = datetime.now().month
-    if month < 10:
-        month = "0" + str(month)
-    else:
-        month = str(month)
-    next_month = odt.month
-    if next_month < 10:
-        next_month = "0" + str(next_month)
-    else:
-        next_month = str(next_month)
-    year = datetime.now().year
-    print(month, next_month, year)
-    links = [
-        site_url + month + "-"  + str(year),
-        site_url + next_month + "-"  + str(year) 
-    ]
-    print(links)
-    return links
-
-def createBasicEvent(event, event_type, venue):
-    new_event, created = DSEvent.objects.update_or_create(
-        event_type = event_type,
-        show_title = event['title'],
-        show_link = event['link'],
-        show_date = event['dateStamp'],
-        show_day = event['dateStamp'],
-        venue = venue
-    )
-    print("New Event: ", new_event)

events/scrapers/zArchive/broken/ical.TriviaMafia.py.bak

@@ -1,139 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-
-from datetime import datetime
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from events.models import Event as DSEvent, Organization
-from dateutil import relativedelta
-
-
-def createEvent(event, Organization, event_type):
-    new_event, created = DSEvent.objects.update_or_create(
-                        event_type = event_type,
-                        show_title = event['strSummary'],
-                        show_link = venue.website,
-                        show_date = event['dateStart']-td,
-                        show_day = event['dateStart']-td,
-                        more_details = event["details"],
-                        venue = venue
-                    )
-    return new_event, created
-
-
-def createVenue(event):
-    venue, created = Organization.objects.get_or_create(
-        name = event['venue'],
-        address = event['address'],
-        city = event['city'],
-    )
-    return venue, created
-
-td = relativedelta.relativedelta(hours=5)
-
-
-event_type = "Ot"
-
-calendar_url = 'https://calendar.google.com/calendar/ical/c_g1i6cbb3glhu6or0hu8kemah7k%40group.calendar.google.com/public/basic.ics'
-
-objIcalData = requests.get(calendar_url)
-
-gcal = iCalendar.from_ical(objIcalData.text)
-
-cfpa_events = []
-tz = pytz.timezone("US/Central")
-
-for component in gcal.walk():
-    event = {}
-    event['strSummary'] = f"{(component.get('SUMMARY'))}"
-    event['strDesc'] = component.get('DESCRIPTION')
-    event['strLocation'] = component.get('LOCATION')
-    event['dateStart'] = component.get('DTSTART')
-    event['dateStamp'] = component.get('DTSTAMP')
-    event['RepeatRule'] = component.get('RRULE')
-    if event['dateStamp'] is not None:
-        event['dateStamp'] = event['dateStamp'].dt
-    if event['dateStart'] is not None:
-        try:
-            event['dateStart'] = event['dateStart'].dt
-        except Exception as e:
-             event['dateStart'] = event['dateStart'].dt
-    event['dateEnd'] = (component.get('DTEND'))
-    if event['dateEnd'] is not None:
-            event['dateEnd'] = event['dateEnd'].dt
-    else:
-        event['dateEnd'] = event['dateStart']
-    if event['strSummary'] != 'None':
-        event['details'] = {
-             "description" : event['strDesc'], 
-             "Location" : event['strLocation'],
-        }
-        try:
-            event['venue'] = event['strLocation'].split(",")[0].strip()
-            event['address'] = event['strLocation'].split(",")[1].strip()
-            event['city'] = event['strLocation'].split(",")[2].strip()
-            try:
-                event['state'] = event['strLocation'].split(",")[3].split(' ')[1]
-                event['zip'] = event['strLocation'].split(",")[3].split(' ')[2]
-            except Exception as error:
-                pass
-        except Exception as error:
-            pass
-        cfpa_events.append(event)
-        # print(event)
-        now_now = datetime.now().astimezone(pytz.utc)
-        try:
-            if event['dateStart'] > now_now:
-                if not event['address'][0].isdigit():
-                    continue
-                venue, created = createVenue(event)
-                new_event, created = createEvent(event, venue, event_type)
-                print(new_event)  
-        except Exception as e:
-            try:
-                if event['dateStart'] > now_now.date():
-                    if not event['address'][0].isdigit():
-                        continue
-                    venue, created = createVenue(event)
-                    new_event, created = createEvent(event, venue, event_type)
-                    print(new_event) 
-            except Exception as e:
-                ppr(event)
-                print(e)
-
-
-# new_events = []
-# for event in cfpa_events:
-#     now_now = datetime.now().astimezone(pytz.utc)
-#     try:
-#         if event['dateStart'] > now_now:
-#             new_events.append(event)
-#     except Exception as e:
-#         try:
-#             if event['dateStart'] > now_now.date():
-#                 new_events.append(event)
-#         except Exception as e:
-#             print(e)
-#             ppr(event)
-
-
-# {'dateEnd': datetime.datetime(2022, 10, 22, 18, 30, tzinfo=<UTC>),
-#  'dateStamp': datetime.datetime(2023, 3, 23, 1, 57, 45, tzinfo=<UTC>),
-#  'dateStart': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
-#  'details': {'DateTime': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
-#              'Location': vText('b'''),
-#              'description': None},
-#  'strDesc': None,
-#  'strLocation': vText('b'''),
-#  'strSummary': 'Nia Class with Beth Giles'}
-
-

events/scrapers/zArchive/broken/ical_run.KJHideaway.StPaul.py

@@ -1,65 +0,0 @@
-import requests, os, sys
-from icalendar import Calendar as iCalendar, Event
-from datetime import datetime
-
-from pprint import pprint as ppr
-import pytz
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from events.models import Event as DSEvent, Organization, Scraper, Calendar
-import events.digitools as digitools
-
-from dateutil import relativedelta
-td = relativedelta.relativedelta(hours=5)
-
-venue, created = Organization.objects.get_or_create(
-     name="KJ's Hideaway",
-     city="Minneapolis",
-     website="",
-    )
-    
-try:
-    scraper, created = Scraper.objects.get_or_create(
-        name="KJ's Hideaway",
-        website="https://calendar.google.com/calendar/ical/sgmok5t13vspeoruhruh33dhj0hgc50q%40import.calendar.google.com/public/basic.ics",
-        calendar = Calendar.objects.get(id=1),
-        items = 0,
-        new_items = 0,
-        last_ran = datetime.now(),
-        )
-except Exception as e:
-    print(e)
-    scraper = Scraper.objects.get(name=venue.name)
-
-item_count_start = scraper.items
-
-event_type = "Mu"
-
-objIcalData = requests.get(scraper.website)
-gcal = iCalendar.from_ical(objIcalData.text)
-tz = pytz.timezone("US/Central")
-events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
-
-for event in events:
-    # ppr(event)
-    now_now = datetime.today().date()
-    e = {}
-    e['calendars'] = event['calendars']
-    try:
-        e['dateStamp'] = event['dateStart'][0]
-    except:
-        e['dateStamp'] = event['dateStart']
-    e['title'] = event['strSummary']
-    e['scraper'] = scraper 
-    e['link'] = venue.website
-    if e['dateStamp'] > now_now:
-        try:
-            digitools.createBasicEvent(e, 'Mu', venue)
-        except Exception as e:
-            print("Error: ", e)
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/zArchive/broken/palmers.mpls.py

@@ -1,77 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-
-from events.models import Event as DSEvent, Organization
-from digitools import getBrowser, createDashURL, createBasicEvent, getSource
-
-venue, created = Organization.objects.get_or_create(
-     name="Palmer's Bar",
-     city="Minneapolis",
-     website="https://palmers-bar.com",
-    )
-
-calendar_url = "https://palmers-bar.com"
-
-DATETIME_FORMAT = '%d %m %Y %I:%M%p'
-
-def get_events(ps, link):
-    contents = ps.xpath('.//*/td')
-    for c in contents:
-        try:
-            # day_num = c.xpath('.//*/div[@class="marker-daynum"]/text()')[0]
-            events = c.xpath('.//*/li')
-            # print(events)
-            for e in events:
-                event = {}
-                event_link = calendar_url + e.xpath('.//a/@href')[0]
-                ps = getSource(br, event_link)
-                new_event = ps.xpath('.//*/h1[@class="eventitem-column-meta"]')
-                event['title'] = new_event.xpath('.//*/h1[@class="event-title"]')
-                event['date'] = new_event.xpath('.//*/time[@class="event-date"]')
-                event['time'] = new_event.xpath('.//*/time[@class="event-time-12hr-start"]')
-                event['link'] = event_link
-
-                # event['month'] = link.split("month=")[1].split("-")[0]
-                # event['year'] = link.split("month=")[1].split("-")[1]
-                # event['title'] = e.xpath('.//h1/a[@class="flyoutitem-link"]/text()')
-                # event['time'] = e.xpath('.//div[@class="flyoutitem-datetime flyoutitem-datetime--12hr"]/text()')
-                # event['date'] = str(day_num) + ' ' + 'time'
-                # event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
-                ppr(event)
-                if len(event['title']):
-                    nevent = {}
-                    nevent['title'] = "".join(event['title']).strip()
-                    event['time'] = event['time'][0].strip().split(" –")[0]
-                    nevent['link'] = "https://palmers-bar.com" + e.xpath('.//a/@href')[0]
-                    event['dateStamp'] = str(day_num) + ' ' + event['month'] + ' ' + event['year'] + ' ' + event['time']
-                    nevent['dateStamp'] = datetime.strptime(event['dateStamp'], DATETIME_FORMAT)
-                    ppr(nevent)
-                    # createBasicEvent(nevent, 'Mu', venue)
-        except Exception as e:
-            continue
-
-if len(sys.argv) >= 2:
-    arg1 = sys.argv[1]
-    br = getBrowser(arg1)
-else:
-    print("No run_env")
-    quit()
-
-links = createDashURL("https://palmers-bar.com/?view=calendar&month=")
-
-for link in links:
-    ps = getSource(br, link)
-    get_events(ps, link)
-    sleep(5)
-
-# ppr(events)
-br.close()

events/scrapers/zArchive/digitools_old.py

@@ -1,132 +0,0 @@
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-from time import sleep
-import pytz
-from lxml import html
-
-import django
-sys.path.append('../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
-django.setup()
-
-from xvfbwrapper import Xvfb
-from selenium import webdriver as wd
-
-from events.models import Event as DSEvent, Organization
-
-tz = pytz.timezone("US/Central")
-td = relativedelta.relativedelta(months=1)
-odt = datetime.now() + td
-
-
-def getSource(browser, link):
-    browser.get(link)
-    sleep(3)
-    ps = html.fromstring(browser.page_source)
-    return ps
-
-def getBrowser(run_env):
-    if run_env == 'dev':
-        print("Chrome is a go!")
-        # chromeOptions = wd.ChromeOptions()
-        # chromeOptions.binary_location = "/Application/Google\ Chrome.app"
-        # chromeDriver = "/opt/homebrew/bin/chromedriver"
-        # br = wd.Chrome(chromeDriver, options=chromeOptions)
-        br = wd.Chrome()
-        return br
-    elif run_env == "def":
-        print("Firefox go vroom")
-        br = wd.Firefox()
-        return br
-    elif run_env == "prod":
-        start_cmd = "Xvfb :91 && export DISPLAY=:91 &"
-        xvfb = Xvfb()
-        os.system(start_cmd)
-        xvfb.start()
-        print("started Xvfb")
-        br = wd.Firefox()
-        return br
-    else:
-        print("Failed", sys.argv, arg1)
-        quit()
-
-def createBasicURL(site_url):
-    month = datetime.now().month
-    next_month = odt.month
-    year = datetime.now().year
-    print(month, next_month, year)
-    links = [
-        site_url + str(month) + "/" + str(year),
-        site_url + str(next_month) + "/" + str(year)
-    ]
-    print(links)
-    return links
-
-def createURLNoZero(site_url):
-    month = datetime.now().month
-    next_month = odt.month
-    year = datetime.now().year
-    links = [
-        site_url + str(year) + "/" + month,
-    ]
-    if next_month == "1":
-        links.append(site_url + str(int(year)+1) + "/" + next_month)
-    else:
-        links.append(site_url + str(year) + "/" + next_month)
-    print(links)
-    return links
-
-def createURL(site_url):
-    month = datetime.now().month
-    if month < 10:
-        month = "0" + str(month)
-    else:
-        month = str(month)
-    next_month = odt.month
-    if next_month < 10:
-        next_month = "0" + str(next_month)
-    else:
-        next_month = str(next_month)
-    year = datetime.now().year
-    links = [
-        site_url + str(year) + "/" + month,
-    ]
-    if next_month == "01":
-        links.append(site_url + str(int(year)+1) + "/" + next_month)
-    else:
-        links.append(site_url + str(year) + "/" + next_month)
-    print(links)
-    return links
-
-def createDashURL(site_url):
-    month = datetime.now().month
-    if month < 10:
-        month = "0" + str(month)
-    else:
-        month = str(month)
-    next_month = odt.month
-    if next_month < 10:
-        next_month = "0" + str(next_month)
-    else:
-        next_month = str(next_month)
-    year = datetime.now().year
-    print(month, next_month, year)
-    links = [
-        site_url + month + "-"  + str(year),
-        site_url + next_month + "-"  + str(year) 
-    ]
-    print(links)
-    return links
-
-def createBasicEvent(event, event_type, venue):
-    new_event, created = DSEvent.objects.update_or_create(
-        calendar = event['calendar'],
-        event_type = event_type,
-        show_title = event['title'],
-        show_link = event['link'],
-        show_date = event['dateStamp'],
-        show_day = event['dateStamp'],
-        venue = venue
-    )
-    print("New Event: ", new_event)

events/scrapers/zArchive/eventario.2.py

@@ -1,91 +0,0 @@
-import json
-
-import os, sys
-from datetime import datetime
-from dateutil import relativedelta
-
-import django
-sys.path.append('../../../../')
-os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
-django.setup()
-
-from time import sleep
-from pprint import pprint as ppr
-import pytz
-
-from events.models import Organization, Scraper, Calendar, Event
-import events.digitools as digitools
-
-
-venue, created = Organization.objects.get_or_create(
-     name="Eventario",
-     city="Medellin",
-     website="https://www.eventario.co",
-     is_venue=True
-    )
-
-scraper,item_count_start,virtcal = digitools.getScraper(venue, 'mde')
-
-DATETIME_FORMAT = '%-d/%-m/%y %I:%M %p'
-
-
-with open('data.json') as f:
-    data = json.load(f)
-
-paisa = []
-
-for d in data:
-    if len(d['dateTime']) != 0:
-        if 'Familia' in d['label']:
-            d['category'] = 'Ot'
-        elif 'Comedia' in d['label']:
-            d['category'] = 'Co'
-        elif ('Magic' in d['title']) or ('Juegos' in d['label']):
-            d['category'] = 'Ot'
-        elif ('Conferencias' in d['label']) or ('Intercambio' in d['label']):
-            d['category'] = 'Ed'
-        else:
-            d['category'] = 'Mu'
-        if "Antioquia" in d['location']:
-            try:
-                d['city'] = d['location'].split(',')[0]
-                paisa.append(d)
-            except:
-                continue
-
-# ppr(paisa)
-with open('data.updated.json', 'w', encoding='utf-8') as f:
-    json.dump(data, f, ensure_ascii=False, indent=4)
-
-
-cal = Calendar.objects.get(shortcode='mde')
-
-
-for d in paisa:
-    d['dateStamp'] =datetime.strptime(d['dateTime'][0], DATETIME_FORMAT)
-    try:
-        nvenue, created = Organization.objects.get_or_create(
-            name=d['venue'],
-            city=d['city'],
-            website=d['venueLink'],
-            address_complete = d['address'], 
-            is_venue=True
-            )
-    except:
-        nvenue = Organization.objects.get(name=d['venue'])
-        nvenue.address_complete = d['address']
-        nvenue.save()
-    new_event, created = Event.objects.update_or_create(
-        event_type = d['category'],
-        show_title = d['title'],
-        show_link = d['link'],
-        show_date = d['dateStamp'],
-        show_day = d['dateStamp'],
-        scraper = scraper,
-        venue = nvenue
-    )
-    new_event.calendar.add(cal)
-    new_event.save()
-    print(new_event)
-
-digitools.updateScraper(scraper, item_count_start)

events/scrapers/zArchive/run_scrapers.sh

@@ -1,50 +0,0 @@
-#!/bin/bash
-
-# BASEDIR=/var/www/digisnaxx.com/
-# DJANGODIR=/var/www/digisnaxx.com/ds_events
-# EVENTDIR=/var/www/digisnaxx.com/ds_events/event_scrapers
-
-BASEDIR=/home/canin/Downloads/DigiSnaxxEvents
-DJANGODIR=/home/canin/Downloads/DigiSnaxxEvents/ds_events
-EVENTDIR=/home/canin/Downloads/DigiSnaxxEvents/ds_events/event_scrapers
-VENUESDIR=/home/canin/Downloads/DigiSnaxxEvents/ds_events/event_scrapers/Working/venues
-ICALDIR=/home/canin/Downloads/DigiSnaxxEvents/ds_events/event_scrapers/Working/iCal
-GOVTDIR=/home/canin/Downloads/DigiSnaxxEvents/ds_events/event_scrapers/Working/govt
-
-export DJANGO_SUPERUSER_EMAIL=canin@dreamfreely.org
-export DJANGO_SUPERUSER_USERNAME=canin
-export DJANGO_SUPERUSER_PASSWORD='hello123'
-
-cd $BASEDIR
-pwd
-source venv/bin/activate
-
-cd $DJANGODIR
-pwd
-mv db.sqlite3 db.sqlite3.bak
-# rm ../db.sqlite3
-# touch db.sqlite3
-python manage.py migrate
-# python manage.py createsuperuser --noinput
-# python manage.py loaddata events/fixtures/organizations.json
-# python manage.py loaddata events/fixtures/promo.json
-
-# cd $EVENTDIR
-# python start_up.py
-
-bash run_venues.sh $1
-bash run_ical.sh 
-bash run_govt.sh $1
-bash run_news.sh $1
-
-python Working/bluesky.py
-python Working/redsky.py
-
-cd $EVENTDIR
-python clean_up.py
-
-deactivate
-bash run_media_update.sh
-
-rm -rf ../*/__pycache__
-rm -rf ../*/*/__pycache__