first commit

event_scrapers/Working/cals/MplStpMag.mn.py

@@ -0,0 +1,102 @@
+import os, sys
+from datetime import datetime
+from dateutil import relativedelta
+
+import django
+sys.path.append('../../../')
+os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
+django.setup()
+
+from time import sleep
+from pprint import pprint as ppr
+import pytz
+
+from events.models import Organization
+from events.digitools import getBrowser, createURL, createBasicEvent, getSource
+
+venue, created = Organization.objects.get_or_create(
+     name="Mpls Stp Mag",
+     city="Minneapolis",
+     website="https://calendar.mspmag.com/calendars/all-events/",
+    )
+
+event_type = ""
+
+# Time Signatures
+tz = pytz.timezone("US/Central")
+td = relativedelta.relativedelta(days=1)
+fortnight = relativedelta.relativedelta(days=14)
+odt = datetime.now() + fortnight
+
+# DATETIME_FORMAT = '%b %d %Y %I:%M %p'
+DATETIME_FORMAT = '%A, %B %d %Y %I:%M%p'
+DATETIME_FORMAT_ALT = '%A, %B %d %Y'
+
+def get_events(ps, event_type):
+    contents = ps.xpath('.//*/div[@class="css-card js-card day-card type-smad expandable"]')
+    for c in contents:
+        try:
+            event = {}
+            event['calendar'] = venue.calendar
+            event_block = c.xpath('.//*/li[@class="card-listings-item event-element"]')
+            date = c.xpath('.//div[@class="day-card__header day-card__header--daily"]/text()')[0].replace("\n", "").strip()
+            if date == "Today":
+                date = datetime.today()
+            elif date == "Tomorrow":
+                date = datetime.today() + td
+            # month = c.xpath('.//*/span[@class="mth"]/text()')[0]
+            year = datetime.now().year
+            # if month == "Jan":
+            #     year = int(year) + 1
+            dateTime = datetime.strptime(date + " " + str(year), DATETIME_FORMAT_ALT)
+            if dateTime > odt:
+                print("DATE TIME ", dateTime)
+                break
+            for ev in event_block:
+                time = ev.xpath('.//*/span[@class="card-listing-item-time"]/text()')[0].replace("@", "").strip()
+                if time == "All day":
+                    time = "12:00pm"
+                    event['title'] = ev.xpath('.//*/div[@class="card-listing-item-title"]/text()')[0] + " (Check link for times.)"
+                elif "-" in time:
+                    time = time.split("-")[0]
+                    event['title'] = ev.xpath('.//*/div[@class="card-listing-item-title"]/text()')[0]
+                else:
+                    event['title'] = ev.xpath('.//*/div[@class="card-listing-item-title"]/text()')[0]
+                
+                event['location'] = ev.xpath('.//*/span[@class="card-listing-item-location"]/text()')[0]
+                if event['location'] == '7th St. Entry':
+                    event['location'] = '7th St Entry'
+                elif event['location'] == '7th Street Entry':
+                    event['location'] = '7th St Entry'
+                elif event['location'] == 'Amsterdam Bar and Hall':
+                    event['location'] = 'Amsterdam Bar & Hall'
+                new_venue, created = Organization.objects.get_or_create(name=event['location'])
+                print("V: ", new_venue, created)
+
+                event['dateTime'] = date + " " + str(year) + " " + time
+                event['link'] = venue.website + c.xpath('.//@data-event')[0]
+                event['dateStamp'] = datetime.strptime(event['dateTime'], DATETIME_FORMAT)
+
+                
+
+                createBasicEvent(event, event_type, new_venue)
+        except Exception as e:
+            print(e)
+            ppr(event)
+            print("\n\n+++\n\n")
+
+if len(sys.argv) >= 2:
+    arg1 = sys.argv[1]
+    br = getBrowser(arg1)
+else:
+    print("No run_env")
+    quit()
+
+# Get Event Page Link(s)
+# links = createURL("https://acmecomedycompany.com/the-club/calendar/")  
+
+ps = getSource(br, venue.website)
+get_events(ps, "Ed")
+sleep(3)
+
+br.close()

event_scrapers/Working/cals/minnestar.py

@@ -0,0 +1,105 @@
+import os, sys
+from datetime import datetime
+from dateutil import relativedelta
+
+import django
+sys.path.append('../../../')
+os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
+django.setup()
+
+from time import sleep
+from pprint import pprint as ppr
+import pytz
+
+from events.models import Organization, Scraper
+from events.digitools import getBrowser, createURL, createBasicEvent, getSource
+
+count = 0
+
+venue, created = Organization.objects.get_or_create(
+     name="Minnestar",
+     city="Minneapolis",
+     website="https://minnestar.org/community/calendar",
+    )
+
+try:
+    scraper, created = Scraper.objects.get_or_create(
+        name=venue.name,
+        website=venue.website,
+        items = 0,
+        last_ran = datetime.now(),
+    )
+except Exception as e:
+    print(e)
+    scraper = Scraper.objects.get(name=venue.name)
+    print("Scraper: ", scraper)
+
+event_type = ""
+
+# Time Signatures
+tz = pytz.timezone("US/Central")
+DATETIME_FORMAT = '%B %d, %Y %I:%M %p'
+DATETIME_FORMAT_2 = '%B %d %Y'
+
+def get_events(ps, event_type):
+    links = ps.xpath('.//*/div[@id="community-calendar-list-view-container"]/a/@href')
+    ppr(links)
+    for l in links:
+        pse = getSource(br, l)
+        sleep(1)
+        event = {}
+        event['calendar'] = venue.calendar
+        event['link'] = l
+        try:
+            starttime = pse.xpath('.//*/time/text()')[0]
+            endtime = pse.xpath('.//*/time/@datetime')[1]
+            event['dateStamp'] = datetime.strptime(starttime, DATETIME_FORMAT) 
+            event['title'] = pse.xpath('.//*/h1[@class="heading-2"]/text()')[0]
+            # event['detail-headers'] = pse.xpath('.//*/ul[@class="eo-event-meta"]/li/strong/text()')
+            # event['details'] = pse.xpath('.//*/ul[@class="eo-event-meta"]/li/text()')
+
+        except:
+            try:
+                event['title'] = pse.xpath('.//*/h1[@class="heading-2"]/text()')[0]
+                starttime = pse.xpath('.//*/time/text()')[0]
+                event['dateStamp'] = datetime.strptime(starttime, DATETIME_FORMAT)
+            except Exception as e:
+                try:
+                    print(e)
+                    print('failed event: ', event)
+                    starttime = pse.xpath('.//*/time/text()')[0]
+                    event['dateStamp'] = datetime.strptime(starttime + ' 2025', DATETIME_FORMAT_2)
+                except Exception as e:
+                    print(e)
+                    print("failed event: ", event)
+        ppr(event)
+        try:
+            createBasicEvent(event, "Ot", venue)
+            scraper.items+=1
+        except Exception as e:
+            print(e)
+            print('failed to create: ', event)
+        
+
+if len(sys.argv) >= 2:
+    arg1 = sys.argv[1]
+    br = getBrowser(arg1)
+else:
+    print("No run_env")
+    quit()
+
+ps = getSource(br, venue.website)
+get_events(ps, "Ot")
+sleep(3)
+
+scraper.save()
+
+# Get Event Page Link(s)
+# links = createURL("https://acmecomedycompany.com/the-club/calendar/")  
+
+# for link in links:
+#     ps = getSource(br, link)
+#     get_events(ps, "Ed")
+#     sleep(3)
+
+br.close()