DigiSnaxx/scrapers
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first push

Browse Source
This commit is contained in:
canin carlos
2025-11-30 16:29:50 -05:00
commit d5055374b4
77 changed files with 9129 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored Normal file
View File

@@ -0,0 +1,6 @@
*/__pycache__
*/*/__pycache__
.env
db.sqlite3
baks
*.bak

90
Templates/TemplateScraper.py Normal file
View File

@@ -0,0 +1,90 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
from events.digitools import getBrowser, createURL, createBasicEvent, getSource
count = 0
venue, created = Organization.objects.get_or_create(
name="Venue Name",
city="Minneapolis",
website="Event Website",
)
try:
scraper, created = Scraper.objects.get_or_create(
name=venue.name,
website=venue.website,
items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
print("Scraper: ", scraper)
event_type = ""
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b %d %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
def get_events(ps, event_type):
contents = ps.xpath('.//*/li[@class="event"]')
for c in contents:
try:
event = {}
day = c.xpath('.//*/span[@class="day"]/text()')[0]
month = c.xpath('.//*/span[@class="mth"]/text()')[0]
year = datetime.now().year
if month == "Jan":
year = int(year) + 1
event['calendar'] = venue.calendar
event['title'] = c.xpath('.//*/span[@class="event_title"]/a/text()')[0]
event['date'] = [month, day, str(year), c.xpath('.//*/span[@class="event_time"]/text()')[0].strip()]
event['date'] = " ".join(event['date'])
event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
event['link'] = c.xpath('.//*/span[@class="event_title"]/a/@href')[0]
print("Event Dict Created")
createBasicEvent(event, event_type, venue)
scraper.items+=1
except Exception as e:
print(e)
ppr(event)
print("\n\n+++\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
ps = getSource(br, venue.website)
get_events(ps, "Ed")
sleep(3)
scraper.save()
# Get Event Page Link(s)
# links = createURL("https://acmecomedycompany.com/the-club/calendar/")
# for link in links:
# ps = getSource(br, link)
# get_events(ps, "Ed")
# sleep(3)
br.close()

84
Templates/ical_run.py.template Normal file
View File

@@ -0,0 +1,84 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from events.models import Event as DSEvent, Organization
venue, created = Organization.objects.get_or_create(
name="location",
city="Minneapolis",
website="",
)
event_type = ""
calendar_url = [
'https://calendar.google.com/calendar/ical/js94epu90r2et31aopons1ifm8%40group.calendar.google.com/public/basic.ics',
'https://calendar.google.com/calendar/ical/6rpooudjg01vc8bjek1snu2ro0%40group.calendar.google.com/public/basic.ics',
'https://calendar.google.com/calendar/ical/teflgutelllvla7r6vfcmjdjjo%40group.calendar.google.com/public/basic.ics'
]
objIcalData = requests.get(calendar_url[1])
gcal = iCalendar.from_ical(objIcalData.text)
cfpa_events = []
tz = pytz.timezone("US/Central")
for component in gcal.walk():
event = {}
event['strSummary'] = f"{(component.get('SUMMARY'))}"
event['strDesc'] = component.get('DESCRIPTION')
event['strLocation'] = component.get('LOCATION')
event['dateStart'] = component.get('DTSTART')
event['dateStamp'] = component.get('DTSTAMP')
if event['dateStamp'] is not None:
event['dateStamp'] = event['dateStamp'].dt
if event['dateStart'] is not None:
try:
event['dateStart'] = event['dateStart'].dt.astimezone(pytz.utc)
except Exception as e:
event['dateStart'] = event['dateStart'].dt
event['dateEnd'] = (component.get('DTEND'))
if event['dateEnd'] is not None:
event['dateEnd'] = event['dateEnd'].dt
else:
event['dateEnd'] = event['dateStart']
if event['strSummary'] != 'None':
event['details'] = {
"description" : event['strDesc'],
"DateTime" : event['dateStart'],
"Location" : event['strLocation'],
}
cfpa_events.append(event)
new_event = DSEvent.objects.update_or_create(
event_type = event_type,
show_title = event['strSummary'],
show_link = event['link'],
show_date = event['dateStart'],
show_day = event['dateStart'].date(),
more_details = event["details"],
venue = venue
)
# {'dateEnd': datetime.datetime(2022, 10, 22, 18, 30, tzinfo=<UTC>),
# 'dateStamp': datetime.datetime(2023, 3, 23, 1, 57, 45, tzinfo=<UTC>),
# 'dateStart': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
# 'details': {'DateTime': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
# 'Location': vText('b'''),
# 'description': None},
# 'strDesc': None,
# 'strLocation': vText('b'''),
# 'strSummary': 'Nia Class with Beth Giles'}

84
Templates/ical_template.py Normal file
View File

@@ -0,0 +1,84 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from events.models import Event as DSEvent, Organization
venue, created = Organization.objects.get_or_create(
name="location",
city="Minneapolis",
website="",
)
event_type = ""
calendar_url = [
'https://calendar.google.com/calendar/ical/js94epu90r2et31aopons1ifm8%40group.calendar.google.com/public/basic.ics',
'https://calendar.google.com/calendar/ical/6rpooudjg01vc8bjek1snu2ro0%40group.calendar.google.com/public/basic.ics',
'https://calendar.google.com/calendar/ical/teflgutelllvla7r6vfcmjdjjo%40group.calendar.google.com/public/basic.ics'
]
objIcalData = requests.get(calendar_url[1])
gcal = iCalendar.from_ical(objIcalData.text)
cfpa_events = []
tz = pytz.timezone("US/Central")
for component in gcal.walk():
event = {}
event['strSummary'] = f"{(component.get('SUMMARY'))}"
event['strDesc'] = component.get('DESCRIPTION')
event['strLocation'] = component.get('LOCATION')
event['dateStart'] = component.get('DTSTART')
event['dateStamp'] = component.get('DTSTAMP')
if event['dateStamp'] is not None:
event['dateStamp'] = event['dateStamp'].dt
if event['dateStart'] is not None:
try:
event['dateStart'] = event['dateStart'].dt.astimezone(pytz.utc)
except Exception as e:
event['dateStart'] = event['dateStart'].dt
event['dateEnd'] = (component.get('DTEND'))
if event['dateEnd'] is not None:
event['dateEnd'] = event['dateEnd'].dt
else:
event['dateEnd'] = event['dateStart']
if event['strSummary'] != 'None':
event['details'] = {
"description" : event['strDesc'],
"DateTime" : event['dateStart'],
"Location" : event['strLocation'],
}
cfpa_events.append(event)
new_event = DSEvent.objects.update_or_create(
event_type = event_type,
show_title = event['strSummary'],
show_link = event['link'],
show_date = event['dateStart'],
show_day = event['dateStart'].date(),
more_details = event["details"],
venue = venue
)
# {'dateEnd': datetime.datetime(2022, 10, 22, 18, 30, tzinfo=<UTC>),
# 'dateStamp': datetime.datetime(2023, 3, 23, 1, 57, 45, tzinfo=<UTC>),
# 'dateStart': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
# 'details': {'DateTime': datetime.datetime(2022, 10, 22, 17, 30, tzinfo=<UTC>),
# 'Location': vText('b'''),
# 'description': None},
# 'strDesc': None,
# 'strLocation': vText('b'''),
# 'strSummary': 'Nia Class with Beth Giles'}

102
Working/cals/MplStpMag.mn.py Normal file
View File

@@ -0,0 +1,102 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization
from events.digitools import getBrowser, createURL, createBasicEvent, getSource
venue, created = Organization.objects.get_or_create(
name="Mpls Stp Mag",
city="Minneapolis",
website="https://calendar.mspmag.com/calendars/all-events/",
)
event_type = ""
# Time Signatures
tz = pytz.timezone("US/Central")
td = relativedelta.relativedelta(days=1)
fortnight = relativedelta.relativedelta(days=14)
odt = datetime.now() + fortnight
# DATETIME_FORMAT = '%b %d %Y %I:%M %p'
DATETIME_FORMAT = '%A, %B %d %Y %I:%M%p'
DATETIME_FORMAT_ALT = '%A, %B %d %Y'
def get_events(ps, event_type):
contents = ps.xpath('.//*/div[@class="css-card js-card day-card type-smad expandable"]')
for c in contents:
try:
event = {}
event['calendar'] = venue.calendar
event_block = c.xpath('.//*/li[@class="card-listings-item event-element"]')
date = c.xpath('.//div[@class="day-card__header day-card__header--daily"]/text()')[0].replace("\n", "").strip()
if date == "Today":
date = datetime.today()
elif date == "Tomorrow":
date = datetime.today() + td
# month = c.xpath('.//*/span[@class="mth"]/text()')[0]
year = datetime.now().year
# if month == "Jan":
# year = int(year) + 1
dateTime = datetime.strptime(date + " " + str(year), DATETIME_FORMAT_ALT)
if dateTime > odt:
print("DATE TIME ", dateTime)
break
for ev in event_block:
time = ev.xpath('.//*/span[@class="card-listing-item-time"]/text()')[0].replace("@", "").strip()
if time == "All day":
time = "12:00pm"
event['title'] = ev.xpath('.//*/div[@class="card-listing-item-title"]/text()')[0] + " (Check link for times.)"
elif "-" in time:
time = time.split("-")[0]
event['title'] = ev.xpath('.//*/div[@class="card-listing-item-title"]/text()')[0]
else:
event['title'] = ev.xpath('.//*/div[@class="card-listing-item-title"]/text()')[0]
event['location'] = ev.xpath('.//*/span[@class="card-listing-item-location"]/text()')[0]
if event['location'] == '7th St. Entry':
event['location'] = '7th St Entry'
elif event['location'] == '7th Street Entry':
event['location'] = '7th St Entry'
elif event['location'] == 'Amsterdam Bar and Hall':
event['location'] = 'Amsterdam Bar & Hall'
new_venue, created = Organization.objects.get_or_create(name=event['location'])
print("V: ", new_venue, created)
event['dateTime'] = date + " " + str(year) + " " + time
event['link'] = venue.website + c.xpath('.//@data-event')[0]
event['dateStamp'] = datetime.strptime(event['dateTime'], DATETIME_FORMAT)
createBasicEvent(event, event_type, new_venue)
except Exception as e:
print(e)
ppr(event)
print("\n\n+++\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
# Get Event Page Link(s)
# links = createURL("https://acmecomedycompany.com/the-club/calendar/")
ps = getSource(br, venue.website)
get_events(ps, "Ed")
sleep(3)
br.close()

105
Working/cals/minnestar.py Normal file
View File

@@ -0,0 +1,105 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
from events.digitools import getBrowser, createURL, createBasicEvent, getSource
count = 0
venue, created = Organization.objects.get_or_create(
name="Minnestar",
city="Minneapolis",
website="https://minnestar.org/community/calendar",
)
try:
scraper, created = Scraper.objects.get_or_create(
name=venue.name,
website=venue.website,
items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
print("Scraper: ", scraper)
event_type = ""
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%B %d, %Y %I:%M %p'
DATETIME_FORMAT_2 = '%B %d %Y'
def get_events(ps, event_type):
links = ps.xpath('.//*/div[@id="community-calendar-list-view-container"]/a/@href')
ppr(links)
for l in links:
pse = getSource(br, l)
sleep(1)
event = {}
event['calendar'] = venue.calendar
event['link'] = l
try:
starttime = pse.xpath('.//*/time/text()')[0]
endtime = pse.xpath('.//*/time/@datetime')[1]
event['dateStamp'] = datetime.strptime(starttime, DATETIME_FORMAT)
event['title'] = pse.xpath('.//*/h1[@class="heading-2"]/text()')[0]
# event['detail-headers'] = pse.xpath('.//*/ul[@class="eo-event-meta"]/li/strong/text()')
# event['details'] = pse.xpath('.//*/ul[@class="eo-event-meta"]/li/text()')
except:
try:
event['title'] = pse.xpath('.//*/h1[@class="heading-2"]/text()')[0]
starttime = pse.xpath('.//*/time/text()')[0]
event['dateStamp'] = datetime.strptime(starttime, DATETIME_FORMAT)
except Exception as e:
try:
print(e)
print('failed event: ', event)
starttime = pse.xpath('.//*/time/text()')[0]
event['dateStamp'] = datetime.strptime(starttime + ' 2025', DATETIME_FORMAT_2)
except Exception as e:
print(e)
print("failed event: ", event)
ppr(event)
try:
createBasicEvent(event, "Ot", venue)
scraper.items+=1
except Exception as e:
print(e)
print('failed to create: ', event)
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
ps = getSource(br, venue.website)
get_events(ps, "Ot")
sleep(3)
scraper.save()
# Get Event Page Link(s)
# links = createURL("https://acmecomedycompany.com/the-club/calendar/")
# for link in links:
# ps = getSource(br, link)
# get_events(ps, "Ed")
# sleep(3)
br.close()

167
Working/govt/MNLeg.py Normal file
View File

@@ -0,0 +1,167 @@
# Install Chromedriver and Quarantine
# xattr -d com.apple.quarantine <name-of-executable>
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from xvfbwrapper import Xvfb
from lxml import html
import pytz
from events.models import Event, Organization, Scraper, Calendar
from events.digitools import getBrowser, createURL, createBasicEvent, getSource, add_calendar
try:
scraper, created = Scraper.objects.get_or_create(
name="MN Legislature",
website="https://www.leg.mn.gov/cal?type=all",
calendar = Calendar.objects.get(shortcode='msp'),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name="MN Legislature")
print("Scraper: ", scraper)
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p'
# Set initial variables for City, etc
calendar_url = 'https://www.leg.mn.gov/cal?type=all'
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
br.get(calendar_url)
sleep(10)
ps = html.fromstring(br.page_source)
commEvents = ps.xpath('.//*/div[@class="card border-dark comm_item cal_item ml-lg-3"]')
senateEvents = ps.xpath('.//*/div[@class="card border-dark senate_item cal_item ml-lg-3"]')
houseEvents = ps.xpath('.//*/div[@class="card border-dark house_item cal_item ml-lg-3"]')
meetings = []
for hE in houseEvents:
details = {}
dateTime = hE.xpath('.//*/b/text()')[0]
try:
title = hE.xpath('.//*/h3/a/text()')[0]
except:
title = hE.xpath('.//*/h3/text()')[0]
try:
link = "https://www.leg.mn.gov/" + hE.xpath('.//*/div[@class="float-right text-center mr-2 d-print-none"]/a/@href')[0]
except:
link = hE.xpath('.//*/h3/a/@href')[0]
details['location'] = hE.xpath('.//*/div[@class=""]/text()')[0]
# print(dateTime, title, link, details['location'])
venue, created = Organization.objects.get_or_create(name="MN House", city="St. Paul")
new_event, created = Event.objects.update_or_create(
event_type = 'Gv',
show_title = title,
show_link = link,
show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
show_day = datetime.strptime(dateTime, DATETIME_FORMAT).date(),
more_details = details['location'],
venue = venue,
scraper = scraper
)
if type(new_event) is tuple:
print("STS: ", new_event)
add_calendar(new_event[0], 'msp')
else:
add_calendar(new_event, 'msp')
scraper.items+=1
for sE in senateEvents:
details = {}
dateTime = sE.xpath('.//*/b/text()')[0]
try:
title = sE.xpath('.//*/h3/a/text()')[0]
except:
title = sE.xpath('.//*/h3/text()')[0]
try:
link = "https://www.leg.mn.gov/" + sE.xpath('.//*/div[@class="float-right text-center mr-2"]/a/@href')[0]
except:
link = sE.xpath('.//*/h3/a/@href')[0]
location_list = sE.xpath('.//*/text()')
if 'Location: ' in location_list:
iN = location_list.index("Location: ")
details['location'] = location_list[iN + 1]
elif 'Senate Floor Session' in location_list:
details['location'] = 'Senate Floor Session'
venue, created = Organization.objects.get_or_create(name="MN Senate", city="St. Paul")
new_event = Event.objects.update_or_create(
event_type = 'Gv',
show_title = title,
show_link = link,
show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
show_day = datetime.strptime(dateTime, DATETIME_FORMAT).date(),
more_details = details['location'],
venue = venue,
scraper = scraper
)
if type(new_event) is tuple:
print("STS: ", new_event)
add_calendar(new_event[0], 'msp')
else:
add_calendar(new_event, 'msp')
scraper.items+=1
for cE in commEvents:
details = {}
dateTime = cE.xpath('.//*/b/text()')[0]
try:
title = cE.xpath('.//*/h3/a/text()')[0]
except:
title = cE.xpath('.//*/h3/text()')[0]
try:
link = "https://www.leg.mn.gov/" + cE.xpath('.//*/div[@class="float-right text-center mr-2"]/a/@href')[0]
except:
link = cE.xpath('.//*/h3/a/@href')[0]
location_list = cE.xpath('.//*/text()')
if 'Room: ' in location_list:
iN = location_list.index("Room: ")
details['location'] = location_list[iN + 1]
# print(dateTime, title, link, details['location'])
venue, created = Organization.objects.get_or_create(name="MN Legislature", city="St. Paul")
new_event = Event.objects.update_or_create(
event_type = 'Gv',
show_title = title,
show_link = link,
show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
show_day = datetime.strptime(dateTime, DATETIME_FORMAT).date(),
more_details = details['location'],
venue = venue,
scraper = scraper
)
if type(new_event) is tuple:
print("STS: ", new_event)
add_calendar(new_event[0], 'msp')
else:
add_calendar(new_event, 'msp')
scraper.items+=1
br.close()
scraper.save()
# br.find_element_by_class_name('fc-btn_allCalendars-button').click()

122
Working/govt/MplsCityCouncil.py Normal file
View File

@@ -0,0 +1,122 @@
import re, os, sys
from datetime import datetime, timedelta
import requests
import json
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event, Organization, Scraper, Calendar
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from xvfbwrapper import Xvfb
from lxml import html
import pytz
from events.digitools import getBrowser, createURL, createBasicEvent, getSource, add_calendar
import events.digitools as digitools
try:
scraper, created = Scraper.objects.get_or_create(
name="Mpls City Council",
website="https://lims.minneapolismn.gov/Calendar/citycouncil/upcoming",
calendar = Calendar.objects.get(shortcode='msp'),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name="Mpls City Council")
print("Scraper: ", scraper)
DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
tz = pytz.timezone("US/Central")
td = timedelta(days=2)
odt = datetime.now() - td
month = odt.strftime("%b")
day = int(datetime.now().day)
if day == 1:
day = 30
year = int(datetime.now().year)
cal_url = 'https://lims.minneapolismn.gov/Calendar/GetCalenderList?fromDate={}%20{},%20{}&toDate=null&meetingType=0&committeeId=null&pageCount=100&offsetStart=0&abbreviation=undefined&keywords=&sortOrder=1'.format(month, day, year)
# print("URL: ", cal_url)
# cal_url = 'https://lims.minneapolismn.gov/Calendar/GetCalenderList?fromDate=Nov%2015,%202025&toDate=null&meetingType=0&committeeId=null&pageCount=50&offsetStart=0&abbreviation=undefined&keywords=&sortOrder=1'
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
br.get(cal_url)
sleep(2)
ps = html.fromstring(br.page_source)
events = ps.xpath('.//body/pre/text()')[0]
new_events = json.loads(events)
# ppr(dict(new_events[0]))
event_list = []
for event in new_events:
e = {}
e['title'] = event['CommitteeName']
e['link'] = scraper.website
e['dateStamp'] = datetime.strptime(event['MeetingTime'], DATETIME_FORMAT)
e['agendaStatus'] = event['AgendaStatus']
e['address'] = event['Address']
e['description'] = event['Description']
e['scraper'] = scraper
e['calendars'] = [scraper.calendar]
try:
# print(event['Location'].split(",")[1])
loc = event['Location'].split(",")[1]
if "City Hall" in loc:
e['location'] = "Mpls City Hall"
venue = Organization.objects.get(
name="Mpls City Hall",
)
digitools.createBasicEvent(e, 'Gv', venue)
elif "Public Service Building" in loc:
e['location'] = "Mpls Public Service Building"
venue = Organization.objects.get(
name="Mpls Public Service Building",
)
digitools.createBasicEvent(e, 'Gv', venue)
except:
# print(event['Location'].split("-")[0])
e['location'] = event['Location'].split("-")[0].strip()
try:
venue, created = Organization.objects.get_or_create(
name=e['location'] ,
city="Minneapolis",
website=scraper.website,
is_venue=False,
address_complete=e['address']
)
except:
venue = Organization.objects.get(
name=e['location'] ,
city="Minneapolis",
)
digitools.createBasicEvent(e, 'Gv', venue)
event_list.append(e)
br.close()
scraper.save()
# br.find_element_by_class_name('fc-btn_allCalendars-button').click()

82
Working/govt/StPaulCityCouncil.py Normal file
View File

@@ -0,0 +1,82 @@
import re, os, sys
from datetime import datetime
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event, Organization, Scraper, Calendar
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from selenium.webdriver.common.by import By
from xvfbwrapper import Xvfb
from lxml import html
import pytz
from events.digitools import getBrowser, createURL, createBasicEvent, getSource, add_calendar
try:
scraper, created = Scraper.objects.get_or_create(
name="St Paul City Council",
website="https://www.stpaul.gov/calendar",
calendar = Calendar.objects.get(shortcode='msp'),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name="St Paul City Council")
print("Scraper: ", scraper)
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%B %d, %Y at %I:%M %p'
calendar_url = 'https://www.stpaul.gov/calendar'
city_site = "https://www.stpaul.gov"
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
br.get(calendar_url)
sleep(3)
def getEvents(br):
ps = html.fromstring(br.page_source)
eventBlocks = ps.xpath('.//*/div[@class="calendar__item views-row"]')
for eB in eventBlocks:
title = eB.xpath('.//div/h3[@class="field-content calendar__title"]/text()')[0]
link = city_site + eB.xpath('.//div/span[@class="field-content calendar__link"]/a/@href')[0]
dateTime = eB.xpath('.//div[@class="views-field views-field-field-calendar-date-value"]/span/text()')[0]
print(dateTime, title, link)
print('\n\n++++\n\n')
venue, created = Organization.objects.get_or_create(name="Somewhere in St Paul", city="St. Paul")
new_event = Event.objects.update_or_create(
event_type = 'Gv',
show_title = title,
show_link = link,
show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
show_day = datetime.strptime(dateTime, DATETIME_FORMAT),
venue = venue,
scraper = scraper
)
add_calendar(new_event, 'msp')
getEvents(br)
sleep(5)
br.get("https://www.stpaul.gov/calendar?page=1")
getEvents(br)
br.close()
# br.find_element_by_class_name('fc-btn_allCalendars-button').click()

120
Working/govt/mngov.py Normal file
View File

@@ -0,0 +1,120 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
td = relativedelta.relativedelta(hours=5)
odt = datetime.now() + td
venue, created = Organization.objects.get_or_create(
name="MN Launch",
city="Minneapolis",
website="https://mn.gov/launchmn/calendar",
)
try:
scraper, created = Scraper.objects.get_or_create(
name=venue.name,
website=venue.website,
calendar = Calendar.objects.get(shortcode='msp'),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
print("Scraper: ", scraper)
event_type = "Ed"
cal_url = "https://timelyapp.time.ly/api/calendars/54705514/export?format=ics&target=copy&start_date=2024-12-13"
calendar_url = 'https://calendar.google.com/calendar/ical/uvkshlggh1h4ck08emab22btkum9hl94%40import.calendar.google.com/public/basic.ics'
objIcalData = requests.get(cal_url)
gcal = iCalendar.from_ical(objIcalData.text)
cfpa_events = []
tz = pytz.timezone("US/Central")
for component in gcal.walk():
event = {}
event['strSummary'] = f"{(component.get('SUMMARY'))}"
event['strDesc'] = component.get('DESCRIPTION')
event['strLocation'] = component.get('LOCATION')
event['dateStart'] = component.get('DTSTART')
event['dateStamp'] = component.get('DTSTAMP')
if event['dateStamp'] is not None:
event['dateStamp'] = event['dateStart'].dt
if event['dateStart'] is not None:
try:
event['dateStart'] = event['dateStart'].dt
except Exception as e:
event['dateStart'] = event['dateStart'].dt
event['dateEnd'] = (component.get('DTEND'))
if event['dateEnd'] is not None:
event['dateEnd'] = event['dateEnd'].dt
else:
event['dateEnd'] = event['dateStart']
if event['strSummary'] != 'None':
event['details'] = {
"description" : event['strDesc'],
"Location" : event['strLocation'],
}
cfpa_events.append(event)
now_now = datetime.now().astimezone(tz)
try:
if event['dateStart'] > now_now:
print(event['strSummary'])
new_event, created = DSEvent.objects.update_or_create(
event_type = event_type,
show_title = event['strSummary'],
show_link = venue.website,
show_date = event['dateStart']-td,
show_day = event['dateStart']-td,
more_details = event["details"],
venue = venue
)
digitools.add_calendar(new_event, 'msp')
scraper.items+=1
if event['strLocation'] != None and event['strLocation'] != 'MN' and event['strLocation'] != 'Online':
loc = event['strLocation'].split('@')
new_venue_name = loc[0]
if len(loc) > 1:
address = loc[1].split(",")
city = address[1].strip()
new_venue, created = Organization.objects.get_or_create(
name=new_venue_name,
city=city,
website="https://mn.gov/launchmn/calendar",
)
new_event.venue = new_venue
new_event.save()
else:
new_event.venue = venue
new_event.save()
except Exception as e:
print(e)
print("Event: ", event['dateStart'], event['strSummary'])
print("Clock: ", now_now)
else:
print("Failed: ", component.get('DESCRIPTION'))
scraper.save()

68
Working/iCal/ical.CAFAC.mpls.py Normal file
View File

@@ -0,0 +1,68 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
venue, created = Organization.objects.get_or_create(
name="Chicago Ave Fire Arts Center",
city="Minneapolis",
website="https://www.cafac.org/classes",
)
event_type = "Ed"
try:
scraper, created = Scraper.objects.get_or_create(
name="Chicago Ave Fire Arts Center",
website="https://calendar.google.com/calendar/ical/9qj2426rukra3jv933nslsf3r8%40group.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(id=1),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
event_type = "Ed"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
for event in events:
# ppr(event)
e = {}
e['calendars'] = event['calendars']
try:
e['dateStamp'] = event['dateStart'][0]
except:
e['dateStamp'] = event['dateStart']
e['title'] = event['strSummary']
e['scraper'] = scraper
e['link'] = venue.website
try:
digitools.createBasicEvent(e, 'Ed', venue)
except Exception as e:
print("Error: ", e)
digitools.updateScraper(scraper, item_count_start)

146
Working/iCal/ical_ingles.py Normal file
View File

@@ -0,0 +1,146 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime, timedelta
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from config.env import env
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
td = relativedelta.relativedelta(hours=5)
venue, created = Organization.objects.get_or_create(
name="idioki",
city="Medellin",
website="https://idioki.com/",
)
event_type = "Mu"
try:
scraper, created = Scraper.objects.get_or_create(
name="idioki",
website="https://calendar.google.com/calendar/ical/46ae0446724b1b3ee83cbd7dbc0db6a235bf97509ad860ca91eada3c267b5e41%40group.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(shortcode='mde'),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
def createEvent(day, date, event, scraper, venue, event_type):
ppr(event)
print("NAME: ", venue.name)
print('\n\n')
if venue.name == "DANCEFREE":
venue.website = "https://www.instagram.com/dancefreeco"
if venue.name == "Vintrash":
venue.website = "https://www.instagram.com/vintrashbar"
if venue.name == "The Wandering Paisa":
venue.website = "https://wanderingpaisahostel.com"
if venue.name == "Dulce Posion":
venue.website = "https://www.instagram.com/dulceposionr"
if venue.name == "Blood Dance Company":
venue.website = "https://www.instagram.com/blooddancecompany"
if venue.name == "OLSA Certified Spanish School":
venue.website = "https://www.olsafoundation.org/"
if event['strSummary'] == "Merli Rooftop Language Exchange":
venue.website = "https://calendar.google.com/calendar/embed?src=46ae0446724b1b3ee83cbd7dbc0db6a235bf97509ad860ca91eada3c267b5e41%40group.calendar.google.com&ctz=America%2FBogota"
if "Concious Warrior" in event['strSummary']:
venue.website = "https://www.consciouscolombia.com/"
# if venue.name == "":
# venue.website = "https://www.consciouscolombia.com/"
# if venue.name == "":
# venue.website = "https://www.consciouscolombia.com/"
# if venue.name == "":
# venue.website = "https://www.consciouscolombia.com/"
venue.save()
days = [day-1, day+6, day+13]
for day in days:
event['dateStamp'] = date + timedelta(days=day)
event['dateStart'] = event['dateStamp']
print("sending")
digitools.createCleanIcalEvent(event, scraper, venue, event_type)
return
def splitLocation(event):
loc_split = event['strLocation'].split(',')
venue_name = loc_split[0]
venue, created = Organization.objects.get_or_create(
name=venue_name,
city="Medellin",
)
event['venue'] = venue
return event
counter = 0
for component in gcal.walk():
event = {}
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
event['strSummary'] = f"{(component.get('SUMMARY'))}"
event['strDesc'] = component.get('DESCRIPTION')
event['strLocation'] = str(component.get('LOCATION'))
# startDate = component.get('DTSTART')
# startTime = startDate.time()
event['dateStart'] = component.get('DTSTART')
event['dateStamp'] = component.get('DTSTAMP')
if event['strSummary'] != 'None':
event['details'] = {
"description" : event['strDesc'],
"Location" : event['strLocation'],
}
if event['dateStamp'] != None:
event['dateStart'] = event['dateStart'].dt
event['dateStart'] = datetime.strptime(str(event['dateStart'])[:-6], '%Y-%m-%d %H:%M:%S')
rules = component.get('RRule')
try:
if rules['FREQ'][0] == 'WEEKLY':
if datetime.today().weekday() != 0:
event = splitLocation(event)
date = datetime.today().date() - timedelta(days=datetime.today().weekday())
date = datetime.combine(date, event['dateStart'].time())
days = ["SU", "MO", "TU", "WE", "TH", "FR", "SA"]
for day in rules['BYDAY']:
day = days.index(day)
createEvent(day, date, event, scraper, event['venue'], "Ed")
except Exception as e:
print("Error: ", e, "\n\n\n\n")
pass
digitools.updateScraper(scraper, item_count_start)
new_time = datetime.now() - timedelta(days=1)
right_bound_time = datetime.now() + timedelta(days=45)
events = DSEvent.objects.filter(show_date__lte=new_time)
events1 = DSEvent.objects.filter(show_date__gte=right_bound_time)
for e in events:
e.delete()
for e in events1:
e.delete()

118
Working/iCal/ical_online_events.py Normal file
View File

@@ -0,0 +1,118 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime, timedelta
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from config.env import env
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
td = relativedelta.relativedelta(hours=5)
venue, created = Organization.objects.get_or_create(
name="Online Events",
city="Online",
website="https://dreamfreely.org/",
)
event_type = "Mu"
try:
scraper, created = Scraper.objects.get_or_create(
name="Online Events",
website="https://calendar.google.com/calendar/ical/p1a4r9glkjpu4u6iv3fkmu8qtc%40group.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(shortcode='000'),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
def createEvent(day, date, event, scraper, venue, event_type):
days = [day-1, day+6, day+13]
for day in days:
event['dateStamp'] = date + timedelta(days=day)
event['dateStart'] = event['dateStamp']
digitools.createCleanIcalEvent(event, scraper, venue, event_type)
return
def splitLocation(event):
loc_split = event['strLocation'].split(',')
venue_name = loc_split[0]
venue, created = Organization.objects.get_or_create(
name=venue_name,
)
event['venue'] = venue
return event
counter = 0
for component in gcal.walk():
event = {}
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
event['strSummary'] = f"{(component.get('SUMMARY'))}"
event['strDesc'] = component.get('DESCRIPTION')
event['strLocation'] = str(component.get('LOCATION'))
# startDate = component.get('DTSTART')
# startTime = startDate.time()
event['dateStart'] = component.get('DTSTART')
event['dateStamp'] = component.get('DTSTAMP')
if event['strSummary'] != 'None':
event['details'] = {
"description" : event['strDesc'],
"Location" : event['strLocation'],
}
if 'Mikel' in event['strSummary']:
print('JELLOO \n\n\n JELOOO')
pass
elif event['dateStamp'] != None:
event['dateStart'] = event['dateStart'].dt
event['dateStart'] = datetime.strptime(str(event['dateStart'])[:-6], '%Y-%m-%d %H:%M:%S')
rules = component.get('RRule')
try:
if rules['FREQ'][0] == 'WEEKLY':
if datetime.today().weekday() != 0:
event = splitLocation(event)
date = datetime.today().date() - timedelta(days=datetime.today().weekday())
date = datetime.combine(date, event['dateStart'].time())
days = ["SU", "MO", "TU", "WE", "TH", "FR", "SA"]
for day in rules['BYDAY']:
day = days.index(day)
createEvent(day, date, event, scraper, event['venue'], "Ed")
except Exception as e:
print("Error (no repeat): ", e)
pass
digitools.updateScraper(scraper, item_count_start)
new_time = datetime.now() - timedelta(days=1)
right_bound_time = datetime.now() + timedelta(days=45)
events = DSEvent.objects.filter(show_date__lte=new_time)
events1 = DSEvent.objects.filter(show_date__gte=right_bound_time)
for e in events:
e.delete()
for e in events1:
e.delete()

65
Working/iCal/ical_run.SocialableCider.mpls.py Normal file
View File

@@ -0,0 +1,65 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
venue, created = Organization.objects.get_or_create(
name="Sociable Ciderwerks",
city="Minneapolis",
website="https://sociablecider.com/events",
)
event_type = "Mu"
try:
scraper, created = Scraper.objects.get_or_create(
name="Sociable Ciderwerks",
website="https://calendar.google.com/calendar/ical/c_oa7uitvkn871o1ojl5e1os4ve8%40group.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(id=1),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
for event in events:
# ppr(event)
e = {}
e['calendars'] = event['calendars']
try:
e['dateStamp'] = event['dateStart'][0]
except:
e['dateStamp'] = event['dateStart']
e['title'] = event['strSummary']
e['scraper'] = scraper
e['link'] = venue.website
try:
digitools.createBasicEvent(e, 'Mu', venue)
except Exception as e:
print("Error: ", e)
digitools.updateScraper(scraper, item_count_start)

66
Working/iCal/ical_run.bunkers.py Normal file
View File

@@ -0,0 +1,66 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
from datetime import datetime
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
venue, created = Organization.objects.get_or_create(
name="Bunkers",
city="Minneapolis",
website="https://bunkersmusic.com/calendar/",
is_venue = True
)
try:
scraper, created = Scraper.objects.get_or_create(
name="Bunkers",
website="https://calendar.google.com/calendar/ical/js94epu90r2et31aopons1ifm8%40group.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(id=1),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
event_type = "Mu"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
events = digitools.getiCalEvents(gcal, scraper, venue, "Mu")
for event in events:
# ppr(event)
e = {}
e['calendars'] = event['calendars']
try:
e['dateStamp'] = event['dateStart'][0]
except:
e['dateStamp'] = event['dateStart']
e['title'] = event['strSummary']
e['scraper'] = scraper
e['link'] = venue.website
try:
digitools.createBasicEvent(e, 'Mu', venue)
except Exception as e:
print("Error: ", e)
# digitools.returniCalEvents(gcal, scraper, venue, "Mu")
digitools.updateScraper(scraper, item_count_start)

93
Working/iCal/ical_run.cfpa.py Normal file
View File

@@ -0,0 +1,93 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime
from dateutil import relativedelta
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
tz = pytz.timezone("US/Central")
td = relativedelta.relativedelta(hours=5)
venue, created = Organization.objects.get_or_create(
name="Center for Performing Arts",
city="Minneapolis",
website="https://www.cfpampls.com/events",
)
try:
scraper, created = Scraper.objects.get_or_create(
name="Center for Performing Arts",
website="https://calendar.google.com/calendar/ical/6rpooudjg01vc8bjek1snu2ro0%40group.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(id=1),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
event_type = "Ed"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
for event in events:
ppr(event)
e = {}
e['calendars'] = event['calendars']
try:
e['dateStamp'] = event['dateStart'][0]
except:
e['dateStamp'] = event['dateStart']
e['title'] = event['strSummary']
e['scraper'] = scraper
e['link'] = venue.website
try:
digitools.createBasicEvent(e, 'Ed', venue)
except Exception as e:
print("Error: ", e)
# now_now = datetime.today().date()
# try:
# print("1Event: ", event['dateStart'])
# if event['dateStart'] > now_now:
# print("Check Check: ", event['dateStart'])
# ppr(event)
# # createIcalEvent(event, scraper, venue, event_type)
# else:
# print("WHAT?")
# ppr(event)
# except Exception as e:
# try:
# event['dateStart'] = event['dateStart'].date()
# print("2Event: ", event['dateStart'])
# if event['dateStart'] > now_now:
# print("Check Check: ", event['dateStart'])
# ppr(event)
# # createIcalEvent(event, scraper, venue, event_type)
# else:
# print("WHAT?")
# ppr(event)
# except Exception as e:
# print("The Error: ", e)
# pass
digitools.updateScraper(scraper, item_count_start)

63
Working/iCal/ical_run.eagles.py Normal file
View File

@@ -0,0 +1,63 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
venue, created = Organization.objects.get_or_create(
name="Eagles #34",
city="Minneapolis",
website="https://www.minneapoliseagles34.org/events-entertainment.html",
)
try:
scraper, created = Scraper.objects.get_or_create(
name="Eagles #34",
website="https://calendar.google.com/calendar/ical/teflgutelllvla7r6vfcmjdjjo%40group.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(id=1),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
event_type = "Mu"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
for event in events:
ppr(event)
e = {}
e['calendars'] = event['calendars']
try:
e['dateStamp'] = event['dateStart'][0]
except:
e['dateStamp'] = event['dateStart']
e['title'] = event['strSummary']
e['scraper'] = scraper
e['link'] = venue.website
try:
digitools.createBasicEvent(e, 'Mu', venue)
except Exception as e:
print("Error: ", e)
digitools.updateScraper(scraper, item_count_start)

67
Working/iCal/ical_run.terminalbar-mpls.py Normal file
View File

@@ -0,0 +1,67 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
td = relativedelta.relativedelta(hours=5)
odt = datetime.now() + td
venue, created = Organization.objects.get_or_create(
name="Terminal Bar",
city="Minneapolis",
website="https://terminalbarmn.com",
)
event_type = "Mu"
try:
scraper, created = Scraper.objects.get_or_create(
name="Terminal Bar",
website="https://calendar.google.com/calendar/ical/terminalbar32%40gmail.com/public/basic.ics",
calendar = Calendar.objects.get(id=1),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
event_type = "Mu"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
for event in events:
# ppr(event)
e = {}
e['calendars'] = event['calendars']
try:
e['dateStamp'] = event['dateStart'][0]
except:
e['dateStamp'] = event['dateStart']
e['title'] = event['strSummary']
e['scraper'] = scraper
e['link'] = venue.website
try:
digitools.createBasicEvent(e, 'Mu', venue)
except Exception as e:
print("Error: ", e)
digitools.updateScraper(scraper, item_count_start)

69
Working/iCal/ical_run.whitesquirrel.py Normal file
View File

@@ -0,0 +1,69 @@
import requests, os, sys
# from icalendar import Calendar as iCalendar, Event
from icalendar import Calendar as iCalendar
from datetime import datetime
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%Y-%m-%d %I:%M %p'
venue, created = Organization.objects.get_or_create(
name="White Squirrel",
city="St. Paul",
website="https://whitesquirrelbar.com",
is_venue = True
)
try:
scraper, created = Scraper.objects.get_or_create(
name="White Squirrel",
website="https://calendar.google.com/calendar/ical/vh5sr9h59nmrs2op5lmptu2fsa344cig%40import.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(shortcode='msp'),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
event_type = "Mu"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.content)
tz = pytz.timezone("US/Central")
events = digitools.getiCalEvents(gcal, scraper, venue, event_type)
for event in events:
ppr(event)
e = {}
e['calendars'] = event['calendars']
try:
e['dateStamp'] = event['dateStart'][0]
except:
e['dateStamp'] = event['dateStart']
e['title'] = event['strSummary']
e['scraper'] = scraper
e['link'] = venue.website
try:
digitools.createBasicEvent(e, event_type, venue)
except Exception as e:
print("Error: ", e)
digitools.updateScraper(scraper, item_count_start)

388
Working/iCal/index.html Normal file
View File

File diff suppressed because one or more lines are too long

498
Working/iCal/ws.ics Normal file
View File

@@ -0,0 +1,498 @@
BEGIN:VCALENDAR
PRODID:-//Google Inc//Google Calendar 70.9054//EN
VERSION:2.0
CALSCALE:GREGORIAN
METHOD:PUBLISH
X-WR-CALNAME:The White Squirrel Bar
X-WR-TIMEZONE:UTC
X-WR-CALDESC:Events for The White Squirrel Bar
BEGIN:VEVENT
DTSTART:20251118T030000Z
DTEND:20251118T060000Z
DTSTAMP:20251116T033816Z
UID:10806-1763413200-1763424000@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/the-deeper-kind-w-23-watts/
CREATED:20251104T233617Z
LAST-MODIFIED:20251105T053618Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:The Deeper Kind w. 23 Watts
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251129T000000Z
DTEND:20251129T020000Z
DTSTAMP:20251116T033816Z
UID:10852-1764352800-1764360000@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/the-mary-cutrufello-band-3/
CREATED:20251115T173617Z
DESCRIPTION:A rock and roll band! Killer harmonies\, rippin’ solos\, cool s
ongs\, funny\nshit in between. \nHailed as a country guitarist\, Mary Cutru
fello grew up on East Coast\nclassic rock. (Yes\, think Springsteen!) This
tight\, taut trio explores\nher songs old and new with the fearless guitars
\, stories\, and\nbetween-song banter she’s known for.
LAST-MODIFIED:20251115T233617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:The Mary Cutrufello Band
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251117T030000Z
DTEND:20251117T060000Z
DTSTAMP:20251116T033816Z
UID:10802-1763326800-1763337600@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/karaoke-with-ally/
CREATED:20251103T163617Z
LAST-MODIFIED:20251103T223619Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Karaoke With Ally!
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251127T000000Z
DTEND:20251127T020000Z
DTSTAMP:20251116T033816Z
UID:10846-1764180000-1764187200@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/family-dinner-w-doug-sarah-4/
CREATED:20251115T013617Z
LAST-MODIFIED:20251115T073617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Family Dinner w. Doug & Sarah
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251116T190000Z
DTEND:20251116T220000Z
DTSTAMP:20251116T033816Z
UID:10147-1763298000-1763308800@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/the-friend-ship/
CREATED:20251103T003616Z
DESCRIPTION:These popular singer/songwriters first got together to perform
on Joel Sax’s debut\nalbum\, ‘Billions of Stars.’ They had so much fun and
got such a great response at their\nsold-out release show that they have jo
ined together to form a new band\, The Friend\nShip! They will bring their
best songs\, sweetest harmonies\, and funniest stories to the\nWhite Squirr
el for a night of beautiful music. \nNikki Lemire is an incredibly in deman
d artist\, having earned accolades from fans and critics\nalike. Natalia To
ledo\, from The Current describes ‘Mend’ It as “a dreamy\, simple\, and\nbe
autiful song led by the sounds of her harp.”\nJoel Sax writes heartfelt son
gs about love\, regret\, and hope for a bigger\, brighter future.\nHis debu
t album title track\, Billions of Stars\, is “… a wonderful mix for a conte
mporary\nfolk song” – Minnesota Sound and ‘A Lovely New Album\,’ by Mostly
Minnesota\nMusic. \nMarc Severin has been a mainstay of the music scene for
many years. His music is\ntrue Americana and his storytelling is midwester
n country at its best. The Friend Ship\nwill be previewing songs from his w
ell anticipated new album coming out this Winter. \nDave Mehling is a highl
y sought-after musician\, producer and songwriter. He has\nproduced albums
for many artists\, including Sarah Morris\, Haley E Rydell\, Joel Sax and\n
Emily Haavik. “You are drawn in with his lyrics. They speak of very univers
al issues of\nlove which we all have\, but few of us have the talent to giv
e them a voice. Dave Mehling\npossesses this rare gift.” – Becca Martin
LAST-MODIFIED:20251103T063617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:The Friend Ship
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251123T000000Z
DTEND:20251123T020000Z
DTSTAMP:20251116T033816Z
UID:10827-1763834400-1763841600@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/giant-valley-string-band/
CREATED:20251109T183617Z
DESCRIPTION:Giant Valley String Band is a Twin Cities acoustic group playin
g an\neclectic mix of covers and originals in Bluegrass and Americana. The\
nband members are Noelle Haland on guitar\, Ted Haland on dobro and\nbass\,
Nate Hess on fiddle and guitar\, Erika Janik on banjo\, Matt\nJensen on ma
ndolin\, and Debra Pflipsen on guitar\, harmonica\,\naccordion\, and bass.
More info at giantvalleystringband.com. \n\n\n\n\nBlue Groove is a multiple
award-winning bluegrass band based in the\nTwin Cities that plays both con
temporary and classic bluegrass tunes\nwith a creative approach that drives
while still maintaining a sense\nof laid-back\, Midwestern chill. That’s w
hat they mean by “groove”. The\nbands inviting sound combined with their op
enness warmth on and\noff-stage have made them crowd favorites at festivals
and concert\nvenues all around Minnesota and the Upper Midwest. \nThe band
covers a number of past and current contemporary bluegrass\nartists. A Blu
e Groove show may feature songs from Alison Krauss or\nRhonda Vincent as we
ll as traditional bluegrass (Bill Monroe\, Flatt &\nScruggs) with all of it
s drive\, improvisation and harmony singing!\nBlue Groove also likes to put
a bluegrass spin on more\ncontemporary/pop artists such as U2\, Passenger\
, and the Doobie\nBrothers! \nMembers of Blue Groove bluegrass band include
Adelle Hyrkas\, lead\nvocals and rhythm guitar\; David Smith\, banjo and v
ocals\; Bob Doe\, lead\nguitar\, dobro\, and vocals\; Pete Mathison bass an
d vocals\, Tom\nWadzinski mandolin and vocals. Ocassionally we may have a g
uest\nfiddler- past guests have included fiddlers Richard Kriehn\, Michael\
nPrewitt\, and AJ Srubas. \nYou can find out more about Blue Groove at www.
bluegroovebluegrass.com
LAST-MODIFIED:20251110T003617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Giant Valley String Band w. Blue Groove
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251121T220000Z
DTEND:20251122T040000Z
DTSTAMP:20251116T033816Z
UID:11255-1763740800-1763762400@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/the-angry-line-cook-4/
CREATED:20251114T133616Z
DESCRIPTION:The Best Smash Burgers in town!
LAST-MODIFIED:20251114T193617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:The Angry Line Cook
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251127T030000Z
DTEND:20251127T060000Z
DTSTAMP:20251116T033816Z
UID:10848-1764190800-1764201600@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/nights-with-tim-2/
CREATED:20251115T013617Z
DESCRIPTION:Dig into the deeper\, progressive side of disco\, funk\, and R&
B from the ’70s and ’80s\, with the occasional turn into house and left-fie
ld gems. An all vinyl night courtesy of local selector Tim. He knows how to
keep the vibes at White Squirrel curious and moving.
LAST-MODIFIED:20251115T073617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Nights with Tim
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251122T190000Z
DTEND:20251122T220000Z
DTSTAMP:20251116T033816Z
UID:10825-1763816400-1763827200@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/caitlin-robertson/
CREATED:20251109T023617Z
LAST-MODIFIED:20251109T083618Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Caitlin Robertson
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251128T220000Z
DTEND:20251129T040000Z
DTSTAMP:20251116T033816Z
UID:11257-1764345600-1764367200@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/the-angry-line-cook-5/
CREATED:20251115T013617Z
DESCRIPTION:The Best Smash Burgers in town!
LAST-MODIFIED:20251115T073617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:The Angry Line Cook
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251122T000000Z
DTEND:20251122T020000Z
DTSTAMP:20251116T033816Z
UID:10821-1763748000-1763755200@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/chris-holm-friends-2/
CREATED:20251108T203618Z
LAST-MODIFIED:20251109T023619Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Chris Holm & Friends
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251123T030000Z
DTEND:20251123T060000Z
DTSTAMP:20251116T033816Z
UID:10829-1763845200-1763856000@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/13-howell/
CREATED:20251110T003617Z
LAST-MODIFIED:20251110T063617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:13 Howell w. Steph Was & The Secret Izz\, TH3
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251120T000000Z
DTEND:20251120T020000Z
DTSTAMP:20251116T033816Z
UID:10813-1763575200-1763582400@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/st-paul-mudsteppers-4/
CREATED:20251107T033617Z
LAST-MODIFIED:20251107T093617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:St Paul Mudsteppers
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251116T000000Z
DTEND:20251116T020000Z
DTSTAMP:20251116T033816Z
UID:10819-1763229600-1763236800@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/in-circles-sunny-day-real-estate-tri
bute-w-120-minutes/
CREATED:20251102T013616Z
DESCRIPTION:It’s the 30th anniversary of Sunny Day Real Estates LP2 record!
In Circles with be performing the album in its entirety. \n 
LAST-MODIFIED:20251102T063617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:In Circles (Sunny Day Real Estate Tribute) w. 120 Minutes
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251124T000000Z
DTEND:20251124T020000Z
DTSTAMP:20251116T033816Z
UID:10833-1763920800-1763928000@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/the-jeff-becker-band/
CREATED:20251110T223617Z
LAST-MODIFIED:20251111T043617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:The Jeff Becker Band
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251122T030000Z
DTEND:20251122T060000Z
DTSTAMP:20251116T033816Z
UID:10823-1763758800-1763769600@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/ancient-waves-2/
CREATED:20251108T203618Z
LAST-MODIFIED:20251109T023619Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Ancient Waves w. Muun Batos & Jarad Miles
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251121T000000Z
DTEND:20251121T020000Z
DTSTAMP:20251116T033816Z
UID:10815-1763661600-1763668800@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/devaney-friends-2/
CREATED:20251107T223617Z
LAST-MODIFIED:20251108T043617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Devaney & Friends w. Aaron James
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251121T030000Z
DTEND:20251121T060000Z
DTSTAMP:20251116T033816Z
UID:10817-1763672400-1763683200@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/country-oke-3/
CREATED:20251108T043618Z
LAST-MODIFIED:20251108T103619Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Country-oke
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251124T030000Z
DTEND:20251124T060000Z
DTSTAMP:20251116T033816Z
UID:10835-1763931600-1763942400@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/bryan-the-haggards/
CREATED:20251111T043617Z
LAST-MODIFIED:20251111T103617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Bryan & The Haggards w. Jump Loop\, The American Songbook
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251125T030000Z
DTEND:20251125T060000Z
DTSTAMP:20251116T033816Z
UID:10839-1764018000-1764028800@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/seven-seasons/
CREATED:20251112T013616Z
LAST-MODIFIED:20251112T073617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Slasher Film
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251126T030000Z
DTEND:20251126T060000Z
DTSTAMP:20251116T033816Z
UID:10841-1764104400-1764115200@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/third-date-tuesday-night-residency-2
/
CREATED:20251113T053616Z
LAST-MODIFIED:20251113T113617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Third Date Tuesday Night Residency
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251117T000000Z
DTEND:20251117T020000Z
DTSTAMP:20251116T033816Z
UID:10800-1763316000-1763323200@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/bingo-with-pete-3/
CREATED:20251103T003616Z
LAST-MODIFIED:20251103T063617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Bingo With Pete!
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251125T000000Z
DTEND:20251125T020000Z
DTSTAMP:20251116T033816Z
UID:10837-1764007200-1764014400@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/devil-dodger/
CREATED:20251111T203617Z
LAST-MODIFIED:20251112T023617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Devil Dodger
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251126T000000Z
DTEND:20251126T020000Z
DTSTAMP:20251116T033816Z
UID:10843-1764093600-1764100800@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/ali-grays-country-jamboree-2/
CREATED:20251112T233616Z
DESCRIPTION:Come on out for a tribute to the Darlings of Country music\, fe
aturing Ali Gray’s Country Jamboree! Hear songs from the catalogs of Lorett
a Lynn\, Tammy Wynette\, Tanya Tucker\, Dolly Parton\, Linda Ronstadt\, Pam
Tillis\, Patty Loveless and many more. We won’t leave out the Kings of Cou
ntry\, sung by guitar legend Dan Neale and drummer extraordinaire Scott Wen
um\, including favorites like Roger Miller\, Hank Williams\, Johnny Cash\,
Buck Owens\, Waylon Jennings and more!  Also featuring Dan Lowinger on bass
! 6-8.
LAST-MODIFIED:20251113T053616Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Ali Gray's Country Jamboree
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251116T030000Z
DTEND:20251116T060000Z
DTSTAMP:20251116T033816Z
UID:10798-1763240400-1763251200@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/lost-island-society-w-woolly-mack-po
ison-ivy-the-people/
CREATED:20251102T183617Z
LAST-MODIFIED:20251103T003617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Lost Island Society w. Woolly Mack\, Poison Ivy & The People
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251123T190000Z
DTEND:20251123T220000Z
DTSTAMP:20251116T033816Z
UID:10831-1763902800-1763913600@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/the-stress-of-her-regard/
CREATED:20251110T003617Z
LAST-MODIFIED:20251110T063617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:The Stress Of Her Regard
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251119T030000Z
DTEND:20251119T060000Z
DTSTAMP:20251116T033816Z
UID:10811-1763499600-1763510400@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/third-date-tuesday-night-residency/
CREATED:20251106T213618Z
LAST-MODIFIED:20251107T033618Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Third Date Tuesday Night Residency
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251119T000000Z
DTEND:20251119T020000Z
DTSTAMP:20251116T033816Z
UID:10809-1763488800-1763496000@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/the-new-havoline-supremes/
CREATED:20251105T053616Z
DESCRIPTION:The New Havoline Supremes are a Twin Cities based band\,  featu
ring the legendary Mary Cutrufello on guitar and vocals\, and Dan Lowinger
on guitar.  Formed in October of 2023\, the band plays a mix of classic cou
ntry and Mary’s original tunes\, and features Greg Schutte (Ryan Bingham\,
Mickey Hart\, Chasity Brown) on drums\, and Erik Lillestol (Cafe Accordion)
on bass.\n\nMary Cutrufello\n\nWith 30 years of experience in the music bu
siness\, Mary Cutrufello is everything from Texas honky-tonk heroine and fi
ery Midwestern roots-rocker to a powerhouse acoustic performer. Connecticut
-raised and Yale-educated\, Mary’s musical journey has taken her from the E
ast Coast to Houston and now to Minnesota.  Career highlights include: the
major label release of    ”When the Night is Through\,” on Mercury Records
in 1998\, appearances on national tv shows such as Tonight Show with Jay Le
no\, appearances at major festivals including Farm Aid\, and tours with Jim
mie Dale Gilmore and Tish Hinojosa.  Mary has seven self-released albums\,
and has played guitar locally for Annie Mack\, Michael Perry and the Long B
eds\, and many others.\n\n\n\nDan Lowinger\, a transplant from Portland\, O
R\, is a guitarist steeped in the tradition of Western swing and classic co
untry. Since moving to the Twin Cities in 2017\, Dan has become a fixture i
n the local country and Americana scene\, playing on a regular basis with a
ll star Western swing band Honky Tonk Jump\, and country rockers Cole Diamo
nd\, while also sharing the stage with local greats\, Martin Zellar (Gear D
addies)\, Tony Andreason (The Trashmen)\, Pat Donohue (Prairie Home Compani
on) and many others! He has performed regionally and toured nationally with
many different country and swing bands from the Pacific Northwest includin
g: The Barn Door Slammers\, Ralph Carney’s Pepper Grinders\, Vince Mira\, a
nd Western Centuries.
LAST-MODIFIED:20251105T113617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:The New Havoline Supremes
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251120T030000Z
DTEND:20251120T060000Z
DTSTAMP:20251116T033816Z
UID:10782-1763586000-1763596800@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/venturer-w-peeler-mild-manner/
CREATED:20251107T223617Z
DESCRIPTION:Venturer is a rock based band with many other genre influences
such as funk\, folk\, prog\, punk\, and jazz. Venturer has played around th
e United States\, mainly in Milwaukee\, Madison\, Chicago\, and Minneapolis
\, but also on the East Coast and greater Midwest.
LAST-MODIFIED:20251108T043617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Venturer w. Peeler & Mild Manner
TRANSP:OPAQUE
END:VEVENT
BEGIN:VEVENT
DTSTART:20251118T000000Z
DTEND:20251118T020000Z
DTSTAMP:20251116T033816Z
UID:10804-1763402400-1763409600@whitesquirrelbar.com
URL:https://whitesquirrelbar.com/event/clovers-daughter/
CREATED:20251103T223616Z
LAST-MODIFIED:20251104T043617Z
SEQUENCE:0
STATUS:CONFIRMED
SUMMARY:Clovers Daughter
TRANSP:OPAQUE
END:VEVENT
END:VCALENDAR

78
Working/news/minnpost.mn.py Normal file
View File

@@ -0,0 +1,78 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
from events.digitools import getBrowser, createURL, createBasicArticle, getSource
org, created = Organization.objects.get_or_create(
name="MinnPost",
city="Minneapolis",
website="https://www.minnpost.com/",
is_venue=False,
)
try:
scraper, created = Scraper.objects.get_or_create(
name=org.name,
website=org.website,
items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=org.name)
print("Scraper: ", scraper)
event_type = "Ja"
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b %d %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
def get_events(ps, event_type):
contents = ps.xpath('.//*/article')
count = 0
ppr(contents)
for c in contents:
try:
if count > 10:
br.close()
quit()
article = {}
article['title'] = c.xpath('.//*/h2[@class="entry-title"]/a/text()')[0]
article['link'] = c.xpath('.//*/h2[@class="entry-title"]/a/@href')[0]
createBasicArticle(article, event_type, org)
ppr(article)
print("Success")
count+=1
except Exception as e:
print(e)
ppr(article)
print("\n\n+++\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
ps = getSource(br, org.website)
get_events(ps, "Ed")
sleep(3)
br.close()
scraper.save()

68
Working/news/racket.mn.py Normal file
View File

@@ -0,0 +1,68 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
from events.digitools import getBrowser, createURL, createBasicArticle, getSource
scraper, created = Scraper.objects.get_or_create(
name="Racket MN",
website="https://racketmn.com",
last_ran = datetime.now(),
)
org, created = Organization.objects.get_or_create(
name="Racket MN",
city="Minneapolis",
website="https://racketmn.com",
is_venue=False,
)
event_type = "Ja"
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b %d %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
def get_events(ps, event_type):
count = 0
contents = ps.xpath('.//*/div[@class="PostCard_stackedWrapper__S21Fy"]') + ps.xpath('.//*/div[@class="PostCard_wrapper__uteO3"]')
for c in contents:
if count > 10:
br.close()
quit()
try:
article = {}
article['title'] = c.xpath('.//div/a/h3/text()')[0]
article['link'] = org.website + c.xpath('.//div/a/@href')[1]
createBasicArticle(article, event_type, org)
count+=1
except Exception as e:
print(e)
ppr(article)
print("\n+++\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
ps = getSource(br, org.website)
get_events(ps, "Ed")
sleep(3)
br.close()
scraper.save()

68
Working/news/sahan.mn.py Normal file
View File

@@ -0,0 +1,68 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
from events.digitools import getBrowser, createURL, createBasicArticle, getSource
scraper, created = Scraper.objects.get_or_create(
name="Sahan Journal",
website="https://sahanjournal.com/",
last_ran = datetime.now(),
)
org, created = Organization.objects.get_or_create(
name="Sahan Journal",
city="Minneapolis",
website="https://sahanjournal.com/",
is_venue=False,
)
event_type = "Ja"
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b %d %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
def get_events(ps, event_type):
contents = ps.xpath('.//*/article')
count = 0
for c in contents:
try:
if count > 10:
br.close()
quit()
article = {}
article['title'] = c.xpath('.//*/h2[@class="entry-title"]/a/text()')[0]
article['link'] = c.xpath('.//*/h2[@class="entry-title"]/a/@href')[0]
createBasicArticle(article, event_type, org)
count+=1
except Exception as e:
print(e)
ppr(article)
print("\n+++\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
ps = getSource(br, org.website)
get_events(ps, "Ed")
sleep(3)
br.close()
scraper.save()

63
Working/news/unicornriot.py Normal file
View File

@@ -0,0 +1,63 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
from events.digitools import getBrowser, createURL, createBasicArticle, getSource
scraper, created = Scraper.objects.get_or_create(
name="Uniocorn Riot",
website="https://unicornriot.ninja/",
last_ran = datetime.now(),
)
org, created = Organization.objects.get_or_create(
name="Uniocorn Riot",
city="Minneapolis",
website="https://unicornriot.ninja/",
is_venue=False,
)
event_type = "Ja"
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b %d %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
def get_events(ps, event_type):
contents = ps.xpath('.//*/article')
for c in contents[:10]:
try:
article = {}
article['title'] = c.xpath('.//*/h3[@class="title entry-title is-3"]/a/text()')[0]
article['link'] = c.xpath('.//*/h3[@class="title entry-title is-3"]/a/@href')[0]
createBasicArticle(article, event_type, org)
except Exception as e:
print(e)
ppr(article)
print("\n+++\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
ps = getSource(br, org.website)
get_events(ps, "Ed")
sleep(3)
br.close()
scraper.save()

132
Working/smedia/bluesky.py Normal file
View File

@@ -0,0 +1,132 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
from atproto import Client
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from socials.models import SocialLink, SocialPost
# from digitools import getBrowser, createURL, createBasicEvent, getSource
tz = pytz.timezone("US/Central")
USERNAME = "dreamfreely.org"
PASSWORD = "gU):3-BA]DaK[_K$DHmI"
client = Client()
client.login(USERNAME, PASSWORD)
feed = client.get_author_feed(USERNAME, limit = 100)
def createSocialLink(post):
new_post, created = SocialLink.objects.update_or_create(
uri = post['uri'],
text = post['text'],
link = post['link'],
handle = post['handle'],
likes = post['likes'],
reposts = post['reposts'],
quotes = post['quotes'],
replies = post['replies'],
created_at = post['created_at'],
platform = 'bluesky',
rt_uri = post['rt_uri'],
rt_text = post['rt_text'],
rt_link = post['rt_link'],
rt_handle = post['rt_handle'],
)
# print(created, new_post)
print("completed write")
tweets = []
print(len(feed.feed))
for post in feed.feed:
post = post.post
print("\n\nNEW POST\n\n")
# try:
# ppr(post.embed.record.record.author.handle)
# ppr(post.embed.record.record.value.text.split("\n")[:2])
# ppr(post.embed.record.record.value.embed.external.uri.split("?")[0])
# ppr(post.embed.record.record.uri.split("feed.post/")[1])
# except:
# pass
if hasattr(post.record.embed, 'external'):
p = {}
try:
p['link'] = post.record.embed.external.uri.split("?")[0]
except:
pass
p['text'] = " ".join(post.record.text.split("\n")[:2])
p['handle'] = post.author.handle
p['uri'] = post.uri.split("feed.post/")[1]
p['likes'] = post.like_count
p['quotes'] = post.quote_count
p['replies'] = post.reply_count
p['reposts'] = post.repost_count
p['created_at'] = post.record.created_at
p['rt_handle'] = "blank"
p['rt_text'] = "blank"
p['rt_uri'] = "blank"
p['rt_link'] = "blank"
elif hasattr(post.embed, 'record'):
p = {}
p['text'] = " ".join(post.record.text.split("\n")[:2])
p['handle'] = post.author.handle
p['uri'] = post.uri.split("feed.post/")[1]
p['likes'] = post.like_count
p['quotes'] = post.quote_count
p['replies'] = post.reply_count
p['reposts'] = post.repost_count
p['created_at'] = post.record.created_at
p['link'] = "blank"
try:
p['rt_handle'] = post.embed.record.record.author.handle
p['rt_text'] = " ".join(post.embed.record.record.value.text.split("\n")[:2])
p['rt_uri'] = post.embed.record.record.uri.split("feed.post/")[1]
p['rt_link'] = post.embed.record.record.value.embed.external.uri.split("?")[0]
except:
p['rt_handle'] = "blank"
p['rt_text'] = "blank"
p['rt_uri'] = "blank"
p['rt_link'] = "blank"
else:
p = {}
p['text'] = " ".join(post.record.text.split("\n")[:2])
p['handle'] = post.author.handle
p['uri'] = post.uri.split("feed.post/")[1]
p['likes'] = post.like_count
p['quotes'] = post.quote_count
p['replies'] = post.reply_count
p['reposts'] = post.repost_count
p['created_at'] = post.record.created_at
p['rt_handle'] = "blank"
p['rt_text'] = "blank"
p['rt_uri'] = "blank"
p['rt_link'] = "blank"
p['link'] = "blank"
# ppr(p)
# tweets.append(p)
try:
print('writing file')
createSocialLink(p)
except Exception as e:
ppr(post.record.embed)
print(e, "\nthis\n\n")

72
Working/smedia/bluesky_media.py Normal file
View File

@@ -0,0 +1,72 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
from atproto import Client
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from socials.models import SocialImg
# from digitools import getBrowser, createURL, createBasicEvent, getSource
tz = pytz.timezone("US/Central")
USERNAME = "dreamfreely.org"
PASSWORD = "gU):3-BA]DaK[_K$DHmI"
client = Client()
client.login(USERNAME, PASSWORD)
feed = client.get_author_feed(USERNAME, limit = 100)
def createSocialImg(post):
new_post, created = SocialImg.objects.update_or_create(
uri = post['uri'],
text = post['text'],
img_link = post['img_link'],
handle = post['handle'],
created_at = post['created_at'],
platform = 'bluesky',
)
print(created, new_post)
tweets = []
print(len(feed.feed))
for post in feed.feed:
post = post.post
# print(post, "\n\n")
# try:
# ppr(post.embed.images[0].fullsize)
# # ppr(post.embed.record.record.value.text.split("\n")[:2])
# # ppr(post.embed.record.record.value.embed.external.uri.split("?")[0])
# # ppr(post.embed.record.record.uri.split("feed.post/")[1])
# except Exception as e:
# print("failed:", e)
if hasattr(post.embed, 'images'):
p = {}
p['img_link'] = post.embed.images[0].fullsize
p['text'] = " ".join(post.record.text.split("\n")[:2])
p['handle'] = post.author.handle
p['uri'] = post.uri.split("feed.post/")[1]
p['created_at'] = post.record.created_at
# ppr(p)
tweets.append(p)
try:
print('writing file')
createSocialImg(p)
except Exception as e:
ppr(post.embed)
print(e, "\nthis\n\n")

72
Working/smedia/redsky.py Normal file
View File

@@ -0,0 +1,72 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
import praw
from socials.models import SocialLink, SocialPost
# from digitools import getBrowser, createURL, createBasicEvent, getSource
tz = pytz.timezone("US/Central")
# timestamp = 1729322547223
# dt_object = datetime.datetime.fromtimestamp(timestamp)
# print(dt_object)
reddit = praw.Reddit(
client_id="rxW3Ywqke6FZDP7pIhYYuw",
client_secret="cg1VNl0I-RTuYUwgz16ryKh2wWKEcA",
password="7CTu4sGFi9E0",
user_agent="CultureClap",
username="cultureclap",
)
def createSocialLink(post):
new_post, created = SocialLink.objects.update_or_create(
text = post['text'],
link = post['link'],
handle = post['handle'],
likes = post['likes'],
replies = post['replies'],
platform = post['platform'],
created_at = post['created_at'],
rt_uri = 'blank',
rt_text = 'blank',
rt_link = 'blank',
rt_handle = 'blank',
)
print(created, new_post)
count = 0
for item in reddit.user.me().upvoted():
rdt = {}
rdt['text'] = item.title + " | " + item.selftext
rdt['handle'] = item.author.name
rdt['link'] = item.url
rdt['likes'] = item.ups
rdt['replies'] = len(item.comments.list())
rdt['created_at'] = datetime.fromtimestamp(item.created_utc)
rdt['platform'] = 'reddit'
try:
print('writing file')
createSocialLink(rdt)
count +=1
if count > 50:
quit()
except Exception as e:
ppr(item)
print(e, "\nthis\n\n")
# ppr(item)

78
Working/venues/AcmeComedy.Mpls.py Normal file
View File

@@ -0,0 +1,78 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
# import django
# sys.path.append('../../../../')
sys.path.append('/var/www/digisnaxx.ado/scrapers')
import digitools as digitools_sm
print("Got DigiTools")
digitools_sm.getReady()
# os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
# django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
import events.digitools as digitools
from events.models import Organization, Scraper, Calendar, Event
venue, created = Organization.objects.get_or_create(
name="Acme Comedy Club",
city="Minneapolis",
website="https://acmecomedycompany.com/the-club/calendar/",
is_venue = True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b %d %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
def get_events(ps, event_type):
contents = ps.xpath('.//*/li[@class="event"]')
for c in contents:
try:
event = {}
day = c.xpath('.//*/span[@class="day"]/text()')[0]
month = c.xpath('.//*/span[@class="mth"]/text()')[0]
year = datetime.now().year
if month == "Jan":
year = int(year) + 1
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
event['title'] = c.xpath('.//*/span[@class="event_title"]/a/text()')[0]
event['date'] = [month, day, str(year), c.xpath('.//*/span[@class="event_time"]/text()')[0].strip()]
event['date'] = " ".join(event['date'])
event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
event['link'] = c.xpath('.//*/span[@class="event_title"]/a/@href')[0]
digitools.createBasicEvent(event, "Co", venue)
scraper.items+=1
except Exception as e:
print(e)
ppr(event)
print("\n\n+++\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
br.close()
quit()
links = digitools.createURL("https://acmecomedycompany.com/the-club/calendar/")
for link in links:
ps = digitools.getSource(br, link)
get_events(ps, "Co")
digitools.updateScraper(scraper, item_count_start)
br.close()

67
Working/venues/Amsterdam.StPaul.py Normal file
View File

@@ -0,0 +1,67 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper, Calendar
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="Amsterdam Bar & Hall",
city="St. Paul",
website="https://www.amsterdambarandhall.com/events-new/",
is_venue=True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
DATETIME_FORMAT = '%B %d %Y %I:%M%p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
def get_events(ps):
contents = ps.xpath('.//*/ul[@class="events-list"]/li')
for c in contents:
try:
event = {}
day = c.xpath('.//*/div[@class="date-day"]/text()')[0]
month = c.xpath('.//*/div[@class="date-month"]/text()')[0]
year = datetime.now().year
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
event['title'] = c.xpath('.//div/h4/a/text()')[0]
event['date'] = [month, day, str(year), c.xpath('.//div[@class="event-info"]/p/text()')[0].split(" ")[0]]
event['date'] = " ".join(event['date'])
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT)
event['link'] = c.xpath('.//div[@class="event-info"]/h4/a/@href')[0]
if " presents" in event['title']:
event['title'] = event['title'].split("presents")[1][1:].strip()
if event['title'].startswith('.'):
print("BLAHH\n")
event['title'] = event['title'][1:].strip()
digitools.createBasicEvent(event, "Mu", venue)
scraper.items+=1
except Exception as e:
print(e)
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
ps = digitools.getSource(br, venue.website)
get_events(ps)
sleep(3)
br.close()
digitools.updateScraper(scraper, item_count_start)

178
Working/venues/FirstAveScrape.py Normal file
View File

@@ -0,0 +1,178 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
from lxml import html
import pytz
from events.models import Organization, Scraper, Event
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="First Avenue",
city="Minneapolis",
website="https://first-avenue.com",
is_venue = True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b %d %Y %I%p'
DATETIME_FORMAT_2 = '%b %d %Y %I:%M%p'
DATETIME_FORMAT_3 = '%b %d %Y'
# Set initial variables for City, etc
month = int(datetime.now().month)
day = int(datetime.now().day)
if month == 12:
next_month = "01"
else:
next_month = month + 1
if next_month < 10:
next_month = "0" + str(next_month)
if month < 10:
month = "0" + str(month)
year = int(datetime.now().year)
calendar_url = 'https://first-avenue.com/shows/?start_date=' + str(year) + str(month) + str(day)
next_month_string = str(next_month) + "01"
if next_month == 1:
calendar_url_2 = 'https://first-avenue.com/shows/?start_date=' + str(year + 1) + next_month_string
else:
if int(next_month) == 1:
calendar_url_2 = 'https://first-avenue.com/shows/?start_date=' + str(year + 1) + next_month_string
else:
calendar_url_2 = 'https://first-avenue.com/shows/?start_date=' + str(year) + next_month_string
print("\n\n", calendar_url, calendar_url_2, "\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
if datetime.now().day < 8:
ps = digitools.getSource(br, calendar_url)
shows = ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')[:63]
elif 7 < datetime.now().day < 15:
ps = digitools.getSource(br, calendar_url)
shows = ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')
elif 14 < datetime.now().day < 21:
ps = digitools.getSource(br, calendar_url)
shows = ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')[:95]
ps = digitools.getSource(br, calendar_url_2)
shows = shows + ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')[:31]
else:
ps = digitools.getSource(br, calendar_url)
shows = ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')
ps = digitools.getSource(br, calendar_url_2)
shows = shows + ps.xpath('.//*/div[@class="show_name content flex-fill"]/div/div/h4/a/@href')[:63]
events = []
def get_info(pse):
event = {}
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
event["venue"] = pse.xpath('.//*/div[@class="content"]/div/div[@class="venue_name"]/text()')[0].replace('\t', '').replace('\n', '').strip()
event["show_title"] = pse.xpath('.//*/span[@class="show_title"]/text()')[0].replace('\t', '').replace('\n', '')
if event["show_title"] == "":
event["show_title"] = pse.xpath('.//*/span[@class="show_title"]/text()')[2].replace('\t', '').replace('\n', '')
event["guests"] = pse.xpath('.//*/div[@class="feature_details_main d-flex align-items-center"]/div/h4/text()')
event["flyer"] = pse.xpath('.//*/img[@class="gig_poster lazy loaded"]/@src')
try:
event = get_date(pse, event)
except Exception as e:
print("date issue: ", e)
try:
event = get_details(pse, event)
except Exception as e:
print("details issue: ", e)
try:
event["date_time"] = datetime.strptime(" ".join(event["date"]) + " " + event["details"]["Doors Open"], DATETIME_FORMAT)
except Exception as e:
print("Using alt date format 2: ", e)
try:
event["date_time"] = datetime.strptime(" ".join(event["date"]) + " " + event["details"]["Doors Open"], DATETIME_FORMAT_2)
ppr(event)
except Exception as e:
print("Using alt date format 3: ", e)
print(event['date'])
event["date_time"] = datetime.strptime(" ".join(event["date"]), DATETIME_FORMAT_3)
return event
def get_date(pse, event):
month = pse.xpath('.//*/div[@class="date_container"]/div/div[@class="month"]/text()')[0].replace('\t', '').replace('\n', '')
day = pse.xpath('.//*/div[@class="date_container"]/div/div[@class="day"]/text()')[0].replace('\t', '').replace('\n', '')
year = pse.xpath('.//*/div[@class="date_container"]/div/div[@class="year"]/text()')[0].replace('\t', '').replace('\n', '')
event["date"] = [month, day, year]
return event
def get_details(pse, event):
try:
details = pse.xpath('.//*/div[@class="show_details text-center"]/div/div/h6/text()')
info = pse.xpath('.//*/div[@class="show_details text-center"]/div/div/h2/text()')
di = zip(details, info)
details = {}
for d,i in di:
details[d] = i
event["details"] = details
return event
except Exception as e:
print("details issue: ", e)
for show in shows:
br.get(show)
sleep(2)
try:
pse = html.fromstring(br.page_source)
except Exception as e:
print(show)
pass
try:
event = get_info(pse)
except Exception as e:
print("get_info error: ", e)
try:
event["link"] = show
if event["venue"] in ["Palace Theater", "Turf Club", "The Fitzgerald Theater", "Amsterdam Bar & Hall"]:
venue, created = Organization.objects.get_or_create(name=event["venue"], is_venue=True, city="St. Paul")
else:
venue, created = Organization.objects.get_or_create(name=event["venue"], is_venue=True, city="Minneapolis")
except Exception as e:
print("Venue creation error: ", e, "\n", event, "\n", event["venue"])
try:
event['dateStamp'] = event['date_time']
event['scraper'] = scraper
new_event, created = digitools.createDetailedEvent(event, "Mu", venue, scraper)
scraper.items+=1
except Exception as e:
print("event creation error: ", e, "\n\n", event, "\n\n", created)
quit()
ppr(events)
br.close()
digitools.updateScraper(scraper, item_count_start)
# br.find_element_by_class_name('fc-btn_allCalendars-button').click()

68
Working/venues/GinkgoCoffee.stp.py Normal file
View File

@@ -0,0 +1,68 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="Ginkgo Coffee",
city="Saint Paul",
website="https://ginkgocoffee.com/events/",
is_venue = True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
event_type = ""
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT_2 = '%b %d %Y %I:%M %p'
DATETIME_FORMAT = '%B %d @ %I:%M %p %Y'
def get_events(ps, event_type):
contents = ps.xpath('.//*/article')
for c in contents:
try:
event = {}
dateTime = c.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0]
month = c.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].split(' ')[0]
year = datetime.now().year
if month == "January":
year = int(year) + 1
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
event['title'] = c.xpath('.//*/h3/a/text()')[0].replace("\n", "").replace("\t", "")
event['date'] = " ".join([ dateTime, str(year)])
event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
event['link'] = c.xpath('.//*/h3/a/@href')[0]
digitools.createBasicEvent(event, event_type, venue)
except Exception as e:
print(e)
ppr(event)
print("\n\n+++\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
ps = digitools.getSource(br, venue.website)
get_events(ps, "Mu")
sleep(3)
br.close()
digitools.updateScraper(scraper, item_count_start)

71
Working/venues/GreenRoom.Mpls.py Normal file
View File

@@ -0,0 +1,71 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="Green Room",
city="Minneapolis",
website="https://www.greenroommn.com/events",
is_venue = True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
event_type = "Mu"
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a %b %d %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
def get_events(ps, event_type):
contents = ps.xpath('.//*/div[@class="vp-event-card vp-venue-greenroom vp-col"]')
for c in contents:
try:
event = {}
time = c.xpath('.//*/span[@class="vp-time"]/text()')[0].strip()
date = c.xpath('.//*/span[@class="vp-date"]/text()')[0].strip()
month = date.split(" ")[1]
year = datetime.now().year
# if month == "Jan":
# year = int(year) + 1
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
event['title'] = c.xpath('.//*/div[@class="vp-event-name"]/text()')[0]
event['datetime'] = date + " " + str(year) + " " + time
event['dateStamp'] = datetime.strptime(event['datetime'], DATETIME_FORMAT)
event['link'] = venue.website + c.xpath('.//a[@class="vp-event-link"]/@href')[0]
digitools.createBasicEvent(event, event_type, venue)
scraper.items+=1
except Exception as e:
print(e)
ppr(event)
print("\n+++\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
ps = digitools.getSource(br, venue.website)
get_events(ps, event_type)
sleep(3)
br.close()
digitools.updateScraper(scraper, item_count_start)

96
Working/venues/HookLadderScrape.py Normal file
View File

@@ -0,0 +1,96 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper, Event
import events.digitools as digitools
from lxml import html
count = 0
venue, created = Organization.objects.get_or_create(
name="Hook & Ladder",
city="Minneapolis",
website="https://thehookmpls.com",
is_venue=True,
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
ppr(scraper)
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a, %b %d, %Y @ %I:%M %p'
# Set initial variables for City, etc
calendar_url = [
"https://thehookmpls.com/events/list/page/1",
"https://thehookmpls.com/events/list/page/2",
"https://thehookmpls.com/events/list/page/3"
]
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
def get_listings(pse, events):
nevents = pse.xpath('.//*/article')
for event in nevents:
e = {}
e['datetime'] = event.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0]
e['show_title'] = event.xpath('.//*/header/h2/a/@title')[0]
e['link'] = event.xpath('.//*/header/h2/a/@href')[0]
try:
e['subtitle'] = event.xpath('.//*/header/div[@class="eventSubHead"]/text()')[0]
except:
continue
try:
e['price'] = event.xpath('.//*/span[@class="tribe-events-c-small-cta__price"]/strong/text()')[0].replace("Tickets ", "")
except:
e['price'] = "See Link"
e['image'] = event.xpath('.//*/img/@data-src')[0]
e["date_time"] = datetime.strptime(e['datetime'], DATETIME_FORMAT)
e['scraper'] = scraper
e['calendar'] = [scraper.calendar]
events.append(e)
events = []
for cal in calendar_url:
br.get(cal)
sleep(3)
pse = html.fromstring(br.page_source)
get_listings(pse, events)
for event in events:
try:
new_event = Event.objects.update_or_create(
scraper = event['scraper'],
event_type = 'Mu',
show_title = event["show_title"],
show_link = event["link"],
show_date = event["date_time"],
show_day = event["date_time"],
guests = " ".join(event["subtitle"]),
venue = venue
)
digitools.add_calendar(new_event, 'msp')
except Exception as e:
print("oops ", e, "\n\n", "Scraper:", scraper)
br.close()
digitools.updateScraper(scraper, item_count_start)

70
Working/venues/MagersQuinn.py Normal file
View File

@@ -0,0 +1,70 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="Magers & Quinn",
city="Minneapolis",
website="https://www.magersandquinn.com/events",
is_venue=False
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
DATETIME_FORMAT = '%A, %B %d , %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
def get_events(ps, event_type):
contents = ps.xpath('.//*/div[@class="day has-event"]')
for c in contents:
try:
event = {}
day = c.xpath('.//*/div[@class="dd"]/text()')[0]
month = c.xpath('.//*/div[@class="month"]/text()')[0]
year = c.xpath('.//*/div[@class="year"]/text()')[0]
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
event['title'] = c.xpath('.//*/h3/text()')[0]
event['date'] = [month, day, year, c.xpath('.//*/p[@class="time"]/text()')[0]]
event['date'] = " ".join(event['date'])
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT)
event['link'] = "https://www.magersandquinn.com" + c.xpath('.//a[@class="event in-store"]/@href')[0]
digitools.createBasicEvent(event, "Ed", venue)
scraper.items+=1
except Exception as e:
event['link'] = "https://www.magersandquinn.com" + c.xpath('.//a[@class="event off-site"]/@href')[0]
print("Error: ", e)
ppr(event)
digitools.createBasicEvent(event, "Ed", venue)
print("\n\n+++\n\n")
links = digitools.createBasicURL("https://www.magersandquinn.com/events/")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
for link in links:
ps = digitools.getSource(br, link)
get_events(ps, "Ed")
sleep(3)
# ppr(events)
br.close()
digitools.updateScraper(scraper, item_count_start)

80
Working/venues/MplsVFW.py Normal file
View File

@@ -0,0 +1,80 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper, Event
import events.digitools as digitools
from selenium.webdriver.common.by import By
from lxml import html
venue, created = Organization.objects.get_or_create(
name="Uptown VFW",
city="Minneapolis",
website="https://noboolpresents.com/venues/uptown-vfw/",
is_venue = True
)
scraper,item_count_start, virtcal = digitools.getScraper(venue, 'msp')
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a %B %d @ %I:%M %p %Y'
DATETIME_FORMAT_2 = '%b %d %I:%M%p %Y'
DATETIME_FORMAT_3 = '%b %d %Y'
# Set initial variables for City, etc
calendar_url = 'https://noboolpresents.com/venues/uptown-vfw/'
current_year = str(datetime.now().year)
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
br.get(calendar_url)
sleep(20)
def getEvents(br):
ps = html.fromstring(br.page_source)
events = ps.xpath('.//*/article')
for event in events:
deets = {}
dateTime = event.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].replace("•", "").strip() + " " + current_year
title = event.xpath('.//*/h2[@class="alt-font"]/a/text()')[0].replace("\n", "").replace("\t", "")
link = event.xpath('.//*/h2[@class="alt-font"]/a/@href')[0]
# deets["tickets"] = event.xpath('.//*/span[@class="tribe-events-c-small-cta__price"]/strong/text()')[0]
try:
new_event, created = Event.objects.update_or_create(
scraper = scraper,
event_type = 'Mu',
show_title = title,
show_link = link,
show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
show_day = datetime.strptime(dateTime, DATETIME_FORMAT),
# more_details = deets["tickets"],
venue = venue
)
digitools.add_calendar(new_event, 'msp')
scraper.items+=1
except Exception as e:
print("oops", e)
getEvents(br)
br.find_element(By.XPATH, './/*/li[@class="tribe-events-c-nav__list-item tribe-events-c-nav__list-item--next"]/a').click()
sleep(5)
getEvents(br)
br.close()
digitools.updateScraper(scraper, item_count_start)

106
Working/venues/ParkwayTheater.py Normal file
View File

@@ -0,0 +1,106 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper, Event as DSEvent
import events.digitools as digitools
try:
venue, created = Organization.objects.get_or_create(
name="Parkway Theater",
city="Minneapolis",
website="https://theparkwaytheater.com",
is_venue = True
)
except Exception as e:
venue = Organization.objects.get(name="Parkway Theater")
scraper,item_count_start, virtcal = digitools.getScraper(venue, 'msp')
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b %d, %Y %I:%M %p'
def get_events(ps, event_type):
contents = ps.xpath('.//*/div[@class="summary-content sqs-gallery-meta-container"]')
img_etc = ps.xpath('.//*/div[@class="summary-thumbnail-outer-container"]/a/div/img/@src')
ps.xpath('.//*/span[@class="event-time-12hr"]/text()')
for c,i in zip(contents,img_etc):
try:
event = {}
event['calendars'] = [scraper.calendar]
event['title'] = c.xpath('.//*/a[@class="summary-title-link"]/text()')[0]
event['link'] = "https://theparkwaytheater.com" + c.xpath('.//*/a[@class="summary-title-link"]/@href')[0]
event['date'] = c.xpath('.//div/div/time/text()')[0] + " " + c.xpath('.//*/span[@class="event-time-12hr"]/text()')[0].split("–")[0].strip()
event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
event['desc'] = c.xpath('.//*/p/text()')[0]
event['img_link'] = i
event['details'] = {
'description': event['desc'],
'img_link': event['img_link'],
}
try:
new_event, created = DSEvent.objects.update_or_create(
scraper = scraper,
event_type = event_type,
show_title = event['title'],
show_link = event['link'],
show_date = datetime.strptime(event['date'], DATETIME_FORMAT),
show_day = datetime.strptime(event['date'], DATETIME_FORMAT),
more_details = event["details"],
venue = venue
)
digitools.add_calendar(new_event, 'msp')
scraper.items+=1
except Exception as e:
try:
event['date'] = c.xpath('.//div/div/time/text()')[0].split("–")[0] + " " + c.xpath('.//*/span[@class="event-time-12hr"]/text()')[0].split("–")[0].strip()
event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
new_event, created = DSEvent.objects.update_or_create(
scraper = scraper,
event_type = event_type,
show_title = event['title'],
show_link = event['link'],
show_date = datetime.strptime(event['date'], DATETIME_FORMAT),
show_day = datetime.strptime(event['date'], DATETIME_FORMAT),
more_details = event["details"],
venue = venue
)
digitools.add_calendar(new_event, 'msp')
scraper.items+=1
except Exception as e:
print(e)
print("\n\n+++\n\n")
except Exception as e:
continue
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
calendar_url = 'https://theparkwaytheater.com/live-events'
ps = digitools.getSource(br, calendar_url)
get_events(ps, "Mu")
calendar_url = "https://theparkwaytheater.com/movies"
ps = digitools.getSource(br, calendar_url)
get_events(ps, "Th")
# ppr(events)
br.close()
digitools.updateScraper(scraper, item_count_start)

97
Working/venues/SPCO.stp.py Normal file
View File

@@ -0,0 +1,97 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="St Paul Chamber Orchestra",
city="St Paul",
website="https://thespco.org",
is_venue = False
)
scraper,item_count_start, virtcal = digitools.getScraper(venue, 'msp')
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%A, %B %d, %Y – %I:%M %p'
def get_events(ps, event_type):
contents = ps.xpath('.//*/div[@class="event-title"]/a/@href')
for c in set(contents):
try:
link = 'https://content.thespco.org' + c
ps = digitools.getSource(br, link)
ntitle = ps.xpath('.//*/article/h1/text()')
subtitle = ps.xpath('.//*/article/h1/em/text()')
event = {}
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
if len(subtitle) == 1:
if len(ntitle) == 2:
title = ntitle[0] + subtitle[0] + ntitle[1]
elif ntitle[0].startswith(" "):
title = subtitle[0] + ntitle[0]
else:
title = ntitle[0] + subtitle[0]
else:
title = ntitle[0]
events = ps.xpath('.//*/div[@class="day"]')
for e in events:
new_venue = e.xpath('.//*/strong[@class="venue"]/text()')[0].strip()
location = e.xpath('.//*/span[@class="location"]/text()')[0].strip()
if 'Minneapolis' in location:
location = 'Minneapolis'
elif 'St. Paul' in location:
location = 'St. Paul'
else:
location = location
venue, created = Organization.objects.get_or_create(
name=new_venue,
city=location,
is_venue = True
)
dateTime = e.xpath('.//*/h3[@class="date"]/text()')[0].replace("\n", "").replace("\t", "").strip()
event['dateStamp'] = datetime.strptime(dateTime, DATETIME_FORMAT)
event['venue'] = venue
event['location'] = location
event['title'] = "SPCO: " + title
event['link'] = link
event_type = "Mu"
digitools.createBasicEvent(event, event_type, venue)
scraper.items+=1
except Exception as e:
print("ERROR: ", e)
print("\n\n+++\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
# Get Event Page Link(s)
links = digitools.createURLNoZero("https://content.thespco.org/events/calendar/")
for link in links:
ps = digitools.getSource(br, link)
get_events(ps, "Mu")
sleep(3)
br.close()
digitools.updateScraper(scraper, item_count_start)

73
Working/venues/WhiteSquirrelScrape.py Normal file
View File

@@ -0,0 +1,73 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%Y-%m-%d %I:%M %p'
venue, created = Organization.objects.get_or_create(
name="White Squirrel",
city="St. Paul",
website="https://whitesquirrelbar.com",
is_venue = True
)
scraper,item_count_start, virtcal = digitools.getScraper(venue, 'msp')
# Set initial variables for City, etc
calendar_url = [
'https://whitesquirrelbar.com/calendar/list/page/1/',
'https://whitesquirrelbar.com/calendar/list/page/2/',
'https://whitesquirrelbar.com/calendar/list/page/3/'
]
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
def get_listings(pse, events):
listings = pse.xpath('.//*/li[@class="tribe-common-g-row tribe-events-calendar-list__event-row"]')
print("get listings", listings)
for l in listings:
event = {}
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
print("1: ", event)
try:
event["image"] = l.xpath('.//*/img/@src')[0]
except:
event["image"] = "none"
event["date"] = l.xpath('.//time/@datetime')[0]
event["time"] = l.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].split("@")[1]
event["title"] = l.xpath('.//*/h3/a/text()')[0].replace("\t", "").replace("\n", "")
event["link"] = l.xpath('.//*/h3/a/@href')[0]
event['datetime'] = event['date'] + " " + event['time']
event["dateStamp"] = datetime.strptime(event['datetime'] , DATETIME_FORMAT)
events.append(event)
digitools.createBasicEvent(event, "Mu", venue)
scraper.items+=1
events = []
for cal in calendar_url:
ps = digitools.getSource(br, cal)
get_listings(ps, events)
br.close()
digitools.updateScraper(scraper, item_count_start)

74
Working/venues/cedar.mpls.py Normal file
View File

@@ -0,0 +1,74 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="Cedar Cultural Center",
city="Minneapolis",
website="https://www.thecedar.org",
is_venue=True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
DATETIME_FORMAT_3 = '%A, %B %d at %I:%M%p %Y'
DATETIME_FORMAT_4 = '%A, %B %d at %I%p %Y'
DATETIME_FORMAT_5 = '%A, %B %d @%I%p %Y'
def get_events(ps):
links = ps.xpath('.//*/div[@class="summary-title"]/a/@href')
for l in links:
if "cedar-news-blog" in l:
continue
pse = digitools.getSource(br, "https://www.thecedar.org" + l)
event = {}
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
event['link'] = "https://www.thecedar.org" + l
try:
time = pse.xpath('.//*/time[@class="event-time-localized-start"]/text()')[0]
date = pse.xpath('.//*/time[@class="event-date"]/text()')[0]
event['title'] = pse.xpath('.//*/h1[@class="eventitem-title"]/text()')[0]
except:
try:
time = pse.xpath('.//*/time[@class="event-time-localized"]/text()')[0]
date = pse.xpath('.//*/time[@class="event-date"]/text()')[0]
event['title'] = pse.xpath('.//*/h1[@class="eventitem-title"]/text()')[0]
except Exception as e:
print(e)
print("failed event: ", event)
dateStamp = date + " " + time
event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
digitools.createBasicEvent(event, "Mu", venue)
scraper.items+=1
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
calendar_url = 'https://www.thecedar.org/listing'
ps = digitools.getSource(br, calendar_url)
get_events(ps)
br.close()
digitools.updateScraper(scraper, item_count_start)

105
Working/venues/club331Scrape.py Normal file
View File

@@ -0,0 +1,105 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from selenium.webdriver.common.by import By
from lxml import html
from events.models import Organization, Scraper, Event
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="Club 331",
city="Minneapolis",
website="https://331club.com",
is_venue=True,
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp')
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b %d %I%p %Y'
DATETIME_FORMAT_2 = '%b %d %I:%M%p %Y'
DATETIME_FORMAT_3 = '%b %d %Y'
# Set initial variables for City, etc
calendar_url = 'https://331club.com/#calendar'
current_year = str(datetime.now().year)
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
br.get(calendar_url)
sleep(3)
br.find_element(By.CLASS_NAME, 'more_events').click()
sleep(2)
ps = html.fromstring(br.page_source)
sleep(3)
dates = ps.xpath('.//*/div[@class="event"]')
dates = dates + ps.xpath('.//*/div[@class="event hidden"]')
def process_times(times):
# print("Times: ", times)
time = []
for t in times:
t = t.replace("\n", "").replace("TBA", "")
if len(t) > 0 and t.endswith("pm"):
if "-" in t:
t = t.split("-")[0] + "pm"
time.append(t)
return time
events = []
for d in dates:
event_date = d.xpath('.//div[@class="event-date"]/span/text()')[:2]
cols = d.xpath('.//div[@class="column"]')
for c in cols:
bands = c.xpath('.//p/a/text()')
links = c.xpath('.//p/a/@href')
time = process_times(c.xpath('.//p/text()'))
event = {}
event["datetime"] = event_date + time + [current_year]
try:
event["date_time"] = datetime.strptime(" ".join(event["datetime"]), DATETIME_FORMAT)
except:
try:
event["date_time"] = datetime.strptime(" ".join(event["datetime"]), DATETIME_FORMAT_2)
except:
try:
event["date_time"] = datetime.strptime(" ".join(event["datetime"]), DATETIME_FORMAT_3)
except:
event["date_time"] = "Invalid"
event["bands"] = (", ").join(bands)
if len(bands) > 0:
event['scraper'] = scraper
event['calendars'] = [scraper.calendar]
event['title'] = event["bands"]
event['date'] = event["date_time"]
event['dateStamp'] = event["date_time"]
event['link'] = "https://331club.com/#calendar"
try:
digitools.createBasicEvent(event, "Mu", venue)
except Exception as e:
print('oops', e)
events.append(event)
br.close()
digitools.updateScraper(scraper, item_count_start)

1295
Working/venues/manual/data.json Normal file
View File

File diff suppressed because it is too large Load Diff

1471
Working/venues/manual/data.updated.json Normal file
View File

File diff suppressed because it is too large Load Diff

169
Working/venues/manual/eventario.py Normal file
View File

@@ -0,0 +1,169 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import json
from selenium.webdriver.common.by import By
from lxml import html
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper, Calendar
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="Events Medellin",
city="Medellin",
website="https://eventario.co/events-category/social/",
is_venue=True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'mde')
DATETIME_FORMAT = '%B %d %Y %I:%M%p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
# with open('data.json') as f:
# totalLinks = json.load(f)
def getLinks(br, url, links):
br.get(url)
sleep(2)
br.execute_script("window.scrollTo(0, window.scrollY + 1500)")
sleep(2)
x = 1
while x == 1:
try:
div = br.find_element(By.XPATH, ".//*/div[@class='feed-pagination flexify']/a")
div.click()
sleep(2)
br.execute_script("window.scrollTo(0, window.scrollY + 1100)")
sleep(2)
except:
x = 0
ps = html.fromstring(br.page_source)
newLinks = []
# newlinks = ps.xpath('.//*/div[@class="e-con-inner"]/*/a/@href')
events = ps.xpath('.//*/div[@class="e-con-inner"]')
for event in events:
e = {}
try:
e['link'] = event.xpath('.//*/a/@href')[0]
e['title'] = event.xpath('.//*/h3/a/text()')[0]
e['venue'] = event.xpath('.//*/ul/li/a/text()')[-1:][0].replace('\n', '').replace('\t', '')
# e['venue'] = event.xpath('.//*/ul/li/a/text()')[-1:][0]
e['venueLink'] = event.xpath('.//*/ul/li/a/@href')[1]
label= event.xpath('.//*/li[@class="elementor-repeater-item-46edd7d flexify ts-action"]/div/text()')
e['label'] = ''.join([x.replace('\t', '').replace('\n', '') for x in label]).strip()
newLinks.append(e)
except Exception as e:
print("Error: ", e)
links = links + newLinks
return links
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
urls = ["https://eventario.co/events-category/social/", "https://eventario.co/events-category/musica/", "https://eventario.co/events-category/cultura/"]
allLinks = []
for url in urls:
allLinks = getLinks(br, url, allLinks)
totalLinks = list({v['title']:v for v in allLinks}.values())
ppr(len(totalLinks))
# sortedlinks = allLinks.sort()
# ppr(sortedlinks)
for event in totalLinks:
br.get(event['link'])
sleep(2)
ps = html.fromstring(br.page_source)
dateTime= ps.xpath('.//*/div[@class="elementor-element elementor-element-d9beb21 elementor-widget elementor-widget-heading"]/span[@class="elementor-heading-title elementor-size-default"]/text()')
event['dateTime'] = [x[3:].split('-')[0].strip() for x in dateTime]
locations = ps.xpath('.//*/div[@class="elementor-element elementor-element-f04aae3 elementor-widget__width-initial elementor-widget-mobile__width-initial elementor-widget elementor-widget-ts-advanced-list"]/*/li[@class="elementor-repeater-item-138dbed flexify ts-action"]/a/text()')
location = [x.replace('\t', '').replace('\n', '') for x in locations]
if len(location) == 2:
event['location'] = "{0}, {1}".format(location[1], location[0])
else:
try:
event['location'] = location[0]
except:
event['location'] = 'n/a'
address= ps.xpath('.//*/ul[@class="flexify simplify-ul ts-advanced-list"]/li[@class="elementor-repeater-item-842568d flexify ts-action"]/div/text()')
try:
event['address'] = [x for x in address if 'Capacidad' not in x and '$' not in x][0]
except:
event['address'] = address
# ppr(event)
sleep(2)
br.close()
data = totalLinks
print("Set:", len(totalLinks))
paisa = []
for d in data:
if len(d['dateTime']) != 0:
if 'Familia' in d['label']:
d['category'] = 'Ot'
elif 'Comedia' in d['label']:
d['category'] = 'Co'
elif ('Magic' in d['title']) or ('Juegos' in d['label']):
d['category'] = 'Ot'
elif ('Conferencias' in d['label']) or ('Intercambio' in d['label']):
d['category'] = 'Ed'
else:
d['category'] = 'Mu'
if "Antioquia" in d['location']:
try:
d['city'] = d['location'].split(',')[0]
paisa.append(d)
except:
continue
cal = Calendar.objects.get(shortcode='mde')
for d in paisa:
d['dateStamp'] =datetime.strptime(d['dateTime'][0], DATETIME_FORMAT)
try:
nvenue, created = Organization.objects.get_or_create(
name=d['venue'],
city=d['city'],
website=d['venueLink'],
address_complete = d['address'],
is_venue=True
)
except:
nvenue = Organization.objects.get(name=d['venue'])
nvenue.address_complete = d['address']
nvenue.save()
new_event, created = Event.objects.update_or_create(
event_type = d['category'],
show_title = d['title'],
show_link = d['link'],
show_date = d['dateStamp'],
show_day = d['dateStamp'],
scraper = scraper,
venue = nvenue
)
new_event.calendar.add(cal)
new_event.save()
print(new_event)
digitools.updateScraper(scraper, item_count_start)

64
Working/workshop/ComedyUnderground.py Normal file
View File

@@ -0,0 +1,64 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
from events.models import Event as DSEvent, Organization
from digitools import getBrowser, createDashURL, createBasicEvent, getSource
venue, created = Organization.objects.get_or_create(
name="Comedy Corner",
city="Minneapolis",
website="https://comedycornerunderground.com/calendar",
)
calendar_url = "https://comedycornerunderground.com/calendar"
DATETIME_FORMAT = '%m %d %I%p %Y'
def get_events(ps, link):
contents = ps.xpath('.//*/td')
for c in contents:
try:
day_num = c.xpath('.//*/div[@class="marker-daynum"]/text()')[0]
events = c.xpath('.//*/li')
# print(events)
for e in events:
event = {}
print(link)
month = link.split("month=")[1].split("-")[0]
year = link.split("month=")[1].split("-")[1]
event['title'] = e.xpath('.//*/span[@class="item-title"]/text()')[0]
event['time'] = e.xpath('.//*/span[@class="item-time item-time--12hr"]/text()')[0].replace("\xa0", "")
event['link'] = "https://comedycornerunderground.com" + e.xpath('.//a/@href')[0]
event['date'] = str(day_num) + ' ' + event['time'] + 'm'
dateStamp = month + ' ' + event['date'] + ' ' + year
event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
createBasicEvent(event, 'Co')
print('\n\n++\n\n')
except Exception as e:
continue
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
links = createDashURL("https://comedycornerunderground.com/calendar?view=calendar&month=")
for link in links:
ps = getSource(br, link)
get_events(ps, link)
sleep(5)
# ppr(events)
br.close()

74
Working/workshop/cabooze.mn.py Normal file
View File

@@ -0,0 +1,74 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="Cabooze",
city="Minneapolis",
website="https://www.cabooze.com/#/events",
is_venue=True
)
scraper,item_count_start = digitools.getScraper(venue)
event_type = ""
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b %d %I:%M %p %Y'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
def get_events(ps, event_type):
print("Getting events ...")
contents = ps.xpath('.//*/div[@class="vp-event-row vp-widget-reset vp-venue-thecabooze"]')
ppr(contents)
for c in contents:
try:
event = {}
date = c.xpath('.//*/span[@class="vp-month-n-day"]/text()')[0]
print(date)
month = date.split(" ")[0]
time = c.xpath('.//*/span[@class="vp-time"]/text()')[0]
year = datetime.now().year
if month == "Jan":
year = int(year) + 1
event['calendar'] = scraper.calendar
event['title'] = c.xpath('.//*/div[@class="vp-event-name"]/text()')[0]
event['date'] = [date, time, str(year)]
event['date'] = " ".join(event['date'])
event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
event['link'] = "https://www.cabooze.com/" + c.xpath('.//a[@class="vp-event-link"]/@href')[0]
print("Event Dict Created")
ppr(event)
digitools.createBasicEvent(event, event_type, venue)
scraper.items+=1
except Exception as e:
print(e)
ppr(event)
print("\n\n+++\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
ps = digitools.getSource(br, venue.website)
get_events(ps, "Mu")
sleep(3)
digitools.updateScraper(scraper, item_count_start)
br.close()

82
Working/workshop/dakota.mpls.py Normal file
View File

@@ -0,0 +1,82 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
count = 0
venue, created = Organization.objects.get_or_create(
name="Dakota",
city="Minneapolis",
website="https://www.dakotacooks.com/events/",
is_venue=True
)
scraper,item_count_start = digitools.getScraper(venue)
event_type = ""
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a %b %d, %Y • %I:%M%p'
DATETIME_FORMAT_2 = '%a %b %d, %Y • %I%p'
def get_events(ps, event_type):
links = ps.xpath('.//*/div[@class="wicked-event-title tooltipstered"]/a/@href')
links = set(links)
for l in links:
pse = getSource(br, l)
event = {}
event['scraper'] = scraper
event['calendar'] = scraper.calendar
event['link'] = l
try:
event['time'] = pse.xpath('.//*/span[@class="text-uppercase"]/text()')[0].strip()
event['dateStamp'] = datetime.strptime(event['time'], DATETIME_FORMAT)
event['title'] = pse.xpath('.//*/div[@class="sidebar-group"]/h1/text()')[0]
# event['detail-headers'] = pse.xpath('.//*/ul[@class="eo-event-meta"]/li/strong/text()')
# event['details'] = pse.xpath('.//*/ul[@class="eo-event-meta"]/li/text()')
except:
try:
event['title'] = pse.xpath('.//*/div[@class="sidebar-group"]/h1/text()')[0]
event['dateStamp'] = datetime.strptime(event['time'], DATETIME_FORMAT_2)
event['time'] = pse.xpath('.//*/span[@class="text-uppercase"]/text()')[0]
except Exception as e:
print(e)
print("failed event: ", event)
ppr(event)
try:
digitools.createBasicEvent(event, "Mu", venue)
scraper.items+=1
except Exception as e:
print(e)
print('failed to create: ', event)
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
ps = digitools.getSource(br, venue.website + '?wicked_month=04&wicked_year=2025&wicked_view=month')
get_events(ps, "Mu")
sleep(1)
digitools.updateScraper(scraper, item_count_start)
br.close()

78
Working/workshop/icehouse.mpls.py Normal file
View File

@@ -0,0 +1,78 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
count = 0
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a, %b %d %Y %I:%M %p SHOW'
DATETIME_FORMAT_2 = '%a, %b %d %Y %I:%M %p SHOW'
venue, created = Organization.objects.get_or_create(
name="Icehouse",
city="Minneapolis",
website = "https://icehouse.turntabletickets.com",
is_venue = True
)
scraper,item_count_start = digitools.getScraper(venue)
def get_events(ps, event_type):
contents = ps.xpath('.//*/div[@class="performances whitespace-pre-line w-full md:w-3/4"]')
for c in contents:
try:
event = {}
event['scraper'] = scraper
event['calendar'] = scraper.calendar
event['title'] = c.xpath('.//*/h3[@class="text-3xl font-semibold font-heading mr-auto"]/text()')[0]
event['link'] = venue.website + c.xpath('.//*/a[@class="show-link"]/@href')[0]
event['date'] = c.xpath('.//*/h4[@class="day-of-week"]/text()')[0]
year = datetime.now().year
if "Brunch" in event['title']:
event['time'] = "11:00 AM SHOW"
else:
event['time'] = c.xpath('.//*/div[@class="performance-btn"]/button/text()')[0]
event['datetime'] = event['date'] + " " + str(year) + " " + event['time']
try:
event['dateStamp'] =datetime.strptime(event['datetime'], DATETIME_FORMAT)
except:
event['datetime'] = event['date'] + " " + str(year) + " " + "07:00 PM SHOW"
event['dateStamp'] =datetime.strptime(event['datetime'], DATETIME_FORMAT)
event['title'] = event['title'] + " (Time Estimated)"
try:
digitools.createBasicEvent(event, event_type, venue)
scraper.items+=1
except Exception as e:
print(e)
quit()
except Exception as e:
ppr(event)
print(e)
quit()
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
ps = digitools.getSource(br, venue.website)
get_events(ps, "Mu")
# ppr(events)
br.close()
digitools.updateScraper(scraper, item_count_start)

86
Working/workshop/pillarforum.mpls.py Normal file
View File

@@ -0,0 +1,86 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
current_year = str(datetime.now().year)
venue, created = Organization.objects.get_or_create(
name="Piller Forum",
city="Minneapolis",
website="https://www.pilllar.com/pages/events",
is_venue = True
)
scraper,item_count_start = digitools.getScraper(venue)
event_type = "Mu"
# Time Signatures
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%b. %d %Y %I:%M %p'
DATETIME_FORMAT_night = '%b. %d %Y %I:%M %p'
DATETIME_FORMAT_2 = '%b. %d %Y %I:%Mam'
def get_events(ps, event_type):
contents = ps.xpath('.//*/div[@class="sse-column sse-half sse-center"]')
for c in contents:
try:
event = {}
event['scraper'] = scraper
event['calendar'] = scraper.calendar
event['link'] = venue.website
# time = c.xpath('.//*/span[@class="vp-time"]/text()')[0].strip()
date = c.xpath('.//h1[@class="sse-size-64"]/text()')[0]
if len(date) > 1:
print(date)
year = datetime.now().year
event_date = date + " " + str(year)
event['title'] = c.xpath('.//p/span/b/text()')[0]
details = c.xpath('.//p/text()')
if 'Music' in details[-1]:
event_time = c.xpath('.//p/text()')[-1].split("Music")[1].strip()
event_type = "Mu"
event_dt = event_date + " " + event_time + " PM"
event['dateStamp'] = datetime.strptime(event_dt, DATETIME_FORMAT_night)
elif len(details) == 1:
try:
event_time = details[0].split("-")[0].strip()
event_dt = event_date + " " + event_time + ' PM'
event['dateStamp'] = datetime.strptime(event_dt, DATETIME_FORMAT_night)
event_type = "Ot"
except Exception as e:
event_time = details[0].split("-")[0].strip()
event_dt = event_date + " " + event_time
event['dateStamp'] = datetime.strptime(event_dt, DATETIME_FORMAT_2)
event_type = "Ot"
digitools.createBasicEvent(event, event_type, venue)
except Exception as e:
print(e)
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
ps = digitools.getSource(br, venue.website)
get_events(ps, event_type)
sleep(3)
br.close()
digitools.updateScraper(scraper, item_count_start)

BIN
__pycache__/digitools.cpython-313.pyc Normal file
View File

Binary file not shown.

8
digitools_smallset.py Normal file
View File

@@ -0,0 +1,8 @@
import os, sys
import django
def getReady():
sys.path.append('/var/www/digisnaxx.ado/api.digisnaxx')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
return

44
run_scrapers.sh Normal file
View File

@@ -0,0 +1,44 @@
#!/bin/bash
DIGIDIR=/var/www/digisnaxx.ado
DJANGODIR=/var/www/digisnaxx.ado/api.digisnaxx
EVENTSDIR=/var/www/digisnaxx.ado/api.digisnaxx/events
FIXTURESDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/fixtures
SCRAPEDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers
SCRIPTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/scripts
VENUESDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/venues
ICALDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/iCal
GOVTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/govt
export DJANGO_SUPERUSER_EMAIL=canin@dreamfreely.org
export DJANGO_SUPERUSER_USERNAME=canin
export DJANGO_SUPERUSER_PASSWORD='hello123'
cd $DIGIDIR
source venv/bin/activate
cd $DJANGODIR
mv *.bak baks/
cp db.sqlite3 $(date -d "today" +"%Y%m%d%H%M").sqlite.bak
# python manage.py migrate
# cd $FIXTURESDIR
# django-admin loaddata orgs.json
# django-admin loaddata promo.json
cd $SCRIPTDIR
python start_up.py
bash run_venues.sh $1
bash run_govt.sh $1
bash run_ical.sh
# # bash run_media_update.sh
# # bash run_news.sh $1
# python clean_up.py
# cd ../
# deactivate
# rm -rf ../*/__pycache__
# rm -rf ../*/*/__pycache__

33
scripts/clean_up.py Normal file
View File

@@ -0,0 +1,33 @@
import re, os, sys
from datetime import datetime, timedelta
from django.db.models import Count
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event, Organization
new_time = datetime.now() - timedelta(days=1)
right_bound_time = datetime.now() + timedelta(days=45)
events = Event.objects.filter(show_date__lte=new_time)
events1 = Event.objects.filter(show_date__gte=right_bound_time)
for e in events:
e.delete()
for e in events1:
e.delete()
org_sin_events = Organization.objects.annotate(num_events = Count('event')).filter(num_events__lt=1)
for o in org_sin_events:
print(o)
# for org in org_sin_events:
# print(org)
# org.delete()
print("completed and cleaned scrapes")

17
scripts/run_govt.sh Normal file
View File

@@ -0,0 +1,17 @@
#!/bin/bash
BASEDIR=/var/www/digisnaxx.ado/api.digisnaxx
EVENTSDIR=/var/www/digisnaxx.ado/api.digisnaxx/events
SCRIPTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/scripts
GOVTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/govt
cd $GOVTDIR
for file in *
do
python "$file" $1
echo "SCRIPT COMPLETE"
done
cd $SCRIPTDIR
pwd
python clean_up.py

16
scripts/run_ical.sh Normal file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
BASEDIR=/var/www/digisnaxx.ado/api.digisnaxx
DJANGODIR=/var/www/digisnaxx.ado/api.digisnaxx/events
SCRIPTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/scripts
ICALDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/iCal
cd $ICALDIR
for file in *
do
python "$file"
echo "SCRIPT COMPLETE"
done
cd $SCRIPTDIR
python clean_up.py

22
scripts/run_media_update.sh Normal file
View File

@@ -0,0 +1,22 @@
#!/bin/bash
ENVDIR=/var/www/digisnaxx.ado/
DJANGODIR=/var/www/digisnaxx.ado/api.digisnaxx/events
EVENTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers
WORKMEDIADIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/smedia
# ENVDIR=/home/canin/Downloads/DigiSnaxxEvents
# DJANGODIR=/home/canin/Documents/repos/digisnaxx/events
# WORKMEDIADIR=/home/canin/Documents/repos/digisnaxx/events/scrapers/Working/smedia
cd $ENVDIR
pwd
source venv/bin/activate
cd $WORKMEDIADIR
python bluesky.py
python bluesky_media.py
python redsky.py
deactivate

16
scripts/run_news.sh Normal file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
BASEDIR=/var/www/digisnaxx.com/api.digisnaxx
DJANGODIR=/var/www/digisnaxx.com/api.digisnaxx/events
SCRIPTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/scripts
NEWSDIR=/var/www/digisnaxx.com/api.digisnaxx/events/scrapers/Working/news
cd $NEWSDIR
for file in *
do
python "$file" $1
echo "SCRIPT COMPLETE"
done
cd $SCRIPTDIR
python clean_up.py

16
scripts/run_venues.sh Normal file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
BASEDIR=/var/www/digisnaxx.ado/api.digisnaxx
DJANGODIR=/var/www/digisnaxx.ado/api.digisnaxx/events
SCRIPTDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/scripts
VENUESDIR=/var/www/digisnaxx.ado/api.digisnaxx/events/scrapers/Working/venues
cd $VENUESDIR
for file in *
do
python "$file" $1
echo "SCRIPT COMPLETE"
done
cd $SCRIPTDIR
python clean_up.py

29
scripts/start_up.py Normal file
View File

@@ -0,0 +1,29 @@
import re, os, sys
from datetime import datetime, timedelta
import django
sys.path.append('../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event,Organization, Promo, Calendar
# venue, created = Organization.objects.get_or_create(name="DreamFreely",
# website="https://www.dreamfreely.org",
# city="Minneapolis",
# contact_name="Canin Carlos",
# contact_email="canin@dreamfreely.org",
# phone_number="6124054535")
# print("Created DreamFreely:", created, venue)
# calendar, created = Calendar.objects.get_or_create(name='Mpls-StP', shortcode='msp', desc='none')
# calendar, created = Calendar.objects.get_or_create(name='Medellin', shortcode='mde', desc='none')
# calendar, created = Calendar.objects.get_or_create(name='Online', shortcode='000', desc='none')
events = Event.objects.all()
for e in events:
e.delete()
print("Events are cleared. Ready to scrape.")

30
zArchive/FaceBook/BillPull.py Normal file
View File

@@ -0,0 +1,30 @@
for sE in senateEvents[:5]:
bills = sE.xpath('.//*/div[@class="mb-1"]/a/text()')
bill_link = sE.xpath('.//*/div[@class="mb-1"]/a/@href')
bill_items = zip(bills, bill_link)
print(bills)
for b,i in bill_items:
if b.startswith("S.F."):
print(b, i, "\n\n")
import os
from twilio.rest import Client
# Find your Account SID and Auth Token at twilio.com/console
# and set the environment variables. See http://twil.io/secure
account_sid = os.environ['ACb416a0b2ed0a1be44c107b8bc1f683c5']
auth_token = os.environ['33cae777f215a003deea6d4a0d5027c2']
client = Client(account_sid, auth_token)
message = client.messages \
.create(
body="Join Earth's mightiest heroes. Like Kevin Bacon.",
from_='+15017122661',
to='+15558675310'
)
print(message.sid)

63
zArchive/FaceBook/Mortimers.mpls.py Normal file
View File

@@ -0,0 +1,63 @@
import re, os, sys
from datetime import datetime
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from events.models import Event, Organization
from pprint import pprint as ppr
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from selenium.webdriver.common.by import By
from xvfbwrapper import Xvfb
from lxml import html
import pytz
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a, %b %d %Y'
calendar_url = "https://www.facebook.com/mortimersmpls/events/"
current_year = str(datetime.now().year)
# Initiate and start the Browser
br = wd.Firefox()
br.get(calendar_url)
sleep(10)
br.find_element(By.XPATH, '//*/div[@class="x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x1ypdohk xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1o1ewxj x3x9cwd x1e5q0jg x13rtm0m x87ps6o x1lku1pv x1a2a7pz x9f619 x3nfvp2 xdt5ytf xl56j7k x1n2onr6 xh8yej3"]').click()
print("Input Login Info")
sleep(30)
ps = html.fromstring(br.page_source)
listings = ps.xpath('.//*/div[@class="x9f619 x1n2onr6 x1ja2u2z x78zum5 x2lah0s x1qughib x6s0dn4 xozqiw3 x1q0g3np x1pi30zi x1swvt13 xsag5q8 xz9dl7a x1n0m28w xp7jhwk x1wsgfga x9otpla"]')
for l in listings:
gT = l.xpath('.//*/span/text()')
dateTime = gT[0]
show_title = gT[1]
link = l.xpath('.//*/a/@href')[0].split("?")[0] + " " + current_year
print(show_title, dateTime, link)
venue, created = Organization.objects.get_or_create(name="Mortimer's")
try:
new_event = Event.objects.update_or_create(
event_type = 'Mu',
show_title = show_title,
show_link = link,
show_date = datetime.strptime(dateTime.split(" AT")[0].strip(), DATETIME_FORMAT),
venue = venue
)
except Exception as e:
print(e, "\n\n++++\n\n")
continue
br.close()

69
zArchive/FaceBook/Mortimers.py Normal file
View File

@@ -0,0 +1,69 @@
import re, os, sys
from datetime import datetime
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from events.models import Event, Organization
from pprint import pprint as ppr
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from xvfbwrapper import Xvfb
import requests
from lxml import html
import pytz
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a, %b %d %Y'
# Set initial variables for City, etc
calendar_url = "https://www.facebook.com/mortimersmpls/events/"
current_year = str(datetime.now().year)
# Initiate and start the Browser
br = wd.Firefox()
br.get(calendar_url)
sleep(10)
br.find_element(By.XPATH, '//*/div[@class="x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x1ypdohk xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1o1ewxj x3x9cwd x1e5q0jg x13rtm0m x87ps6o x1lku1pv x1a2a7pz x9f619 x3nfvp2 xdt5ytf xl56j7k x1n2onr6 xh8yej3"]').click()
print("Input Login Info")
sleep(30)
ps = html.fromstring(br.page_source)
listings = ps.xpath('.//*/div[@class="x9f619 x1n2onr6 x1ja2u2z x78zum5 x2lah0s x1qughib x6s0dn4 xozqiw3 x1q0g3np x1pi30zi x1swvt13 xsag5q8 xz9dl7a x1n0m28w xp7jhwk x1wsgfga x9otpla"]')
for l in listings:
gT = l.xpath('.//*/span/text()')
dateTime = gT[0]
show_title = gT[1]
link = l.xpath('.//*/a/@href')[0].split("?")[0] + " " + current_year
print(show_title, dateTime, link)
venue, created = Organization.objects.get_or_create(name="Mortimer's")
try:
new_event = Event.objects.update_or_create(
event_type = 'Mu',
show_title = show_title,
show_link = link,
show_date = datetime.strptime(dateTime.split(" AT")[0].strip(), DATETIME_FORMAT),
venue = venue
)
except Exception as e:
print(e, "\n\n++++\n\n")
continue
br.close()

85
zArchive/FaceBook/pillarforum.mpls.py Normal file
View File

@@ -0,0 +1,85 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from selenium.webdriver.common.by import By
from xvfbwrapper import Xvfb
from lxml import html
import pytz
from events.models import Event as DSEvent, Organization
from digitools import getBrowser, createURL, createBasicEvent, getSource
exit()
current_year = str(datetime.now().year)
venue, created = Organization.objects.get_or_create(
name="Piller Forum",
city="Minneapolis",
website="https://www.pilllar.com/pages/events",
)
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%B %d %I:%M%p %Y'
DATETIME_FORMAT = '%B %A %d %I:%M-%I:%M%p'
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
br.get(venue.website)
else:
print("No run_env")
quit()
try:
br.find_element(By.XPATH, '//*[@class="privy-dismiss-content"]').click()
except Exception as e:
print(e)
pass
months = br.find_elements(By.XPATH, '//*[@class="sse-display"]')
for month in months:
month_name = month.find_element(By.XPATH, './/*[@class="sse-size-28"]/u').text.capitalize()
events = month.find_elements(By.XPATH, './/p')
for event in events:
e = {}
eventTitle = event.text
try:
e['title'] = " ".join(eventTitle.split("-")[1].split(" ")[1:-2])
if 'Music' in eventTitle:
e['event_type'] = "Mu"
elif 'The Growth Arc' in eventTitle:
e['event_type'] = "Ot"
e['dateTime'] = " ".join([month_name, date, "7:00pm", current_year])
e['dateStamp'] = datetime.strptime(e['dateTime'], DATETIME_FORMAT)
e['title'] = "The Growth Arc - Relationship Support Space"
e['link'] = venue.website
elif 'Event' in eventTitle:
e['event_type'] = "Mu"
else:
e['event_type'] = "Ot"
date = eventTitle.split(":")[0].split(" ")[1].replace("th", "").replace("nd", "").replace("rd", "").replace("st", "")
time = eventTitle.split("-")[1].split(" ")[-2:][0]
e['dateTime'] = " ".join([month_name, date, time, current_year])
e['dateStamp'] = datetime.strptime(e['dateTime'], DATETIME_FORMAT)
e['link'] = venue.website
createBasicEvent(e, venue)
except Exception as e:
print("error ", eventTitle)
print(e)
sleep(3)
# ppr(events)
br.close()

61
zArchive/bluesky_scrape_old.py Normal file
View File

@@ -0,0 +1,61 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
from atproto import Client
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from socials.models import SocialLink
# from digitools import getBrowser, createURL, createBasicEvent, getSource
tz = pytz.timezone("US/Central")
USERNAME = "dreamfreely.org"
PASSWORD = "Futbol21!@"
client = Client()
client.login(USERNAME, PASSWORD)
feed = client.get_author_feed(USERNAME, limit = 100)
def createSocialLink(post):
new_post, created = SocialLink.objects.update_or_create(
cid = post['link_id'],
uri = post['uri'],
text = post['text'],
link = post['link'],
handle = post['handle'],
likes = post['likes'],
reposts = post['reposts'],
quotes = post['quotes'],
replies = post['replies'],
created_at = post['created_at']
)
print(created, new_post)
for post in feed.feed:
post = post.post
if hasattr(post.record.embed, 'external'):
p = {}
p['link'] = post.record.embed.external.uri.split("?")[0]
p['text'] = " ".join(post.record.text.split("\n")[:2])
p['handle'] = post.author.handle
p['link_id'] = post.uri.split("feed.post/")[-1]
p['uri'] = post.uri
p['likes'] = post.like_count
p['quotes'] = post.quote_count
p['replies'] = post.reply_count
p['reposts'] = post.repost_count
p['created_at'] = post.record.created_at
try:
createSocialLink(p)
except Exception as e:
print(e)

82
zArchive/broken/BirchbarkBooks.py Normal file
View File

@@ -0,0 +1,82 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
from events.models import Event as DSEvent, Organization
from digitools import getBrowser, createURL, createBasicEvent, getSource
current_year = str(datetime.now().year)
venue, created = Organization.objects.get_or_create(
name="Birchbark Books",
city="Minneapolis",
website="https://birchbarkbooks.com/pages/events",
)
DATETIME_FORMAT = '%A, %B %d @ %I:%M%p %Y'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
DATETIME_FORMAT_3 = '%A, %B %d at %I:%M%p %Y'
DATETIME_FORMAT_4 = '%A, %B %d at %I%p %Y'
DATETIME_FORMAT_5 = '%A, %B %d @%I%p %Y'
def get_events(ps):
contents = ps.xpath('.//*/div[@class="feature-row"]')
# ppr("contents:", contents)
for c in contents:
try:
event = {}
event['title'] = c.xpath('.//*/p[@class="h3"]/text()')[0].strip()
event['link'] = "https://birchbarkbooks.com/pages/events"
event['date'] = c.xpath('.//*/p[@class="accent-subtitle"]/text()')[0].replace("Central", "") + " " + current_year
event['date_num'] = event['date'].split(" ")[2].replace("th", "").replace("st", "").replace("rd", "").replace("nd", "")
event['date'] = event['date'].split(" ")
event['date'][2] = event['date_num']
event['date'] = " ".join(event['date'])
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT)
createBasicEvent(event, "Ed", venue)
except Exception as e:
try:
print(e)
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_2)
createBasicEvent(event, "Ed", venue)
print("\n\n+++\n\n")
except Exception as e:
try:
print(e)
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_3)
createBasicEvent(event, "Ed", venue)
print("\n\n+++\n\n")
except Exception as e:
try:
print(e)
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_4)
createBasicEvent(event, "Ed", venue)
print("\n\n+++\n\n")
except Exception as e:
print(e)
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_5)
createBasicEvent(event, "Ed", venue)
print("\n\n+++\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
calendar_url = 'https://birchbarkbooks.com/pages/events'
ps = getSource(br, calendar_url)
get_events(ps)
# ppr(events)
br.close()

70
zArchive/broken/EastsideLibrary.py Normal file
View File

@@ -0,0 +1,70 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
current_year = str(datetime.now().year)
venue, created = Organization.objects.get_or_create(
name="Eastside Freedom Library",
city="Minneapolis",
website="https://eastsidefreedomlibrary.org/events/",
is_venue=True
)
scraper,item_count_start = digitools.getScraper(venue)
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%B %d @ %I:%M %p %Y'
def get_events(ps):
contents = ps.xpath('.//*/article')
# ppr("contents:", contents)
for c in contents:
try:
event = {}
event['scraper'] = scraper
event['calendar'] = scraper.calendar
event['title'] = c.xpath('.//*/h3[@class="tribe-events-calendar-list__event-title tribe-common-h6 tribe-common-h4--min-medium"]/a/text()')[0].strip()
event['link'] = c.xpath('.//*/h3[@class="tribe-events-calendar-list__event-title tribe-common-h6 tribe-common-h4--min-medium"]/a/@href')[0]
event['date'] = c.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].strip() + " " + current_year
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT)
try:
new_event = digitools.createBasicEvent(event, "Ed", venue)
scraper.items+=1
except Exception as e:
print(e)
ppr(event)
print("\n+++\n")
except Exception as e:
print(e)
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
calendar_url = 'https://eastsidefreedomlibrary.org/events/'
ps = digitools.getSource(br, calendar_url)
get_events(ps)
# ppr(events)
br.close()
digitools.updateScraper(scraper, item_count_start)

30
zArchive/broken/FaceBook/BillPull.py Normal file
View File

@@ -0,0 +1,30 @@
for sE in senateEvents[:5]:
bills = sE.xpath('.//*/div[@class="mb-1"]/a/text()')
bill_link = sE.xpath('.//*/div[@class="mb-1"]/a/@href')
bill_items = zip(bills, bill_link)
print(bills)
for b,i in bill_items:
if b.startswith("S.F."):
print(b, i, "\n\n")
import os
from twilio.rest import Client
# Find your Account SID and Auth Token at twilio.com/console
# and set the environment variables. See http://twil.io/secure
account_sid = os.environ['ACb416a0b2ed0a1be44c107b8bc1f683c5']
auth_token = os.environ['33cae777f215a003deea6d4a0d5027c2']
client = Client(account_sid, auth_token)
message = client.messages \
.create(
body="Join Earth's mightiest heroes. Like Kevin Bacon.",
from_='+15017122661',
to='+15558675310'
)
print(message.sid)

63
zArchive/broken/FaceBook/Mortimers.mpls.py Normal file
View File

@@ -0,0 +1,63 @@
import re, os, sys
from datetime import datetime
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from events.models import Event, Organization
from pprint import pprint as ppr
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from selenium.webdriver.common.by import By
from xvfbwrapper import Xvfb
from lxml import html
import pytz
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a, %b %d %Y'
calendar_url = "https://www.facebook.com/mortimersmpls/events/"
current_year = str(datetime.now().year)
# Initiate and start the Browser
br = wd.Firefox()
br.get(calendar_url)
sleep(10)
br.find_element(By.XPATH, '//*/div[@class="x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x1ypdohk xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1o1ewxj x3x9cwd x1e5q0jg x13rtm0m x87ps6o x1lku1pv x1a2a7pz x9f619 x3nfvp2 xdt5ytf xl56j7k x1n2onr6 xh8yej3"]').click()
print("Input Login Info")
sleep(30)
ps = html.fromstring(br.page_source)
listings = ps.xpath('.//*/div[@class="x9f619 x1n2onr6 x1ja2u2z x78zum5 x2lah0s x1qughib x6s0dn4 xozqiw3 x1q0g3np x1pi30zi x1swvt13 xsag5q8 xz9dl7a x1n0m28w xp7jhwk x1wsgfga x9otpla"]')
for l in listings:
gT = l.xpath('.//*/span/text()')
dateTime = gT[0]
show_title = gT[1]
link = l.xpath('.//*/a/@href')[0].split("?")[0] + " " + current_year
print(show_title, dateTime, link)
venue, created = Organization.objects.get_or_create(name="Mortimer's")
try:
new_event = Event.objects.update_or_create(
event_type = 'Mu',
show_title = show_title,
show_link = link,
show_date = datetime.strptime(dateTime.split(" AT")[0].strip(), DATETIME_FORMAT),
venue = venue
)
except Exception as e:
print(e, "\n\n++++\n\n")
continue
br.close()

69
zArchive/broken/FaceBook/Mortimers.py Normal file
View File

@@ -0,0 +1,69 @@
import re, os, sys
from datetime import datetime
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from events.models import Event, Organization
from pprint import pprint as ppr
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from xvfbwrapper import Xvfb
import requests
from lxml import html
import pytz
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a, %b %d %Y'
# Set initial variables for City, etc
calendar_url = "https://www.facebook.com/mortimersmpls/events/"
current_year = str(datetime.now().year)
# Initiate and start the Browser
br = wd.Firefox()
br.get(calendar_url)
sleep(10)
br.find_element(By.XPATH, '//*/div[@class="x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x1ypdohk xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1o1ewxj x3x9cwd x1e5q0jg x13rtm0m x87ps6o x1lku1pv x1a2a7pz x9f619 x3nfvp2 xdt5ytf xl56j7k x1n2onr6 xh8yej3"]').click()
print("Input Login Info")
sleep(30)
ps = html.fromstring(br.page_source)
listings = ps.xpath('.//*/div[@class="x9f619 x1n2onr6 x1ja2u2z x78zum5 x2lah0s x1qughib x6s0dn4 xozqiw3 x1q0g3np x1pi30zi x1swvt13 xsag5q8 xz9dl7a x1n0m28w xp7jhwk x1wsgfga x9otpla"]')
for l in listings:
gT = l.xpath('.//*/span/text()')
dateTime = gT[0]
show_title = gT[1]
link = l.xpath('.//*/a/@href')[0].split("?")[0] + " " + current_year
print(show_title, dateTime, link)
venue, created = Organization.objects.get_or_create(name="Mortimer's")
try:
new_event = Event.objects.update_or_create(
event_type = 'Mu',
show_title = show_title,
show_link = link,
show_date = datetime.strptime(dateTime.split(" AT")[0].strip(), DATETIME_FORMAT),
venue = venue
)
except Exception as e:
print(e, "\n\n++++\n\n")
continue
br.close()

72
zArchive/broken/acadia.py Normal file
View File

@@ -0,0 +1,72 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
from events.models import Event as DSEvent, Organization
from digitools import getBrowser, createDashURL, createBasicEvent, getSource
venue, created = Organization.objects.get_or_create(
name="Acadia Cafe",
city="Minneapolis",
website="https://acadiacafe.com",
)
calendar_url = "https://www.acadiacafe.com/events"
DATETIME_FORMAT = '%d %m %Y %I:%M%p'
def get_events(ps, link):
contents = ps.xpath('.//*/div[@class="cl-view-month__day__event__title"]')
print(contents)
quit()
for c in contents:
try:
day_num = c.xpath('.//*/div[@class="marker-daynum"]/text()')[0]
events = c.xpath('.//*/li')
# print(events)
for e in events:
event = {}
event['month'] = link.split("month=")[1].split("-")[0]
event['year'] = link.split("month=")[1].split("-")[1]
event['title'] = e.xpath('.//h1/a[@class="flyoutitem-link"]/text()')
event['time'] = e.xpath('.//div[@class="flyoutitem-datetime flyoutitem-datetime--12hr"]/text()')
event['link'] = e.xpath('.//a/@href')[0]
event['date'] = str(day_num) + ' ' + 'time'
# event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
if len(event['title']):
nevent = {}
nevent['title'] = "".join(event['title']).strip()
event['time'] = event['time'][0].strip().split(" –")[0]
nevent['link'] = "https://palmers-bar.com" + e.xpath('.//a/@href')[0]
event['dateStamp'] = str(day_num) + ' ' + event['month'] + ' ' + event['year'] + ' ' + event['time']
nevent['dateStamp'] = datetime.strptime(event['dateStamp'], DATETIME_FORMAT)
createBasicEvent(nevent, 'Mu', venue)
except Exception as e:
continue
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
ps = getSource(br, calendar_url)
sleep(5)
get_events(ps, calendar_url)
sleep(5)
# ppr(events)
br.close()

64
zArchive/broken/cedar.mpls.py Normal file
View File

@@ -0,0 +1,64 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from xvfbwrapper import Xvfb
from lxml import html
import pytz
from events.models import Event as DSEvent, Organization
from digitools import getBrowser, createBasicEvent, getSource
venue, created = Organization.objects.get_or_create(
name="Cedar Cultural Center",
city="Minneapolis",
website="https://www.thecedar.org/listing",
)
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
DATETIME_FORMAT_3 = '%A, %B %d at %I:%M%p %Y'
DATETIME_FORMAT_4 = '%A, %B %d at %I%p %Y'
DATETIME_FORMAT_5 = '%A, %B %d @%I%p %Y'
def get_events(ps):
links = ps.xpath('.//*/div[@class="summary-title"]/a/@href')
# ppr("contents:", contents)
for l in links:
br.get("https://www.thecedar.org" + l)
sleep(3)
pse = html.fromstring(br.page_source)
event = {}
time = pse.xpath('.//*/time[@class="event-time-12hr-start"]/text()')[0]
date = pse.xpath('.//*/time[@class="event-date"]/text()')[0]
event['title'] = pse.xpath('.//*/h1[@class="eventitem-title"]/text()')[0]
dateStamp = date + " " + time
event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
event['link'] = "https://www.thecedar.org" + l
createBasicEvent(event, "Mu", venue)
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
calendar_url = 'https://www.thecedar.org/listing'
ps = getSource(br, calendar_url)
get_events(ps)
# ppr(events)
br.close()

117
zArchive/broken/digitools.py Normal file
View File

@@ -0,0 +1,117 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
from time import sleep
import pytz
from lxml import html
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from xvfbwrapper import Xvfb
from selenium import webdriver as wd
from events.models import Event as DSEvent, Organization
tz = pytz.timezone("US/Central")
td = relativedelta.relativedelta(months=1)
odt = datetime.now() + td
def getSource(browser, link):
browser.get(link)
sleep(5)
ps = html.fromstring(browser.page_source)
return ps
def getBrowser(run_env):
if run_env == 'dev':
print("Chrome is a go!")
# chromeOptions = wd.ChromeOptions()
# chromeOptions.binary_location = "/Application/Google\ Chrome.app"
# chromeDriver = "/opt/homebrew/bin/chromedriver"
# br = wd.Chrome(chromeDriver, options=chromeOptions)
br = wd.Chrome()
return br
elif run_env == "def":
print("Firefox go vroom")
br = wd.Firefox()
return br
elif run_env == "prod":
start_cmd = "Xvfb :91 && export DISPLAY=:91 &"
xvfb = Xvfb()
os.system(start_cmd)
xvfb.start()
print("started Xvfb")
br = wd.Firefox()
return br
else:
print("Failed", sys.argv, arg1)
quit()
def createBasicURL(site_url):
month = datetime.now().month
next_month = odt.month
year = datetime.now().year
print(month, next_month, year)
links = [
site_url + str(month) + "/" + str(year),
site_url + str(next_month) + "/" + str(year)
]
print(links)
return links
def createURL(site_url):
month = datetime.now().month
if month < 10:
month = "0" + str(month)
else:
month = str(month)
next_month = odt.month
if next_month < 10:
next_month = "0" + str(next_month)
else:
next_month = str(next_month)
year = datetime.now().year
links = [
site_url + str(year) + "/" + month,
]
if next_month == "01":
links.append(site_url + str(int(year)+1) + "/" + next_month)
else:
links.append(site_url + str(year) + "/" + next_month)
print(links)
return links
def createDashURL(site_url):
month = datetime.now().month
if month < 10:
month = "0" + str(month)
else:
month = str(month)
next_month = odt.month
if next_month < 10:
next_month = "0" + str(next_month)
else:
next_month = str(next_month)
year = datetime.now().year
print(month, next_month, year)
links = [
site_url + month + "-" + str(year),
site_url + next_month + "-" + str(year)
]
print(links)
return links
def createBasicEvent(event, event_type, venue):
new_event, created = DSEvent.objects.update_or_create(
event_type = event_type,
show_title = event['title'],
show_link = event['link'],
show_date = event['dateStamp'],
show_day = event['dateStamp'],
venue = venue
)
print("New Event: ", new_event)

65
zArchive/broken/ical_run.KJHideaway.StPaul.py Normal file
View File

@@ -0,0 +1,65 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
venue, created = Organization.objects.get_or_create(
name="KJ's Hideaway",
city="Minneapolis",
website="",
)
try:
scraper, created = Scraper.objects.get_or_create(
name="KJ's Hideaway",
website="https://calendar.google.com/calendar/ical/sgmok5t13vspeoruhruh33dhj0hgc50q%40import.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(id=1),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
event_type = "Mu"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
for event in events:
# ppr(event)
now_now = datetime.today().date()
e = {}
e['calendars'] = event['calendars']
try:
e['dateStamp'] = event['dateStart'][0]
except:
e['dateStamp'] = event['dateStart']
e['title'] = event['strSummary']
e['scraper'] = scraper
e['link'] = venue.website
if e['dateStamp'] > now_now:
try:
digitools.createBasicEvent(e, 'Mu', venue)
except Exception as e:
print("Error: ", e)
digitools.updateScraper(scraper, item_count_start)

77
zArchive/broken/palmers.mpls.py Normal file
View File

@@ -0,0 +1,77 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
from events.models import Event as DSEvent, Organization
from digitools import getBrowser, createDashURL, createBasicEvent, getSource
venue, created = Organization.objects.get_or_create(
name="Palmer's Bar",
city="Minneapolis",
website="https://palmers-bar.com",
)
calendar_url = "https://palmers-bar.com"
DATETIME_FORMAT = '%d %m %Y %I:%M%p'
def get_events(ps, link):
contents = ps.xpath('.//*/td')
for c in contents:
try:
# day_num = c.xpath('.//*/div[@class="marker-daynum"]/text()')[0]
events = c.xpath('.//*/li')
# print(events)
for e in events:
event = {}
event_link = calendar_url + e.xpath('.//a/@href')[0]
ps = getSource(br, event_link)
new_event = ps.xpath('.//*/h1[@class="eventitem-column-meta"]')
event['title'] = new_event.xpath('.//*/h1[@class="event-title"]')
event['date'] = new_event.xpath('.//*/time[@class="event-date"]')
event['time'] = new_event.xpath('.//*/time[@class="event-time-12hr-start"]')
event['link'] = event_link
# event['month'] = link.split("month=")[1].split("-")[0]
# event['year'] = link.split("month=")[1].split("-")[1]
# event['title'] = e.xpath('.//h1/a[@class="flyoutitem-link"]/text()')
# event['time'] = e.xpath('.//div[@class="flyoutitem-datetime flyoutitem-datetime--12hr"]/text()')
# event['date'] = str(day_num) + ' ' + 'time'
# event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
ppr(event)
if len(event['title']):
nevent = {}
nevent['title'] = "".join(event['title']).strip()
event['time'] = event['time'][0].strip().split(" –")[0]
nevent['link'] = "https://palmers-bar.com" + e.xpath('.//a/@href')[0]
event['dateStamp'] = str(day_num) + ' ' + event['month'] + ' ' + event['year'] + ' ' + event['time']
nevent['dateStamp'] = datetime.strptime(event['dateStamp'], DATETIME_FORMAT)
ppr(nevent)
# createBasicEvent(nevent, 'Mu', venue)
except Exception as e:
continue
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
links = createDashURL("https://palmers-bar.com/?view=calendar&month=")
for link in links:
ps = getSource(br, link)
get_events(ps, link)
sleep(5)
# ppr(events)
br.close()

132
zArchive/digitools_old.py Normal file
View File

@@ -0,0 +1,132 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
from time import sleep
import pytz
from lxml import html
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from xvfbwrapper import Xvfb
from selenium import webdriver as wd
from events.models import Event as DSEvent, Organization
tz = pytz.timezone("US/Central")
td = relativedelta.relativedelta(months=1)
odt = datetime.now() + td
def getSource(browser, link):
browser.get(link)
sleep(3)
ps = html.fromstring(browser.page_source)
return ps
def getBrowser(run_env):
if run_env == 'dev':
print("Chrome is a go!")
# chromeOptions = wd.ChromeOptions()
# chromeOptions.binary_location = "/Application/Google\ Chrome.app"
# chromeDriver = "/opt/homebrew/bin/chromedriver"
# br = wd.Chrome(chromeDriver, options=chromeOptions)
br = wd.Chrome()
return br
elif run_env == "def":
print("Firefox go vroom")
br = wd.Firefox()
return br
elif run_env == "prod":
start_cmd = "Xvfb :91 && export DISPLAY=:91 &"
xvfb = Xvfb()
os.system(start_cmd)
xvfb.start()
print("started Xvfb")
br = wd.Firefox()
return br
else:
print("Failed", sys.argv, arg1)
quit()
def createBasicURL(site_url):
month = datetime.now().month
next_month = odt.month
year = datetime.now().year
print(month, next_month, year)
links = [
site_url + str(month) + "/" + str(year),
site_url + str(next_month) + "/" + str(year)
]
print(links)
return links
def createURLNoZero(site_url):
month = datetime.now().month
next_month = odt.month
year = datetime.now().year
links = [
site_url + str(year) + "/" + month,
]
if next_month == "1":
links.append(site_url + str(int(year)+1) + "/" + next_month)
else:
links.append(site_url + str(year) + "/" + next_month)
print(links)
return links
def createURL(site_url):
month = datetime.now().month
if month < 10:
month = "0" + str(month)
else:
month = str(month)
next_month = odt.month
if next_month < 10:
next_month = "0" + str(next_month)
else:
next_month = str(next_month)
year = datetime.now().year
links = [
site_url + str(year) + "/" + month,
]
if next_month == "01":
links.append(site_url + str(int(year)+1) + "/" + next_month)
else:
links.append(site_url + str(year) + "/" + next_month)
print(links)
return links
def createDashURL(site_url):
month = datetime.now().month
if month < 10:
month = "0" + str(month)
else:
month = str(month)
next_month = odt.month
if next_month < 10:
next_month = "0" + str(next_month)
else:
next_month = str(next_month)
year = datetime.now().year
print(month, next_month, year)
links = [
site_url + month + "-" + str(year),
site_url + next_month + "-" + str(year)
]
print(links)
return links
def createBasicEvent(event, event_type, venue):
new_event, created = DSEvent.objects.update_or_create(
calendar = event['calendar'],
event_type = event_type,
show_title = event['title'],
show_link = event['link'],
show_date = event['dateStamp'],
show_day = event['dateStamp'],
venue = venue
)
print("New Event: ", new_event)

91
zArchive/eventario.2.py Normal file
View File

@@ -0,0 +1,91 @@
import json
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper, Calendar, Event
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="Eventario",
city="Medellin",
website="https://www.eventario.co",
is_venue=True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'mde')
DATETIME_FORMAT = '%-d/%-m/%y %I:%M %p'
with open('data.json') as f:
data = json.load(f)
paisa = []
for d in data:
if len(d['dateTime']) != 0:
if 'Familia' in d['label']:
d['category'] = 'Ot'
elif 'Comedia' in d['label']:
d['category'] = 'Co'
elif ('Magic' in d['title']) or ('Juegos' in d['label']):
d['category'] = 'Ot'
elif ('Conferencias' in d['label']) or ('Intercambio' in d['label']):
d['category'] = 'Ed'
else:
d['category'] = 'Mu'
if "Antioquia" in d['location']:
try:
d['city'] = d['location'].split(',')[0]
paisa.append(d)
except:
continue
# ppr(paisa)
with open('data.updated.json', 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=4)
cal = Calendar.objects.get(shortcode='mde')
for d in paisa:
d['dateStamp'] =datetime.strptime(d['dateTime'][0], DATETIME_FORMAT)
try:
nvenue, created = Organization.objects.get_or_create(
name=d['venue'],
city=d['city'],
website=d['venueLink'],
address_complete = d['address'],
is_venue=True
)
except:
nvenue = Organization.objects.get(name=d['venue'])
nvenue.address_complete = d['address']
nvenue.save()
new_event, created = Event.objects.update_or_create(
event_type = d['category'],
show_title = d['title'],
show_link = d['link'],
show_date = d['dateStamp'],
show_day = d['dateStamp'],
scraper = scraper,
venue = nvenue
)
new_event.calendar.add(cal)
new_event.save()
print(new_event)
digitools.updateScraper(scraper, item_count_start)

50
zArchive/run_scrapers.sh Normal file
View File

@@ -0,0 +1,50 @@
#!/bin/bash
# BASEDIR=/var/www/digisnaxx.com/
# DJANGODIR=/var/www/digisnaxx.com/ds_events
# EVENTDIR=/var/www/digisnaxx.com/ds_events/event_scrapers
BASEDIR=/home/canin/Downloads/DigiSnaxxEvents
DJANGODIR=/home/canin/Downloads/DigiSnaxxEvents/ds_events
EVENTDIR=/home/canin/Downloads/DigiSnaxxEvents/ds_events/event_scrapers
VENUESDIR=/home/canin/Downloads/DigiSnaxxEvents/ds_events/event_scrapers/Working/venues
ICALDIR=/home/canin/Downloads/DigiSnaxxEvents/ds_events/event_scrapers/Working/iCal
GOVTDIR=/home/canin/Downloads/DigiSnaxxEvents/ds_events/event_scrapers/Working/govt
export DJANGO_SUPERUSER_EMAIL=canin@dreamfreely.org
export DJANGO_SUPERUSER_USERNAME=canin
export DJANGO_SUPERUSER_PASSWORD='hello123'
cd $BASEDIR
pwd
source venv/bin/activate
cd $DJANGODIR
pwd
mv db.sqlite3 db.sqlite3.bak
# rm ../db.sqlite3
# touch db.sqlite3
python manage.py migrate
# python manage.py createsuperuser --noinput
# python manage.py loaddata events/fixtures/organizations.json
# python manage.py loaddata events/fixtures/promo.json
# cd $EVENTDIR
# python start_up.py
bash run_venues.sh $1
bash run_ical.sh
bash run_govt.sh $1
bash run_news.sh $1
python Working/bluesky.py
python Working/redsky.py
cd $EVENTDIR
python clean_up.py
deactivate
bash run_media_update.sh
rm -rf ../*/__pycache__
rm -rf ../*/*/__pycache__