DigiSnaxx/scrapers
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first push

Browse Source
This commit is contained in:
canin carlos
2025-11-30 16:29:50 -05:00
commit d5055374b4
77 changed files with 9129 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
zArchive/broken/BirchbarkBooks.py Normal file
View File

@@ -0,0 +1,82 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
from events.models import Event as DSEvent, Organization
from digitools import getBrowser, createURL, createBasicEvent, getSource
current_year = str(datetime.now().year)
venue, created = Organization.objects.get_or_create(
name="Birchbark Books",
city="Minneapolis",
website="https://birchbarkbooks.com/pages/events",
)
DATETIME_FORMAT = '%A, %B %d @ %I:%M%p %Y'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
DATETIME_FORMAT_3 = '%A, %B %d at %I:%M%p %Y'
DATETIME_FORMAT_4 = '%A, %B %d at %I%p %Y'
DATETIME_FORMAT_5 = '%A, %B %d @%I%p %Y'
def get_events(ps):
contents = ps.xpath('.//*/div[@class="feature-row"]')
# ppr("contents:", contents)
for c in contents:
try:
event = {}
event['title'] = c.xpath('.//*/p[@class="h3"]/text()')[0].strip()
event['link'] = "https://birchbarkbooks.com/pages/events"
event['date'] = c.xpath('.//*/p[@class="accent-subtitle"]/text()')[0].replace("Central", "") + " " + current_year
event['date_num'] = event['date'].split(" ")[2].replace("th", "").replace("st", "").replace("rd", "").replace("nd", "")
event['date'] = event['date'].split(" ")
event['date'][2] = event['date_num']
event['date'] = " ".join(event['date'])
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT)
createBasicEvent(event, "Ed", venue)
except Exception as e:
try:
print(e)
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_2)
createBasicEvent(event, "Ed", venue)
print("\n\n+++\n\n")
except Exception as e:
try:
print(e)
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_3)
createBasicEvent(event, "Ed", venue)
print("\n\n+++\n\n")
except Exception as e:
try:
print(e)
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_4)
createBasicEvent(event, "Ed", venue)
print("\n\n+++\n\n")
except Exception as e:
print(e)
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT_5)
createBasicEvent(event, "Ed", venue)
print("\n\n+++\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
calendar_url = 'https://birchbarkbooks.com/pages/events'
ps = getSource(br, calendar_url)
get_events(ps)
# ppr(events)
br.close()

70
zArchive/broken/EastsideLibrary.py Normal file
View File

@@ -0,0 +1,70 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
current_year = str(datetime.now().year)
venue, created = Organization.objects.get_or_create(
name="Eastside Freedom Library",
city="Minneapolis",
website="https://eastsidefreedomlibrary.org/events/",
is_venue=True
)
scraper,item_count_start = digitools.getScraper(venue)
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%B %d @ %I:%M %p %Y'
def get_events(ps):
contents = ps.xpath('.//*/article')
# ppr("contents:", contents)
for c in contents:
try:
event = {}
event['scraper'] = scraper
event['calendar'] = scraper.calendar
event['title'] = c.xpath('.//*/h3[@class="tribe-events-calendar-list__event-title tribe-common-h6 tribe-common-h4--min-medium"]/a/text()')[0].strip()
event['link'] = c.xpath('.//*/h3[@class="tribe-events-calendar-list__event-title tribe-common-h6 tribe-common-h4--min-medium"]/a/@href')[0]
event['date'] = c.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].strip() + " " + current_year
event['dateStamp'] =datetime.strptime(event['date'], DATETIME_FORMAT)
try:
new_event = digitools.createBasicEvent(event, "Ed", venue)
scraper.items+=1
except Exception as e:
print(e)
ppr(event)
print("\n+++\n")
except Exception as e:
print(e)
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
calendar_url = 'https://eastsidefreedomlibrary.org/events/'
ps = digitools.getSource(br, calendar_url)
get_events(ps)
# ppr(events)
br.close()
digitools.updateScraper(scraper, item_count_start)

30
zArchive/broken/FaceBook/BillPull.py Normal file
View File

@@ -0,0 +1,30 @@
for sE in senateEvents[:5]:
bills = sE.xpath('.//*/div[@class="mb-1"]/a/text()')
bill_link = sE.xpath('.//*/div[@class="mb-1"]/a/@href')
bill_items = zip(bills, bill_link)
print(bills)
for b,i in bill_items:
if b.startswith("S.F."):
print(b, i, "\n\n")
import os
from twilio.rest import Client
# Find your Account SID and Auth Token at twilio.com/console
# and set the environment variables. See http://twil.io/secure
account_sid = os.environ['ACb416a0b2ed0a1be44c107b8bc1f683c5']
auth_token = os.environ['33cae777f215a003deea6d4a0d5027c2']
client = Client(account_sid, auth_token)
message = client.messages \
.create(
body="Join Earth's mightiest heroes. Like Kevin Bacon.",
from_='+15017122661',
to='+15558675310'
)
print(message.sid)

63
zArchive/broken/FaceBook/Mortimers.mpls.py Normal file
View File

@@ -0,0 +1,63 @@
import re, os, sys
from datetime import datetime
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from events.models import Event, Organization
from pprint import pprint as ppr
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from selenium.webdriver.common.by import By
from xvfbwrapper import Xvfb
from lxml import html
import pytz
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a, %b %d %Y'
calendar_url = "https://www.facebook.com/mortimersmpls/events/"
current_year = str(datetime.now().year)
# Initiate and start the Browser
br = wd.Firefox()
br.get(calendar_url)
sleep(10)
br.find_element(By.XPATH, '//*/div[@class="x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x1ypdohk xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1o1ewxj x3x9cwd x1e5q0jg x13rtm0m x87ps6o x1lku1pv x1a2a7pz x9f619 x3nfvp2 xdt5ytf xl56j7k x1n2onr6 xh8yej3"]').click()
print("Input Login Info")
sleep(30)
ps = html.fromstring(br.page_source)
listings = ps.xpath('.//*/div[@class="x9f619 x1n2onr6 x1ja2u2z x78zum5 x2lah0s x1qughib x6s0dn4 xozqiw3 x1q0g3np x1pi30zi x1swvt13 xsag5q8 xz9dl7a x1n0m28w xp7jhwk x1wsgfga x9otpla"]')
for l in listings:
gT = l.xpath('.//*/span/text()')
dateTime = gT[0]
show_title = gT[1]
link = l.xpath('.//*/a/@href')[0].split("?")[0] + " " + current_year
print(show_title, dateTime, link)
venue, created = Organization.objects.get_or_create(name="Mortimer's")
try:
new_event = Event.objects.update_or_create(
event_type = 'Mu',
show_title = show_title,
show_link = link,
show_date = datetime.strptime(dateTime.split(" AT")[0].strip(), DATETIME_FORMAT),
venue = venue
)
except Exception as e:
print(e, "\n\n++++\n\n")
continue
br.close()

69
zArchive/broken/FaceBook/Mortimers.py Normal file
View File

@@ -0,0 +1,69 @@
import re, os, sys
from datetime import datetime
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from events.models import Event, Organization
from pprint import pprint as ppr
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from xvfbwrapper import Xvfb
import requests
from lxml import html
import pytz
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%a, %b %d %Y'
# Set initial variables for City, etc
calendar_url = "https://www.facebook.com/mortimersmpls/events/"
current_year = str(datetime.now().year)
# Initiate and start the Browser
br = wd.Firefox()
br.get(calendar_url)
sleep(10)
br.find_element(By.XPATH, '//*/div[@class="x1i10hfl xjbqb8w x6umtig x1b1mbwd xaqea5y xav7gou x1ypdohk xe8uvvx xdj266r x11i5rnm xat24cr x1mh8g0r xexx8yu x4uap5 x18d9i69 xkhd6sd x16tdsg8 x1hl2dhg xggy1nq x1o1ewxj x3x9cwd x1e5q0jg x13rtm0m x87ps6o x1lku1pv x1a2a7pz x9f619 x3nfvp2 xdt5ytf xl56j7k x1n2onr6 xh8yej3"]').click()
print("Input Login Info")
sleep(30)
ps = html.fromstring(br.page_source)
listings = ps.xpath('.//*/div[@class="x9f619 x1n2onr6 x1ja2u2z x78zum5 x2lah0s x1qughib x6s0dn4 xozqiw3 x1q0g3np x1pi30zi x1swvt13 xsag5q8 xz9dl7a x1n0m28w xp7jhwk x1wsgfga x9otpla"]')
for l in listings:
gT = l.xpath('.//*/span/text()')
dateTime = gT[0]
show_title = gT[1]
link = l.xpath('.//*/a/@href')[0].split("?")[0] + " " + current_year
print(show_title, dateTime, link)
venue, created = Organization.objects.get_or_create(name="Mortimer's")
try:
new_event = Event.objects.update_or_create(
event_type = 'Mu',
show_title = show_title,
show_link = link,
show_date = datetime.strptime(dateTime.split(" AT")[0].strip(), DATETIME_FORMAT),
venue = venue
)
except Exception as e:
print(e, "\n\n++++\n\n")
continue
br.close()

72
zArchive/broken/acadia.py Normal file
View File

@@ -0,0 +1,72 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
from events.models import Event as DSEvent, Organization
from digitools import getBrowser, createDashURL, createBasicEvent, getSource
venue, created = Organization.objects.get_or_create(
name="Acadia Cafe",
city="Minneapolis",
website="https://acadiacafe.com",
)
calendar_url = "https://www.acadiacafe.com/events"
DATETIME_FORMAT = '%d %m %Y %I:%M%p'
def get_events(ps, link):
contents = ps.xpath('.//*/div[@class="cl-view-month__day__event__title"]')
print(contents)
quit()
for c in contents:
try:
day_num = c.xpath('.//*/div[@class="marker-daynum"]/text()')[0]
events = c.xpath('.//*/li')
# print(events)
for e in events:
event = {}
event['month'] = link.split("month=")[1].split("-")[0]
event['year'] = link.split("month=")[1].split("-")[1]
event['title'] = e.xpath('.//h1/a[@class="flyoutitem-link"]/text()')
event['time'] = e.xpath('.//div[@class="flyoutitem-datetime flyoutitem-datetime--12hr"]/text()')
event['link'] = e.xpath('.//a/@href')[0]
event['date'] = str(day_num) + ' ' + 'time'
# event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
if len(event['title']):
nevent = {}
nevent['title'] = "".join(event['title']).strip()
event['time'] = event['time'][0].strip().split(" –")[0]
nevent['link'] = "https://palmers-bar.com" + e.xpath('.//a/@href')[0]
event['dateStamp'] = str(day_num) + ' ' + event['month'] + ' ' + event['year'] + ' ' + event['time']
nevent['dateStamp'] = datetime.strptime(event['dateStamp'], DATETIME_FORMAT)
createBasicEvent(nevent, 'Mu', venue)
except Exception as e:
continue
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
ps = getSource(br, calendar_url)
sleep(5)
get_events(ps, calendar_url)
sleep(5)
# ppr(events)
br.close()

64
zArchive/broken/cedar.mpls.py Normal file
View File

@@ -0,0 +1,64 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
from selenium import webdriver as wd
from xvfbwrapper import Xvfb
from lxml import html
import pytz
from events.models import Event as DSEvent, Organization
from digitools import getBrowser, createBasicEvent, getSource
venue, created = Organization.objects.get_or_create(
name="Cedar Cultural Center",
city="Minneapolis",
website="https://www.thecedar.org/listing",
)
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
DATETIME_FORMAT_3 = '%A, %B %d at %I:%M%p %Y'
DATETIME_FORMAT_4 = '%A, %B %d at %I%p %Y'
DATETIME_FORMAT_5 = '%A, %B %d @%I%p %Y'
def get_events(ps):
links = ps.xpath('.//*/div[@class="summary-title"]/a/@href')
# ppr("contents:", contents)
for l in links:
br.get("https://www.thecedar.org" + l)
sleep(3)
pse = html.fromstring(br.page_source)
event = {}
time = pse.xpath('.//*/time[@class="event-time-12hr-start"]/text()')[0]
date = pse.xpath('.//*/time[@class="event-date"]/text()')[0]
event['title'] = pse.xpath('.//*/h1[@class="eventitem-title"]/text()')[0]
dateStamp = date + " " + time
event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
event['link'] = "https://www.thecedar.org" + l
createBasicEvent(event, "Mu", venue)
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
calendar_url = 'https://www.thecedar.org/listing'
ps = getSource(br, calendar_url)
get_events(ps)
# ppr(events)
br.close()

117
zArchive/broken/digitools.py Normal file
View File

@@ -0,0 +1,117 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
from time import sleep
import pytz
from lxml import html
import django
sys.path.append('../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from xvfbwrapper import Xvfb
from selenium import webdriver as wd
from events.models import Event as DSEvent, Organization
tz = pytz.timezone("US/Central")
td = relativedelta.relativedelta(months=1)
odt = datetime.now() + td
def getSource(browser, link):
browser.get(link)
sleep(5)
ps = html.fromstring(browser.page_source)
return ps
def getBrowser(run_env):
if run_env == 'dev':
print("Chrome is a go!")
# chromeOptions = wd.ChromeOptions()
# chromeOptions.binary_location = "/Application/Google\ Chrome.app"
# chromeDriver = "/opt/homebrew/bin/chromedriver"
# br = wd.Chrome(chromeDriver, options=chromeOptions)
br = wd.Chrome()
return br
elif run_env == "def":
print("Firefox go vroom")
br = wd.Firefox()
return br
elif run_env == "prod":
start_cmd = "Xvfb :91 && export DISPLAY=:91 &"
xvfb = Xvfb()
os.system(start_cmd)
xvfb.start()
print("started Xvfb")
br = wd.Firefox()
return br
else:
print("Failed", sys.argv, arg1)
quit()
def createBasicURL(site_url):
month = datetime.now().month
next_month = odt.month
year = datetime.now().year
print(month, next_month, year)
links = [
site_url + str(month) + "/" + str(year),
site_url + str(next_month) + "/" + str(year)
]
print(links)
return links
def createURL(site_url):
month = datetime.now().month
if month < 10:
month = "0" + str(month)
else:
month = str(month)
next_month = odt.month
if next_month < 10:
next_month = "0" + str(next_month)
else:
next_month = str(next_month)
year = datetime.now().year
links = [
site_url + str(year) + "/" + month,
]
if next_month == "01":
links.append(site_url + str(int(year)+1) + "/" + next_month)
else:
links.append(site_url + str(year) + "/" + next_month)
print(links)
return links
def createDashURL(site_url):
month = datetime.now().month
if month < 10:
month = "0" + str(month)
else:
month = str(month)
next_month = odt.month
if next_month < 10:
next_month = "0" + str(next_month)
else:
next_month = str(next_month)
year = datetime.now().year
print(month, next_month, year)
links = [
site_url + month + "-" + str(year),
site_url + next_month + "-" + str(year)
]
print(links)
return links
def createBasicEvent(event, event_type, venue):
new_event, created = DSEvent.objects.update_or_create(
event_type = event_type,
show_title = event['title'],
show_link = event['link'],
show_date = event['dateStamp'],
show_day = event['dateStamp'],
venue = venue
)
print("New Event: ", new_event)

65
zArchive/broken/ical_run.KJHideaway.StPaul.py Normal file
View File

@@ -0,0 +1,65 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
venue, created = Organization.objects.get_or_create(
name="KJ's Hideaway",
city="Minneapolis",
website="",
)
try:
scraper, created = Scraper.objects.get_or_create(
name="KJ's Hideaway",
website="https://calendar.google.com/calendar/ical/sgmok5t13vspeoruhruh33dhj0hgc50q%40import.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(id=1),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
event_type = "Mu"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
events = digitools.getiCalEvents(gcal, scraper, venue, "Ed")
for event in events:
# ppr(event)
now_now = datetime.today().date()
e = {}
e['calendars'] = event['calendars']
try:
e['dateStamp'] = event['dateStart'][0]
except:
e['dateStamp'] = event['dateStart']
e['title'] = event['strSummary']
e['scraper'] = scraper
e['link'] = venue.website
if e['dateStamp'] > now_now:
try:
digitools.createBasicEvent(e, 'Mu', venue)
except Exception as e:
print("Error: ", e)
digitools.updateScraper(scraper, item_count_start)

77
zArchive/broken/palmers.mpls.py Normal file
View File

@@ -0,0 +1,77 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
import django
sys.path.append('../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from time import sleep
from pprint import pprint as ppr
from events.models import Event as DSEvent, Organization
from digitools import getBrowser, createDashURL, createBasicEvent, getSource
venue, created = Organization.objects.get_or_create(
name="Palmer's Bar",
city="Minneapolis",
website="https://palmers-bar.com",
)
calendar_url = "https://palmers-bar.com"
DATETIME_FORMAT = '%d %m %Y %I:%M%p'
def get_events(ps, link):
contents = ps.xpath('.//*/td')
for c in contents:
try:
# day_num = c.xpath('.//*/div[@class="marker-daynum"]/text()')[0]
events = c.xpath('.//*/li')
# print(events)
for e in events:
event = {}
event_link = calendar_url + e.xpath('.//a/@href')[0]
ps = getSource(br, event_link)
new_event = ps.xpath('.//*/h1[@class="eventitem-column-meta"]')
event['title'] = new_event.xpath('.//*/h1[@class="event-title"]')
event['date'] = new_event.xpath('.//*/time[@class="event-date"]')
event['time'] = new_event.xpath('.//*/time[@class="event-time-12hr-start"]')
event['link'] = event_link
# event['month'] = link.split("month=")[1].split("-")[0]
# event['year'] = link.split("month=")[1].split("-")[1]
# event['title'] = e.xpath('.//h1/a[@class="flyoutitem-link"]/text()')
# event['time'] = e.xpath('.//div[@class="flyoutitem-datetime flyoutitem-datetime--12hr"]/text()')
# event['date'] = str(day_num) + ' ' + 'time'
# event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
ppr(event)
if len(event['title']):
nevent = {}
nevent['title'] = "".join(event['title']).strip()
event['time'] = event['time'][0].strip().split(" –")[0]
nevent['link'] = "https://palmers-bar.com" + e.xpath('.//a/@href')[0]
event['dateStamp'] = str(day_num) + ' ' + event['month'] + ' ' + event['year'] + ' ' + event['time']
nevent['dateStamp'] = datetime.strptime(event['dateStamp'], DATETIME_FORMAT)
ppr(nevent)
# createBasicEvent(nevent, 'Mu', venue)
except Exception as e:
continue
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = getBrowser(arg1)
else:
print("No run_env")
quit()
links = createDashURL("https://palmers-bar.com/?view=calendar&month=")
for link in links:
ps = getSource(br, link)
get_events(ps, link)
sleep(5)
# ppr(events)
br.close()