DigiSnaxx/scrapers
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more scrapers

Browse Source
This commit is contained in:
canin carlos
2026-01-24 19:01:47 -05:00
parent 7013d8327a
commit 3c4a41ae2c
58 changed files with 1614 additions and 2988 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
Working/govt/co.denver.py Normal file
View File

@@ -0,0 +1,66 @@
import os, sys
from datetime import datetime
from dateutil import relativedelta
sys.path.append('/var/www/digisnaxx.ado/scrapers')
import dtss
dtss.getReady()
from time import sleep
from pprint import pprint as ppr
import pytz
from events.models import Organization, Scraper
import events.digitools as digitools
venue, created = Organization.objects.get_or_create(
name="Denver City Council",
city="Denver",
website="https://denver.legistar.com/Calendar.aspx",
is_venue=True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, venue.website, 'dia')
event_type = "Gv"
# Time Signatures
tz_str = " -0600 UTC"
DATETIME_FORMAT = '%m/%d/%Y %I:%M %p %z %Z'
DATETIME_FORMAT_2 = '%B %d, %Y'
def get_events(ps, event_type):
print("Getting events ...")
contents = ps.xpath('.//*/tr[@class="rgRow"]')
for c in contents:
try:
event = {}
event['scraper'] = scraper
event['link'] = venue.website
event['calendars'] = [scraper.calendar]
event['title'] = c.xpath('.//*/a/text()')[0]
event['time'] = c.xpath('.//*/span/text()')[2]
event['date'] = c.xpath('.//td[@class="rgSorted"]/text()')[0]
event['dateStamp'] = datetime.strptime(event['date']+" "+event['time'] + tz_str, DATETIME_FORMAT)
ppr(event)
digitools.createBasicEvent(event, event_type, venue)
scraper.items+=1
except Exception as e:
print(e)
ppr(event)
print("\n\n+++\n\n")
if len(sys.argv) >= 2:
arg1 = sys.argv[1]
br = digitools.getBrowser(arg1)
else:
print("No run_env")
quit()
ps = digitools.getSource(br, venue.website)
sleep(1)
get_events(ps, "Gv")
sleep(2)
digitools.updateScraper(scraper, item_count_start)
br.close()

8
Working/govt/ma.boston.citycouncil.py
View File

@@ -20,13 +20,13 @@ venue, created = Organization.objects.get_or_create(
is_venue=True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'bos')
scraper,item_count_start,virtcal = digitools.getScraper(venue, venue.website, 'bos')
event_type = "Gv"
# Time Signatures
tz = pytz.timezone("US/Eastern")
DATETIME_FORMAT = '%B %d, %Y %I:%M%p'
tz_str = " -0500 UTC"
DATETIME_FORMAT = '%B %d, %Y %I:%M%p %z %Z'
def get_events(ps, event_type):
print("Getting events ...")
@@ -40,7 +40,7 @@ def get_events(ps, event_type):
event['title'] = c.xpath('.//*/div[@class="n-li-t"]/a/text()')[0]
event['link'] = venue.website + c.xpath('.//*/div[@class="n-li-t"]/a/@href')[0]
event['date'] = c.xpath('.//*/li[@class="dl-i"]/span[@class="dl-d"]/text()')[0].replace('\n', '').split("-")[0].strip()
event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT)
event['dateStamp'] = datetime.strptime(event['date'] + tz_str, DATETIME_FORMAT)
digitools.createBasicEvent(event, event_type, venue)
ppr(event)
scraper.items+=1

16
Working/govt/mn.legislature.py
View File

@@ -35,8 +35,8 @@ except Exception as e:
print("Scraper: ", scraper)
tz = pytz.timezone("US/Central")
DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p'
tz_str = " -0600 UTC"
DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p %z %Z'
# Set initial variables for City, etc
calendar_url = 'https://www.leg.mn.gov/cal?type=all'
@@ -76,8 +76,8 @@ for hE in houseEvents:
show_title = title,
show_link = link,
show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
show_day = datetime.strptime(dateTime, DATETIME_FORMAT).date(),
more_details = details['location'],
show_day = datetime.strptime(dateTime + tz_str, DATETIME_FORMAT).date(),
# more_details = details['location'],
venue = venue,
scraper = scraper
)
@@ -113,8 +113,8 @@ for sE in senateEvents:
show_title = title,
show_link = link,
show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
show_day = datetime.strptime(dateTime, DATETIME_FORMAT).date(),
more_details = details['location'],
show_day = datetime.strptime(dateTime + tz_str, DATETIME_FORMAT).date(),
# more_details = details['location'],
venue = venue,
scraper = scraper
)
@@ -147,8 +147,8 @@ for cE in commEvents:
show_title = title,
show_link = link,
show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
show_day = datetime.strptime(dateTime, DATETIME_FORMAT).date(),
more_details = details['location'],
show_day = datetime.strptime(dateTime + tz_str, DATETIME_FORMAT).date(),
# more_details = details['location'],
venue = venue,
scraper = scraper
)

6
Working/govt/mn.mngov.py
View File

@@ -49,6 +49,8 @@ gcal = iCalendar.from_ical(objIcalData.text)
cfpa_events = []
tz = pytz.timezone("US/Central")
tz_str = " -0600 UTC"
for component in gcal.walk():
event = {}
@@ -84,8 +86,8 @@ for component in gcal.walk():
event_type = event_type,
show_title = event['strSummary'],
show_link = venue.website,
show_date = event['dateStart']-td,
show_day = event['dateStart']-td,
show_date = event['dateStart'],
show_day = event['dateStart'],
more_details = event["details"],
venue = venue
)

8
Working/govt/mn.mpls.citycouncil.py
View File

@@ -36,11 +36,15 @@ except Exception as e:
scraper = Scraper.objects.get(name="Mpls City Council")
print("Scraper: ", scraper)
DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S %z %Z'
tz = pytz.timezone("US/Central")
td = timedelta(days=2)
odt = datetime.now() - td
tz_str = " -0600 UTC"
month = odt.strftime("%b")
day = int(datetime.now().day)
@@ -74,7 +78,7 @@ for event in new_events:
e = {}
e['title'] = event['CommitteeName']
e['link'] = scraper.website
e['dateStamp'] = datetime.strptime(event['MeetingTime'], DATETIME_FORMAT)
e['dateStamp'] = datetime.strptime(event['MeetingTime'] + tz_str, DATETIME_FORMAT)
e['agendaStatus'] = event['AgendaStatus']
e['address'] = event['Address']
e['description'] = event['Description']

5
Working/govt/mn.stp.citycouncil.py
View File

@@ -32,8 +32,9 @@ except Exception as e:
print("Scraper: ", scraper)
tz = pytz.timezone("US/Central")
tz_str = " -0600 UTC"
DATETIME_FORMAT = '%B %d, %Y at %I:%M %p'
DATETIME_FORMAT = '%B %d, %Y at %I:%M %p %z %Z'
calendar_url = 'https://www.stpaul.gov/calendar'
city_site = "https://www.stpaul.gov"
@@ -64,7 +65,7 @@ def getEvents(br):
event_type = 'Gv',
show_title = title,
show_link = link,
show_date = datetime.strptime(dateTime, DATETIME_FORMAT),
show_date = datetime.strptime(dateTime + tz_str, DATETIME_FORMAT),
show_day = datetime.strptime(dateTime, DATETIME_FORMAT),
venue = venue,
scraper = scraper

12
Working/govt/or.portland.py
View File

@@ -20,14 +20,16 @@ venue, created = Organization.objects.get_or_create(
is_venue=True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'pdx')
scraper,item_count_start,virtcal = digitools.getScraper(venue, venue.website, 'pdx')
event_type = "Gv"
# Time Signatures
tz = pytz.timezone("US/Eastern")
DATETIME_FORMAT = '%B %d, %Y %I:%M %p'
DATETIME_FORMAT_2 = '%B %d, %Y'
tz_str = " -0800 UTC"
DATETIME_FORMAT = '%B %d, %Y %I:%M %p %z %Z'
DATETIME_FORMAT_2 = '%B %d, %Y %z %Z'
def get_events(ps, event_type):
print("Getting events ...")
@@ -44,9 +46,9 @@ def get_events(ps, event_type):
event['time'] = c.xpath('.//*/span[@class="pe-4"]/text()')[0].replace("\n", "").strip()
# event['time2'] = c.xpath('.//*/span[@class="pe-4"]/text()')
try:
event['dateStamp'] = datetime.strptime(event['date']+" "+event['time'], DATETIME_FORMAT)
event['dateStamp'] = datetime.strptime(event['date']+" "+event['time'] + tz_str, DATETIME_FORMAT)
except:
event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT_2)
event['dateStamp'] = datetime.strptime(event['date'] + tz_str, DATETIME_FORMAT_2)
# ppr(event)

7
Working/govt/pa.philadelpha.py
View File

@@ -20,13 +20,14 @@ venue, created = Organization.objects.get_or_create(
is_venue=True
)
scraper,item_count_start,virtcal = digitools.getScraper(venue, 'phl')
scraper,item_count_start,virtcal = digitools.getScraper(venue, venue.website, 'phl')
event_type = "Gv"
# Time Signatures
tz = pytz.timezone("US/Eastern")
DATETIME_FORMAT = '%m/%d/%Y %I:%M %p'
tz_str = " -0500 UTC"
DATETIME_FORMAT = '%m/%d/%Y %I:%M %p %z %Z'
DATETIME_FORMAT_2 = '%B %d, %Y'
def get_events(ps, event_type):
@@ -41,7 +42,7 @@ def get_events(ps, event_type):
event['title'] = c.xpath('.//*/a/text()')[0]
event['time'] = c.xpath('.//*/span/text()')[2]
event['date'] = c.xpath('.//td[@class="rgSorted"]/text()')[0]
event['dateStamp'] = datetime.strptime(event['date']+" "+event['time'], DATETIME_FORMAT)
event['dateStamp'] = datetime.strptime(event['date']+" "+event['time'] + tz_str, DATETIME_FORMAT)
# ppr(event)
digitools.createBasicEvent(event, event_type, venue)
scraper.items+=1