more scrapers

This commit is contained in:
2026-01-24 19:01:47 -05:00
parent 7013d8327a
commit 3c4a41ae2c
58 changed files with 1614 additions and 2988 deletions

View File

@@ -22,17 +22,13 @@ venue, created = Organization.objects.get_or_create(
scraper,item_count_start,virtcal = digitools.getScraper(venue,venue.website, 'msp')
DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p'
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
DATETIME_FORMAT_3 = '%A, %B %d at %I:%M%p %Y'
DATETIME_FORMAT_4 = '%A, %B %d at %I%p %Y'
DATETIME_FORMAT_5 = '%A, %B %d @%I%p %Y'
tz_str = " -0600 UTC"
DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p %z %Z'
def get_events(ps):
links = ps.xpath('.//*/div[@class="summary-title"]/a/@href')
print("Length of Links: ", len(links))
links = list(set(links))
print("New Length of Links: ", len(links))
for l in links:
if "cedar-news-blog" in l:
continue
@@ -54,7 +50,7 @@ def get_events(ps):
print(e)
print("failed event: ", event)
dateStamp = date + " " + time
event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
event['dateStamp'] = datetime.strptime(dateStamp + tz_str, DATETIME_FORMAT)
digitools.createBasicEvent(event, "Mu", venue)
scraper.items+=1
@@ -71,4 +67,4 @@ ps = digitools.getSource(br, calendar_url)
get_events(ps)
br.close()
digitools.updateScraper(scraper, item_count_start)
digitools.updateScraper(scraper, item_count_start)