more scrapers
This commit is contained in:
@@ -22,17 +22,13 @@ venue, created = Organization.objects.get_or_create(
|
||||
|
||||
scraper,item_count_start,virtcal = digitools.getScraper(venue,venue.website, 'msp')
|
||||
|
||||
DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p'
|
||||
DATETIME_FORMAT_2 = '%A, %B %d @ %I%p %Y'
|
||||
DATETIME_FORMAT_3 = '%A, %B %d at %I:%M%p %Y'
|
||||
DATETIME_FORMAT_4 = '%A, %B %d at %I%p %Y'
|
||||
DATETIME_FORMAT_5 = '%A, %B %d @%I%p %Y'
|
||||
tz_str = " -0600 UTC"
|
||||
DATETIME_FORMAT = '%A, %B %d, %Y %I:%M %p %z %Z'
|
||||
|
||||
|
||||
def get_events(ps):
|
||||
links = ps.xpath('.//*/div[@class="summary-title"]/a/@href')
|
||||
print("Length of Links: ", len(links))
|
||||
links = list(set(links))
|
||||
print("New Length of Links: ", len(links))
|
||||
for l in links:
|
||||
if "cedar-news-blog" in l:
|
||||
continue
|
||||
@@ -54,7 +50,7 @@ def get_events(ps):
|
||||
print(e)
|
||||
print("failed event: ", event)
|
||||
dateStamp = date + " " + time
|
||||
event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT)
|
||||
event['dateStamp'] = datetime.strptime(dateStamp + tz_str, DATETIME_FORMAT)
|
||||
digitools.createBasicEvent(event, "Mu", venue)
|
||||
scraper.items+=1
|
||||
|
||||
@@ -71,4 +67,4 @@ ps = digitools.getSource(br, calendar_url)
|
||||
get_events(ps)
|
||||
br.close()
|
||||
|
||||
digitools.updateScraper(scraper, item_count_start)
|
||||
digitools.updateScraper(scraper, item_count_start)
|
||||
|
||||
Reference in New Issue
Block a user