lots of updates :/
This commit is contained in:
@@ -1,12 +1,14 @@
|
||||
import re, os, sys
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
import requests
|
||||
import json
|
||||
|
||||
import django
|
||||
sys.path.append('../../../')
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
|
||||
sys.path.append('../../../../')
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
|
||||
django.setup()
|
||||
|
||||
from events.models import Event, Organization, Scraper
|
||||
from events.models import Event, Organization, Scraper, Calendar
|
||||
|
||||
from time import sleep
|
||||
from pprint import pprint as ppr
|
||||
@@ -17,13 +19,17 @@ from selenium.webdriver.common.by import By
|
||||
from xvfbwrapper import Xvfb
|
||||
from lxml import html
|
||||
import pytz
|
||||
from events.digitools import getBrowser, createURL, createBasicEvent, getSource
|
||||
from events.digitools import getBrowser, createURL, createBasicEvent, getSource, add_calendar
|
||||
|
||||
import events.digitools as digitools
|
||||
|
||||
try:
|
||||
scraper, created = Scraper.objects.get_or_create(
|
||||
name="Mpls City Council",
|
||||
website="https://lims.minneapolismn.gov/Calendar/citycouncil/upcoming",
|
||||
calendar = Calendar.objects.get(shortcode='msp'),
|
||||
items = 0,
|
||||
new_items = 0,
|
||||
last_ran = datetime.now(),
|
||||
)
|
||||
except Exception as e:
|
||||
@@ -31,11 +37,24 @@ except Exception as e:
|
||||
scraper = Scraper.objects.get(name="Mpls City Council")
|
||||
print("Scraper: ", scraper)
|
||||
|
||||
DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
|
||||
tz = pytz.timezone("US/Central")
|
||||
td = timedelta(days=2)
|
||||
odt = datetime.now() - td
|
||||
|
||||
DATETIME_FORMAT = '%A, %b %d, %Y %I:%M %p'
|
||||
month = odt.strftime("%b")
|
||||
day = int(datetime.now().day)
|
||||
|
||||
calendar_url = 'https://lims.minneapolismn.gov/Calendar/citycouncil/upcoming'
|
||||
if day == 1:
|
||||
day = 30
|
||||
|
||||
year = int(datetime.now().year)
|
||||
|
||||
cal_url = 'https://lims.minneapolismn.gov/Calendar/GetCalenderList?fromDate={}%20{},%20{}&toDate=null&meetingType=0&committeeId=null&pageCount=100&offsetStart=0&abbreviation=undefined&keywords=&sortOrder=1'.format(month, day, year)
|
||||
|
||||
# print("URL: ", cal_url)
|
||||
|
||||
# cal_url = 'https://lims.minneapolismn.gov/Calendar/GetCalenderList?fromDate=Nov%2015,%202025&toDate=null&meetingType=0&committeeId=null&pageCount=50&offsetStart=0&abbreviation=undefined&keywords=&sortOrder=1'
|
||||
|
||||
if len(sys.argv) >= 2:
|
||||
arg1 = sys.argv[1]
|
||||
@@ -44,56 +63,60 @@ else:
|
||||
print("No run_env")
|
||||
quit()
|
||||
|
||||
br.get(calendar_url)
|
||||
sleep(25)
|
||||
# br.find_element(By.XPATH, '//*/li[@class="tab-header-small"]/a').click()
|
||||
# sleep(15)
|
||||
# all_entries = Select(br.find_element(By.XPATH, '//*/select'))
|
||||
# all_entries.select_by_value('50')
|
||||
# sleep(15)
|
||||
|
||||
br.get(cal_url)
|
||||
sleep(2)
|
||||
ps = html.fromstring(br.page_source)
|
||||
events = ps.xpath('.//body/pre/text()')[0]
|
||||
new_events = json.loads(events)
|
||||
# ppr(dict(new_events[0]))
|
||||
event_list = []
|
||||
|
||||
dayBlocks = ps.xpath('.//*/div[@class="ng-scope"]')
|
||||
meetings = []
|
||||
|
||||
for dB in dayBlocks:
|
||||
date = dB.xpath('.//div[@class="row"]/div/span[@class="ng-binding"]/text()')[0]
|
||||
events = dB.xpath('.//div[@class="upcoming ng-scope"]/div')
|
||||
for event in events:
|
||||
time = event.xpath('.//div/text()')[0]
|
||||
title = event.xpath('.//div/a/text()')[0].strip()
|
||||
if not len(title) > 0:
|
||||
title = event.xpath('.//div/span/a/text()')[0].strip()
|
||||
link = event.xpath('.//div/a/@href')[0]
|
||||
if link.startswith("/Download/"):
|
||||
link = calendar_url
|
||||
else:
|
||||
link = "https://lims.minneapolismn.gov" + link
|
||||
location = title.split(',')[-1].strip()
|
||||
mtg_title = title.split(',')[:-1]
|
||||
if len(mtg_title) > 1:
|
||||
mtg_title = (' -').join(mtg_title).strip()
|
||||
else:
|
||||
mtg_title = mtg_title[0].strip()
|
||||
dateTime = datetime.strptime(date + " " + time, DATETIME_FORMAT)
|
||||
if location == "City Hall":
|
||||
location = "Mpls City Hall"
|
||||
print(dateTime, location, mtg_title, link)
|
||||
print('\n\n++++\n\n')
|
||||
venue, created = Organization.objects.get_or_create(name=location, city="Minneapolis")
|
||||
new_event = Event.objects.update_or_create(
|
||||
calendar = 'msp'
|
||||
event_type = 'Gv',
|
||||
show_title = mtg_title,
|
||||
show_link = link,
|
||||
show_date = dateTime,
|
||||
show_day = dateTime,
|
||||
venue = venue
|
||||
)
|
||||
scraper.items+=1
|
||||
|
||||
for event in new_events:
|
||||
e = {}
|
||||
e['title'] = event['CommitteeName']
|
||||
e['link'] = scraper.website
|
||||
e['dateStamp'] = datetime.strptime(event['MeetingTime'], DATETIME_FORMAT)
|
||||
e['agendaStatus'] = event['AgendaStatus']
|
||||
e['address'] = event['Address']
|
||||
e['description'] = event['Description']
|
||||
e['scraper'] = scraper
|
||||
e['calendars'] = [scraper.calendar]
|
||||
try:
|
||||
# print(event['Location'].split(",")[1])
|
||||
loc = event['Location'].split(",")[1]
|
||||
if "City Hall" in loc:
|
||||
e['location'] = "Mpls City Hall"
|
||||
venue = Organization.objects.get(
|
||||
name="Mpls City Hall",
|
||||
)
|
||||
digitools.createBasicEvent(e, 'Gv', venue)
|
||||
elif "Public Service Building" in loc:
|
||||
e['location'] = "Mpls Public Service Building"
|
||||
venue = Organization.objects.get(
|
||||
name="Mpls Public Service Building",
|
||||
)
|
||||
digitools.createBasicEvent(e, 'Gv', venue)
|
||||
except:
|
||||
# print(event['Location'].split("-")[0])
|
||||
e['location'] = event['Location'].split("-")[0].strip()
|
||||
try:
|
||||
venue, created = Organization.objects.get_or_create(
|
||||
name=e['location'] ,
|
||||
city="Minneapolis",
|
||||
website=scraper.website,
|
||||
is_venue=False,
|
||||
address_complete=e['address']
|
||||
)
|
||||
except:
|
||||
venue = Organization.objects.get(
|
||||
name=e['location'] ,
|
||||
city="Minneapolis",
|
||||
)
|
||||
digitools.createBasicEvent(e, 'Gv', venue)
|
||||
event_list.append(e)
|
||||
|
||||
|
||||
br.close()
|
||||
|
||||
scraper.save()
|
||||
# br.find_element_by_class_name('fc-btn_allCalendars-button').click()
|
||||
|
||||
Reference in New Issue
Block a user