import re, os, sys from datetime import datetime, timedelta import requests import json import django sys.path.append('../../../../') os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local' django.setup() from events.models import Event, Organization, Scraper, Calendar from time import sleep from pprint import pprint as ppr from selenium import webdriver as wd from selenium.webdriver.support.ui import Select from selenium.webdriver.common.by import By from xvfbwrapper import Xvfb from lxml import html import pytz from events.digitools import getBrowser, createURL, createBasicEvent, getSource, add_calendar import events.digitools as digitools try: scraper, created = Scraper.objects.get_or_create( name="Mpls City Council", website="https://lims.minneapolismn.gov/Calendar/citycouncil/upcoming", calendar = Calendar.objects.get(shortcode='msp'), items = 0, new_items = 0, last_ran = datetime.now(), ) except Exception as e: print(e) scraper = Scraper.objects.get(name="Mpls City Council") print("Scraper: ", scraper) DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S' tz = pytz.timezone("US/Central") td = timedelta(days=2) odt = datetime.now() - td month = odt.strftime("%b") day = int(datetime.now().day) if day == 1: day = 30 year = int(datetime.now().year) cal_url = 'https://lims.minneapolismn.gov/Calendar/GetCalenderList?fromDate={}%20{},%20{}&toDate=null&meetingType=0&committeeId=null&pageCount=100&offsetStart=0&abbreviation=undefined&keywords=&sortOrder=1'.format(month, day, year) # print("URL: ", cal_url) # cal_url = 'https://lims.minneapolismn.gov/Calendar/GetCalenderList?fromDate=Nov%2015,%202025&toDate=null&meetingType=0&committeeId=null&pageCount=50&offsetStart=0&abbreviation=undefined&keywords=&sortOrder=1' if len(sys.argv) >= 2: arg1 = sys.argv[1] br = getBrowser(arg1) else: print("No run_env") quit() br.get(cal_url) sleep(2) ps = html.fromstring(br.page_source) events = ps.xpath('.//body/pre/text()')[0] new_events = json.loads(events) # ppr(dict(new_events[0])) event_list = [] for event in new_events: e = {} e['title'] = event['CommitteeName'] e['link'] = scraper.website e['dateStamp'] = datetime.strptime(event['MeetingTime'], DATETIME_FORMAT) e['agendaStatus'] = event['AgendaStatus'] e['address'] = event['Address'] e['description'] = event['Description'] e['scraper'] = scraper e['calendars'] = [scraper.calendar] try: # print(event['Location'].split(",")[1]) loc = event['Location'].split(",")[1] if "City Hall" in loc: e['location'] = "Mpls City Hall" venue = Organization.objects.get( name="Mpls City Hall", ) digitools.createBasicEvent(e, 'Gv', venue) elif "Public Service Building" in loc: e['location'] = "Mpls Public Service Building" venue = Organization.objects.get( name="Mpls Public Service Building", ) digitools.createBasicEvent(e, 'Gv', venue) except: # print(event['Location'].split("-")[0]) e['location'] = event['Location'].split("-")[0].strip() try: venue, created = Organization.objects.get_or_create( name=e['location'] , city="Minneapolis", website=scraper.website, is_venue=False, address_complete=e['address'] ) except: venue = Organization.objects.get( name=e['location'] , city="Minneapolis", ) digitools.createBasicEvent(e, 'Gv', venue) event_list.append(e) br.close() scraper.save() # br.find_element_by_class_name('fc-btn_allCalendars-button').click()