first push

Working/govt/MplsCityCouncil.py

@@ -0,0 +1,122 @@
+import re, os, sys
+from datetime import datetime, timedelta
+import requests
+import json
+
+import django
+sys.path.append('../../../../')
+os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local'
+django.setup()
+
+from events.models import Event, Organization, Scraper, Calendar
+
+from time import sleep
+from pprint import pprint as ppr
+from selenium import webdriver as wd
+from selenium.webdriver.support.ui import Select
+from selenium.webdriver.common.by import By
+
+from xvfbwrapper import Xvfb
+from lxml import html
+import pytz
+from events.digitools import getBrowser, createURL, createBasicEvent, getSource, add_calendar
+
+import events.digitools as digitools
+
+try:
+    scraper, created = Scraper.objects.get_or_create(
+        name="Mpls City Council",
+        website="https://lims.minneapolismn.gov/Calendar/citycouncil/upcoming",
+        calendar = Calendar.objects.get(shortcode='msp'),
+        items = 0,
+        new_items = 0,
+        last_ran = datetime.now(),
+    )
+except Exception as e:
+    print(e)
+    scraper = Scraper.objects.get(name="Mpls City Council")
+    print("Scraper: ", scraper)
+
+DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
+tz = pytz.timezone("US/Central")
+td = timedelta(days=2)
+odt = datetime.now() - td
+
+month = odt.strftime("%b")
+day = int(datetime.now().day)
+
+if day == 1:
+    day = 30
+
+year = int(datetime.now().year)
+
+cal_url = 'https://lims.minneapolismn.gov/Calendar/GetCalenderList?fromDate={}%20{},%20{}&toDate=null&meetingType=0&committeeId=null&pageCount=100&offsetStart=0&abbreviation=undefined&keywords=&sortOrder=1'.format(month, day, year)
+
+# print("URL: ", cal_url)
+
+# cal_url = 'https://lims.minneapolismn.gov/Calendar/GetCalenderList?fromDate=Nov%2015,%202025&toDate=null&meetingType=0&committeeId=null&pageCount=50&offsetStart=0&abbreviation=undefined&keywords=&sortOrder=1'
+
+if len(sys.argv) >= 2:
+    arg1 = sys.argv[1]
+    br = getBrowser(arg1)
+else:
+    print("No run_env")
+    quit()
+
+br.get(cal_url)
+sleep(2)
+ps = html.fromstring(br.page_source)
+events = ps.xpath('.//body/pre/text()')[0]
+new_events = json.loads(events)
+# ppr(dict(new_events[0]))
+event_list = []
+
+for event in new_events:
+    e = {}
+    e['title'] = event['CommitteeName']
+    e['link'] = scraper.website
+    e['dateStamp'] = datetime.strptime(event['MeetingTime'], DATETIME_FORMAT)
+    e['agendaStatus'] = event['AgendaStatus']
+    e['address'] = event['Address']
+    e['description'] = event['Description']
+    e['scraper'] = scraper
+    e['calendars'] = [scraper.calendar]
+    try:
+        # print(event['Location'].split(",")[1])
+        loc = event['Location'].split(",")[1]
+        if "City Hall" in loc:
+            e['location'] = "Mpls City Hall"
+            venue = Organization.objects.get(
+                name="Mpls City Hall",
+            )
+            digitools.createBasicEvent(e, 'Gv', venue)
+        elif "Public Service Building" in loc:
+            e['location'] = "Mpls Public Service Building"
+            venue = Organization.objects.get(
+                name="Mpls Public Service Building",
+            )
+            digitools.createBasicEvent(e, 'Gv', venue)
+    except:
+        # print(event['Location'].split("-")[0])
+        e['location'] = event['Location'].split("-")[0].strip()
+        try:
+            venue, created = Organization.objects.get_or_create(
+                    name=e['location'] ,
+                    city="Minneapolis",
+                    website=scraper.website,
+                    is_venue=False,
+                    address_complete=e['address']
+                )
+        except:
+            venue = Organization.objects.get(
+                    name=e['location'] ,
+                    city="Minneapolis",
+                )
+        digitools.createBasicEvent(e, 'Gv', venue)
+    event_list.append(e)
+    
+
+br.close()
+
+scraper.save()
+# br.find_element_by_class_name('fc-btn_allCalendars-button').click()