import os, sys from datetime import datetime from dateutil import relativedelta import django sys.path.append('../../../../') os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local' django.setup() from time import sleep from pprint import pprint as ppr import pytz from events.models import Organization, Scraper, Event import events.digitools as digitools from selenium.webdriver.common.by import By from lxml import html venue, created = Organization.objects.get_or_create( name="Uptown VFW", city="Minneapolis", website="https://noboolpresents.com/venues/uptown-vfw/", is_venue = True ) scraper,item_count_start, virtcal = digitools.getScraper(venue, 'msp') tz = pytz.timezone("US/Central") DATETIME_FORMAT = '%a %B %d @ %I:%M %p %Y' DATETIME_FORMAT_2 = '%b %d %I:%M%p %Y' DATETIME_FORMAT_3 = '%b %d %Y' # Set initial variables for City, etc calendar_url = 'https://noboolpresents.com/venues/uptown-vfw/' current_year = str(datetime.now().year) if len(sys.argv) >= 2: arg1 = sys.argv[1] br = digitools.getBrowser(arg1) else: print("No run_env") quit() br.get(calendar_url) sleep(20) def getEvents(br): ps = html.fromstring(br.page_source) events = ps.xpath('.//*/article') for event in events: deets = {} dateTime = event.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].replace("•", "").strip() + " " + current_year title = event.xpath('.//*/h2[@class="alt-font"]/a/text()')[0].replace("\n", "").replace("\t", "") link = event.xpath('.//*/h2[@class="alt-font"]/a/@href')[0] # deets["tickets"] = event.xpath('.//*/span[@class="tribe-events-c-small-cta__price"]/strong/text()')[0] try: new_event, created = Event.objects.update_or_create( scraper = scraper, event_type = 'Mu', show_title = title, show_link = link, show_date = datetime.strptime(dateTime, DATETIME_FORMAT), show_day = datetime.strptime(dateTime, DATETIME_FORMAT), # more_details = deets["tickets"], venue = venue ) digitools.add_calendar(new_event, 'msp') scraper.items+=1 except Exception as e: print("oops", e) getEvents(br) br.find_element(By.XPATH, './/*/li[@class="tribe-events-c-nav__list-item tribe-events-c-nav__list-item--next"]/a').click() sleep(5) getEvents(br) br.close() digitools.updateScraper(scraper, item_count_start)