import os, sys from datetime import datetime from dateutil import relativedelta sys.path.append('/var/www/digisnaxx.ado/scrapers') import dtss dtss.getReady() from time import sleep from pprint import pprint as ppr import pytz from events.models import Organization, Scraper, Event import events.digitools as digitools from selenium.webdriver.common.by import By from lxml import html venue, created = Organization.objects.get_or_create( name="Uptown VFW", city="Minneapolis", website="https://noboolpresents.com/venues/uptown-vfw/", is_venue = True ) scraper,item_count_start, virtcal = digitools.getScraper(venue, venue.website, 'msp') tz_str = " -0600 UTC" DATETIME_FORMAT = '%a %B %d @ %I:%M %p %Y %z %Z' DATETIME_FORMAT_2 = '%a %B %d @ %I:%M %p %Y %z %Z' calendar_url = 'https://noboolpresents.com/venues/uptown-vfw/' current_year = str(datetime.now().year) if len(sys.argv) >= 2: arg1 = sys.argv[1] br = digitools.getBrowser(arg1) else: print("No run_env") quit() br.get(calendar_url) sleep(20) def getEvents(br): ps = html.fromstring(br.page_source) events = ps.xpath('.//*/article') for event in events: deets = {} dateTime = event.xpath('.//*/span[@class="tribe-event-date-start"]/text()')[0].replace("•", "").strip() + " " + current_year title = event.xpath('.//*/h2[@class="alt-font"]/a/text()')[0].replace("\n", "").replace("\t", "") link = event.xpath('.//*/h2[@class="alt-font"]/a/@href')[0] try: print("First Try", dateTime) new_event, created = Event.objects.update_or_create( scraper = scraper, event_type = 'Mu', show_title = title, show_link = link, show_date = datetime.strptime(dateTime + tz_str, DATETIME_FORMAT), venue = venue ) digitools.add_calendar(new_event, 'msp') scraper.items+=1 except Exception as e: try: print("New Try", dateTime) new_event, created = Event.objects.update_or_create( scraper = scraper, event_type = 'Mu', show_title = title, show_link = link, show_date = datetime.strptime(dateTime.strip() + tz_str, DATETIME_FORMAT_2), venue = venue ) digitools.add_calendar(new_event, 'msp') scraper.items+=1 except Exception as e: print("oops", e) ppr(event) getEvents(br) br.find_element(By.XPATH, './/*/li[@class="tribe-events-c-nav__list-item tribe-events-c-nav__list-item--next"]/a').click() sleep(5) getEvents(br) br.close() digitools.updateScraper(scraper, item_count_start)