import os, sys from datetime import datetime from dateutil import relativedelta from time import sleep import pytz from lxml import html from pprint import pprint as ppr import django from xvfbwrapper import Xvfb from selenium import webdriver as wd from events.models import Event as DSEvent, Organization, Promo, Scraper, Calendar tz = pytz.timezone("US/Central") td = relativedelta.relativedelta(months=1) odt = datetime.now() + td def getReady(): sys.path.append('../../') os.environ['DJANGO_SETTINGS_MODULE'] = 'config.django.local' django.setup() return def getScraper(venue, cal): virtcal = Calendar.objects.get(shortcode='000') try: scraper, created = Scraper.objects.get_or_create( name=venue.name, website=venue.website, calendar = Calendar.objects.get(shortcode=cal), items = 0, new_items = 0, last_ran = datetime.now(), ) except Exception as e: print(e) scraper = Scraper.objects.get(name=venue.name) num_of_events = DSEvent.objects.filter(scraper=scraper) scraper.items = len(num_of_events) scraper.save() print("Scraper: ", scraper) pass return scraper, scraper.items, virtcal def updateScraper(scraper, item_count_start): num_of_events = DSEvent.objects.filter(scraper=scraper) scraper.items = len(num_of_events) scraper.new_items = len(num_of_events) - item_count_start scraper.last_ran = datetime.now() scraper.save() return def getSource(browser, link): browser.get(link) sleep(5) ps = html.fromstring(browser.page_source) return ps def getBrowser(run_env): if run_env == 'dev': print("Chrome is a go!") # chromeOptions = wd.ChromeOptions() # chromeOptions.binary_location = "/Application/Google\ Chrome.app" # chromeDriver = "/opt/homebrew/bin/chromedriver" # br = wd.Chrome(chromeDriver, options=chromeOptions) br = wd.Chrome() return br elif run_env == "def": print("Firefox go vroom") br = wd.Firefox() return br elif run_env == "prod": start_cmd = "Xvfb :91 && export DISPLAY=:91 &" xvfb = Xvfb() os.system(start_cmd) xvfb.start() print("started Xvfb") br = wd.Firefox() return br else: print("Failed", sys.argv, arg1) quit() def createBasicURL(site_url): month = datetime.now().month next_month = odt.month year = datetime.now().year links = [ site_url + str(month) + "/" + str(year), site_url + str(next_month) + "/" + str(year) ] return links def createURLNoZero(site_url): month = datetime.now().month next_month = odt.month year = datetime.now().year links = [ site_url + str(year) + "/" + str(month), ] if next_month == "1": links.append(site_url + str(int(year)+1) + "/" + str(next_month)) else: links.append(site_url + str(year) + "/" + str(next_month)) return links def createURL(site_url): month = datetime.now().month if month < 10: month = "0" + str(month) else: month = str(month) next_month = odt.month if next_month < 10: next_month = "0" + str(next_month) else: next_month = str(next_month) year = datetime.now().year links = [ site_url + str(year) + "/" + month, ] if next_month == "01": links.append(site_url + str(int(year)+1) + "/" + next_month) else: links.append(site_url + str(year) + "/" + next_month) return links def createDashURL(site_url): month = datetime.now().month if month < 10: month = "0" + str(month) else: month = str(month) next_month = odt.month if next_month < 10: next_month = "0" + str(next_month) else: next_month = str(next_month) year = datetime.now().year links = [ site_url + month + "-" + str(year), site_url + next_month + "-" + str(year) ] print(links) return links def add_calendar(event, calendar): print("\nNEW EVENT DIGITOOLS: ", event, type(event), calendar) if type(event) is tuple: print("STS: ", event) event = event[0] cal = Calendar.objects.get(shortcode=calendar) event.calendar.add(cal) event.save() return event def add_calendars(event, data): if type(data['calendars']) is not list: event.calendar.add(data['calendars']) else: for cal in data['calendars']: event.calendar.add(cal) event.save() return event def createBasicEvent(event, event_type, venue): new_event, created = DSEvent.objects.update_or_create( event_type = event_type, show_title = event['title'], show_link = event['link'], show_date = event['dateStamp'], show_day = event['dateStamp'], scraper = event['scraper'], venue = venue ) new_event = add_calendars(new_event, event) return new_event, created def createBasiciCalEvent(event, event_type, venue): new_event, created = DSEvent.objects.update_or_create( event_type = event_type, show_title = event['title'][0], show_link = event['link'], show_date = datetime.strptime(str(event['dateStamp'][0]), '%Y-%m-%d %H:%M:%S'), show_day = datetime.strptime(str(event['dateStamp'][0]), '%Y-%m-%d %H:%M:%S'), scraper = event['scraper'], venue = venue ) new_event = add_calendars(new_event, event) return new_event, created def createDetailedEvent(event, event_type, venue, scraper): new_event, created = DSEvent.objects.update_or_create( event_type = event_type, show_title = event["show_title"], show_link = event["link"], show_date = event["dateStamp"], show_day = event['dateStamp'], guests = " ".join(event["guests"]), more_details = event["details"], scraper = event['scraper'], venue = venue ) new_event = add_calendars(new_event, event) return new_event, created def createBasicArticle(article, event_type, organization): new_article, created = Promo.objects.update_or_create( promo_type = 'Ja', title = article['title'], target_link = article['link'], published = True, organization = organization ) return new_article, created def createIcalEvent(event, scraper, venue, event_type): new_date = event['dateStart'] new_event = {} new_event['scraper'] = scraper new_event['calendar'] = scraper.calendar new_event['title'] = event['strSummary'], new_event['date'] = str(new_date)[:-6], new_event['dateStamp'] = str(new_date)[:-6], new_event['link'] = venue.website print("NEW iCal Event") ppr(new_event) createBasiciCalEvent(new_event, event_type, venue) def createCleanIcalEvent(event, scraper, venue, event_type): new_date = event['dateStart'] new_event = {} new_event['scraper'] = scraper new_event['calendars'] = scraper.calendar new_event['title'] = event['strSummary'], new_event['date'] = str(new_date), new_event['dateStamp'] = str(new_date), new_event['link'] = venue.website createBasiciCalEvent(new_event, event_type, venue) def getiCalEvents(gcal, scraper, venue, event_type): events = [] for component in gcal.walk(): event = {} event['scraper'] = scraper event['calendars'] = [scraper.calendar] event['strSummary'] = f"{(component.get('SUMMARY'))}" event['strDesc'] = component.get('DESCRIPTION') event['strLocation'] = component.get('LOCATION') event['dateStart'] = component.get('DTSTART') event['dateStamp'] = component.get('DTSTAMP') if event['dateStamp'] is not None: event['dateStamp'] = event['dateStamp'].dt if event['dateStart'] is not None: try: event['dateStart'] = event['dateStart'].dt except Exception as e: event['dateStart'] = event['dateStart'].dt event['dateEnd'] = (component.get('DTEND')) if event['dateEnd'] is not None: event['dateEnd'] = event['dateEnd'].dt else: event['dateEnd'] = event['dateStart'] if event['strSummary'] != 'None': event['details'] = { "description" : event['strDesc'], "Location" : event['strLocation'], } events.append(event) return events def returniCalEvents(gcal, scraper, venue, event_type): events = [] for component in gcal.walk(): event = {} event['scraper'] = scraper event['calendars'] = [scraper.calendar] event['strSummary'] = f"{(component.get('SUMMARY'))}" event['strDesc'] = component.get('DESCRIPTION') event['strLocation'] = component.get('LOCATION') event['dateStart'] = component.get('DTSTART') event['dateStamp'] = component.get('DTSTAMP') if event['dateStamp'] is not None: event['dateStamp'] = event['dateStamp'].dt if event['dateStart'] is not None: try: event['dateStart'] = event['dateStart'].dt except Exception as e: event['dateStart'] = event['dateStart'].dt event['dateEnd'] = (component.get('DTEND')) if event['dateEnd'] is not None: event['dateEnd'] = event['dateEnd'].dt else: event['dateEnd'] = event['dateStart'] if event['strSummary'] != 'None': event['details'] = { "description" : event['strDesc'], "Location" : event['strLocation'], } events.append(event) now_now = datetime.today().date() try: print("1Event: ", event['dateStart']) if event['dateStart'] > now_now: print("Check Check: ", event['dateStart']) createIcalEvent(event, scraper, venue, event_type) else: print("WHAT?") ppr(event) except Exception as e: try: event['dateStart'] = event['dateStart'].date() print("2Event: ", event['dateStart']) if event['dateStart'] > now_now: print("Check Check: ", event['dateStart']) createIcalEvent(event, scraper, venue, event_type) else: print("WHAT?") ppr(event) except Exception as e: print("The Error: ", e) pass