import os, sys from datetime import datetime from dateutil import relativedelta sys.path.append('/var/www/digisnaxx.ado/scrapers') import dtss dtss.getReady() from time import sleep from pprint import pprint as ppr from events.models import Organization, Scraper import events.digitools as digitools venue, created = Organization.objects.get_or_create( name="Comedy Corner", city="Minneapolis", website="https://comedycornerunderground.com/", ) scraper,item_count_start,virtcal = digitools.getScraper(venue, 'msp') DATETIME_FORMAT = '%m %d %Y %I:%M %p' def get_events(ps): sleep(3) contents = ps.xpath('.//*/div[@class="calendar-day"]') month = [] year = [] print("Contents: ", contents) for c in contents: try: day_num = c.xpath('.//*/div[@class="day-number"]/text()')[0] events = c.xpath('.//div[@class="event-item"]') # print(events) for e in events: event = {} event['scraper'] = scraper event['calendars'] = [scraper.calendar] event['title'] = e.xpath('.//*/div[@class="event-name"]/text()')[0] time = e.xpath('.//*/div[@class="event-name"]/text()')[0] event['link'] = e.xpath('.//a/@href')[0] event['date'] = str(day_num) + ' ' + event['time'] + 'm' dateStamp = month + ' ' + day_num + ' ' + year + time event['dateStamp'] = datetime.strptime(dateStamp, DATETIME_FORMAT) ppr(event) # digitools.createBasicEvent(event, 'Co') print('\n\n++\n\n') except Exception as e: print("\n\n",e,"\n\n") continue if len(sys.argv) >= 2: arg1 = sys.argv[1] br = digitools.getBrowser(arg1) else: print("No run_env") quit() ps = digitools.getSource(br, venue.website) br.execute_script("window.scrollTo(0, window.scrollY + 2500)") sleep(8) cal = ps.xpath('.//*/div[@id="ccu-calendar"]') print(cal) calendar = ps.xpath('.//*/div[@id="ccu-calendar"]/*') print(calendar) contents = ps.xpath('.//*/div[@id="ccu-calendar"]/*/div[@class="calendar-grid"]') print(contents) br.close() quit() get_events(ps) sleep(5) # ppr(events) br.close()