import os, sys from datetime import datetime, timedelta from dateutil import relativedelta sys.path.append('/var/www/digisnaxx.ado/scrapers') import dtss dtss.getReady() from time import sleep from pprint import pprint as ppr import pytz from selenium.webdriver.common.by import By from events.models import Organization, Scraper, Calendar, Event import events.digitools as digitools tz_str = "+0100 UTC" DATETIME_FORMAT = '%b %d %Y %H:%M %z %Z' venue, created = Organization.objects.get_or_create( name="Flex", city="Vienna", website="https://flex.at/", is_venue = True ) scraper,item_count_start,virtcal = digitools.getScraper(venue, venue.website, 'vie') scraper.items = 0 scraper.save() def getSite(br, website): ps = digitools.getSource(br, website) get_events(ps, "Mu") sleep(3) def get_events(ps, event_type): contents = ps.xpath('.//*/div[@class="ectbe-inner-wrapper ectbe-simple-event "]') for c in contents: try: event = {} date = c.xpath('.//div[@class="ectbe-date-wrp elementor-repeater-item-dd81c19"]/span/text()') time = c.xpath('.//*/div[@class="ectbe-evt-time elementor-repeater-item-fd61cbc"]/text()')[0].split('-')[0].strip() year = datetime.now().year if date[1] == 'März': date[1] = "Mar" event['date'] = (' ').join([date[1].replace(".", ""), date[0], str(year), time, tz_str]) event['scraper'] = scraper event['calendars'] = [scraper.calendar] event['title'] = c.xpath('.//*/h2/text()')[0] try: event['dateStamp'] = datetime.strptime(event['date'], DATETIME_FORMAT) except Exception as e: print(e) pass event['link'] = c.xpath('.//a[@class="ectbe-evt-read-more"]/@href')[0] digitools.createBasicEvent(event, "Mu", venue) scraper.items+=1 # ppr(event) except Exception as e: print("\nError: ", e) ppr(event) # print("\n+++\n") pass if len(sys.argv) >= 2: arg1 = sys.argv[1] br = digitools.getBrowser(arg1) else: print("No run_env") br.close() quit() getSite(br, venue.website) digitools.updateScraper(scraper, item_count_start) br.close()