DigiSnaxx/api.digisnaxx
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scraper updates

Browse Source
This commit is contained in:
canincarlos
2025-10-11 10:58:35 -05:00
parent 0e686c91e4
commit aef63f1a08
13 changed files with 157 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
*/__pycache__
*/*/__pycache__

BIN
db.sqlite3
View File

Binary file not shown.

2
ds_events/settings.py
View File

@@ -49,7 +49,7 @@ INSTALLED_APPS = [
'rest_framework.authtoken',
'socials',
'events',
'academia_nuts',
# 'academia_nuts',
# 'leg_info',
]

4
events/admin.py
View File

@@ -13,8 +13,8 @@ class OrganizationAdmin(admin.ModelAdmin):
class EventAdmin(admin.ModelAdmin):
# prepopulated_fields = {"slug": ("shortname",)}
list_display = ( "show_title", "event_type", "show_date",)
list_filter = ("venue", "event_type")
list_display = ( "show_title", "event_type", "show_date", "calendar")
list_filter = ("calendar", "venue", "event_type",)
class PromoAdmin(admin.ModelAdmin):
# prepopulated_fields = {"slug": ("shortname",)}

45
events/digitools.py
View File

@@ -197,7 +197,29 @@ def createBasicArticle(article, event_type, organization):
)
return new_article, created
def getiCalEvents(gcal, scraper):
def createIcalEvent(event, scraper, venue, event_type):
new_date = event['dateStart']
new_event = {}
new_event['scraper'] = scraper
new_event['calendar'] = scraper.calendar
new_event['title'] = event['strSummary'],
new_event['date'] = str(new_date)[:-6],
new_event['dateStamp'] = str(new_date)[:-6],
new_event['link'] = venue.website
createBasiciCalEvent(new_event, event_type, venue)
def createCleanIcalEvent(event, scraper, venue, event_type):
new_date = event['dateStart']
new_event = {}
new_event['scraper'] = scraper
new_event['calendar'] = scraper.calendar
new_event['title'] = event['strSummary'],
new_event['date'] = str(new_date),
new_event['dateStamp'] = str(new_date),
new_event['link'] = venue.website
createBasiciCalEvent(new_event, event_type, venue)
def getiCalEvents(gcal, scraper, venue):
for component in gcal.walk():
event = {}
event['scraper'] = scraper
@@ -229,30 +251,13 @@ def getiCalEvents(gcal, scraper):
try:
print("1Event: ", event['dateStart'])
if event['dateStart'] > now_now:
new_date = event['dateStart']-td
new_event = {}
new_event['scraper'] = scraper
new_event['calendar'] = scraper.calendar
new_event['title'] = event['strSummary'],
new_event['date'] = str(new_date),
new_event['dateStamp'] = str(new_date),
new_event['link'] = venue.website
createBasiciCalEvent(new_event, "Mu", venue)
createIcalEvent(event, scraper, venue)
except Exception as e:
try:
event['dateStart'] = event['dateStart'].date()
print("1Event: ", event['dateStart'])
if event['dateStart'] > now_now:
new_date = event['dateStart']-td
print("The new Date: ", new_date, type(new_date))
new_event = {}
new_event['scraper'] = scraper
new_event['calendar'] = scraper.calendar
new_event['title'] = event['strSummary'],
new_event['date'] = new_date,
new_event['dateStamp'] = new_date,
new_event['link'] = venue.website
createBasiciCalEvent(new_event, "Mu", venue)
createIcalEvent(event, scraper, venue)
except Exception as e:
print("The Error: ", e)
pass

2
events/scrapers/Working/iCal/ical.CAFAC.mpls.py
View File

@@ -46,5 +46,5 @@ event_type = "Ed"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
digitools.getiCalEvents(gcal, scraper)
digitools.getiCalEvents(gcal, scraper, venue)
digitools.updateScraper(scraper, item_count_start)

118
events/scrapers/Working/iCal/ical_ingles.py Normal file
View File

@@ -0,0 +1,118 @@
import requests, os, sys
from icalendar import Calendar as iCalendar, Event
from datetime import datetime, timedelta
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
from pprint import pprint as ppr
import pytz
import django
sys.path.append('../../../../')
os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
venue, created = Organization.objects.get_or_create(
name="idioki",
city="Medellin",
website="https://idioki.com/",
)
event_type = "Mu"
try:
scraper, created = Scraper.objects.get_or_create(
name="idioki",
website="https://calendar.google.com/calendar/ical/46ae0446724b1b3ee83cbd7dbc0db6a235bf97509ad860ca91eada3c267b5e41%40group.calendar.google.com/public/basic.ics",
calendar = Calendar.objects.get(shortcode='mde'),
items = 0,
new_items = 0,
last_ran = datetime.now(),
)
except Exception as e:
print(e)
scraper = Scraper.objects.get(name=venue.name)
item_count_start = scraper.items
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
def createEvent(day, date, event, scraper, venue, event_type):
days = [day-1, day+6, day+13]
for day in days:
event['dateStamp'] = date + timedelta(days=day)
event['dateStart'] = event['dateStamp']
digitools.createCleanIcalEvent(event, scraper, venue, event_type)
return
def splitLocation(event):
loc_split = event['strLocation'].split(',')
venue_name = loc_split[0]
venue, created = Organization.objects.get_or_create(
name=venue_name,
city="Medellin",
website="https://idioki.com/",
)
event['venue'] = venue
return event
counter = 0
for component in gcal.walk():
event = {}
event['scraper'] = scraper
event['calendar'] = scraper.calendar
event['strSummary'] = f"{(component.get('SUMMARY'))}"
event['strDesc'] = component.get('DESCRIPTION')
event['strLocation'] = str(component.get('LOCATION'))
# startDate = component.get('DTSTART')
# startTime = startDate.time()
event['dateStart'] = component.get('DTSTART')
event['dateStamp'] = component.get('DTSTAMP')
if event['strSummary'] != 'None':
event['details'] = {
"description" : event['strDesc'],
"Location" : event['strLocation'],
}
if event['dateStamp'] != None:
event['dateStart'] = event['dateStart'].dt
event['dateStart'] = datetime.strptime(str(event['dateStart'])[:-6], '%Y-%m-%d %H:%M:%S')
rules = component.get('RRule')
if rules['FREQ'][0] == 'WEEKLY':
if datetime.today().weekday() != 0:
event = splitLocation(event)
date = datetime.today().date() - timedelta(days=datetime.today().weekday())
date = datetime.combine(date, event['dateStart'].time())
days = ["SU", "MO", "TU", "WE", "TH", "FR", "SA"]
for day in rules['BYDAY']:
day = days.index(day)
loc_split = event['strLocation'].split(",")
venue, created = Organization.objects.get_or_create(
name=loc_split[0],
city="Medellin",
website="https://idioki.com/",
)
createEvent(day, date, event, scraper, venue, "Ed")
digitools.updateScraper(scraper, item_count_start)
new_time = datetime.now() - timedelta(days=1)
right_bound_time = datetime.now() + timedelta(days=45)
events = DSEvent.objects.filter(show_date__lte=new_time)
events1 = DSEvent.objects.filter(show_date__gte=right_bound_time)
for e in events:
e.delete()
for e in events1:
e.delete()

4
events/scrapers/Working/iCal/ical_run.KJHideaway.StPaul.py
View File

@@ -11,6 +11,8 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'ds_events.settings'
django.setup()
from events.models import Event as DSEvent, Organization, Scraper, Calendar
import events.digitools as digitools
from dateutil import relativedelta
td = relativedelta.relativedelta(hours=5)
@@ -40,5 +42,5 @@ event_type = "Mu"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
digitools.getiCalEvents(gcal, scraper)
digitools.getiCalEvents(gcal, scraper, venue)
digitools.updateScraper(scraper, item_count_start)

2
events/scrapers/Working/iCal/ical_run.SocialableCider.mpls.py
View File

@@ -44,5 +44,5 @@ item_count_start = scraper.items
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
digitools.getiCalEvents(gcal, scraper)
digitools.getiCalEvents(gcal, scraper, venue)
digitools.updateScraper(scraper, item_count_start)

2
events/scrapers/Working/iCal/ical_run.bunkers.py
View File

@@ -44,5 +44,5 @@ event_type = "Mu"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
digitools.getiCalEvents(gcal, scraper)
digitools.getiCalEvents(gcal, scraper, venue)
digitools.updateScraper(scraper, item_count_start)

2
events/scrapers/Working/iCal/ical_run.cfpa.py
View File

@@ -44,5 +44,5 @@ event_type = "Ed"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
digitools.getiCalEvents(gcal, scraper)
digitools.getiCalEvents(gcal, scraper, venue)
digitools.updateScraper(scraper, item_count_start)

2
events/scrapers/Working/iCal/ical_run.eagles.py
View File

@@ -42,5 +42,5 @@ event_type = "Mu"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
digitools.getiCalEvents(gcal, scraper)
digitools.getiCalEvents(gcal, scraper, venue)
digitools.updateScraper(scraper, item_count_start)

2
events/scrapers/Working/iCal/ical_run.terminalbar-mpls.py
View File

@@ -46,5 +46,5 @@ event_type = "Mu"
objIcalData = requests.get(scraper.website)
gcal = iCalendar.from_ical(objIcalData.text)
tz = pytz.timezone("US/Central")
digitools.getiCalEvents(gcal, scraper)
digitools.getiCalEvents(gcal, scraper, venue)
digitools.updateScraper(scraper, item_count_start)