diff --git a/config/django/base.py b/config/django/base.py index 10bacb8..c55eb58 100644 --- a/config/django/base.py +++ b/config/django/base.py @@ -20,6 +20,7 @@ INSTALLED_APPS = [ 'socials', 'events', 'contracts', + 'datasnaxx', 'config', # 'academia_nuts', # 'leg_info', diff --git a/contracts/migrations/0015_contract_contract_value_int.py b/contracts/migrations/0015_contract_contract_value_int.py new file mode 100644 index 0000000..93e63ee --- /dev/null +++ b/contracts/migrations/0015_contract_contract_value_int.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.1 on 2026-04-05 02:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contracts', '0014_alter_paragraph_options'), + ] + + operations = [ + migrations.AddField( + model_name='contract', + name='contract_value_int', + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/contracts/migrations/0016_alter_contract_contract_value_int.py b/contracts/migrations/0016_alter_contract_contract_value_int.py new file mode 100644 index 0000000..7660b07 --- /dev/null +++ b/contracts/migrations/0016_alter_contract_contract_value_int.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.1 on 2026-04-05 02:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contracts', '0015_contract_contract_value_int'), + ] + + operations = [ + migrations.AlterField( + model_name='contract', + name='contract_value_int', + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/contracts/models.py b/contracts/models.py index e4b7add..24d81fc 100644 --- a/contracts/models.py +++ b/contracts/models.py @@ -42,17 +42,6 @@ class Tags(models.Model): return u'%s' % self.name -class Tags(models.Model): - name = models.CharField(max_length=31, unique=True) - desc = models.TextField(blank=True, null=True) - - def __unicode__(self): - return "%s" % self.name - - def __str__(self): - return u'%s' % self.name - - class Company(models.Model): name = models.CharField(max_length=63) @@ -120,6 +109,7 @@ class Contract(models.Model): award_num = models.CharField(max_length=31, blank=True, null=True) unq_entity_id = models.CharField(max_length=31, blank=True, null=True) contract_value = models.CharField(max_length=31, blank=True, null=True) + contract_value_int = models.FloatField(blank=True, null=True) orig_set_aside = models.CharField(max_length=127, blank=True, null=True) prod_svc_code = models.CharField(max_length=127, blank=True, null=True) naics_code = models.CharField(max_length=127, blank=True, null=True) diff --git a/datasnaxx/__init__.py b/datasnaxx/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datasnaxx/admin.py b/datasnaxx/admin.py new file mode 100644 index 0000000..0194190 --- /dev/null +++ b/datasnaxx/admin.py @@ -0,0 +1,15 @@ +from django.contrib import admin +from .models import * + +# class ContractAdmin(admin.ModelAdmin): +# # prepopulated_fields = {"slug": ("shortname",)} +# list_display = ("notice_id", "pub_date") + + + +# Register your models here. +admin.site.register(Subscriber) +admin.site.register(DigiStat) +admin.site.register(Platform) + + diff --git a/datasnaxx/apps.py b/datasnaxx/apps.py new file mode 100644 index 0000000..e8eabbe --- /dev/null +++ b/datasnaxx/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ContractsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'datasnaxx' diff --git a/datasnaxx/digitools.py b/datasnaxx/digitools.py new file mode 100644 index 0000000..f4a5d42 --- /dev/null +++ b/datasnaxx/digitools.py @@ -0,0 +1,70 @@ + + +import requests, sys # pip install requests +import jwt # pip install pyjwt +from datetime import datetime as date +import json +import os + +sys.path.append('/var/www/digisnaxx.ado/scrapers') +import dtss +dtss.getReady() + +from datasnaxx.models import Subscriber + +ADMIN_API = "66b2f3f778c05d6323155324:bd7d2034c1e0e52de8c549e8e5e3b8927032eacbe04647b6ae626fda4e08d821" +API_URL = "https://canin.dreamfreely.org" +JSON_PATH = "" + +key = ADMIN_API +id, secret = key.split(':') + +iat = int(date.now().timestamp()) + +header = {'alg': 'HS256', 'typ': 'JWT', 'kid': id} +payload = { + 'iat': iat, + 'exp': iat + 5 * 60, + 'aud': '/v4/admin/' +} + +token = jwt.encode(payload, bytes.fromhex(secret), + algorithm='HS256', headers=header) + +# Make an authenticated request to create a post + +url = API_URL + "/ghost/api/admin/members/" +headers = {'Authorization': 'Ghost {}'.format(token)} + +r = requests.get(url, headers=headers) + +data = json.loads(r.text) +subs = data['members'] + +for s in subs: + d = {} + d['email' ] = s['email'] + d['sub'] = s['subscribed'] + d['open_rate'] = s['email_open_rate'] + d['created_at'] = s['created_at'] + sub, created = Subscriber.objects.get_or_create( + email=d['email'], + subscribed=d['sub'], + created_at = d['created_at'], + open_rate = d['open_rate'], + ) + if len(s['labels']) > 0: + sub.label = s['labels'][0]['name'] + sub.save() + if len(s['subscriptions']) > 0: + sub_desc = s['subscriptions'][0] + sub.status = sub_desc['status'] + sub.start_date = sub_desc['start_date'] + sub.amount = sub_desc['price']['amount'] + sub.interval = sub_desc['price']['interval'] + sub.reoccuring = sub_desc['price']['type'] + sub.tier = sub_desc['tier']['name'] + sub.save() + + + diff --git a/datasnaxx/migrations/0001_initial.py b/datasnaxx/migrations/0001_initial.py new file mode 100644 index 0000000..949a25a --- /dev/null +++ b/datasnaxx/migrations/0001_initial.py @@ -0,0 +1,60 @@ +# Generated by Django 6.0.1 on 2026-04-05 20:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='DigiStat', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=15)), + ('stat', models.IntegerField()), + ], + options={ + 'verbose_name_plural': 'DigiStats', + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='Platform', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=15)), + ('description', models.TextField()), + ('mau', models.IntegerField()), + ('dau', models.IntegerField()), + ('engage_rate', models.FloatField()), + ('market_cap', models.IntegerField()), + ('stat', models.IntegerField()), + ], + ), + migrations.CreateModel( + name='Subscriber', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('email', models.CharField()), + ('subscribed', models.BooleanField(blank=True, null=True)), + ('created_at', models.DateTimeField()), + ('label', models.CharField(max_length=15)), + ('status', models.CharField(max_length=7)), + ('start_date', models.DateTimeField()), + ('amount', models.SmallIntegerField()), + ('interval', models.CharField(max_length=7)), + ('reoccuring', models.BooleanField()), + ('tier', models.CharField(max_length=15)), + ], + options={ + 'verbose_name_plural': 'Subscribers', + 'ordering': ['created_at'], + 'unique_together': {('email', 'created_at')}, + }, + ), + ] diff --git a/datasnaxx/migrations/0002_remove_platform_stat_alter_platform_dau_and_more.py b/datasnaxx/migrations/0002_remove_platform_stat_alter_platform_dau_and_more.py new file mode 100644 index 0000000..0594e53 --- /dev/null +++ b/datasnaxx/migrations/0002_remove_platform_stat_alter_platform_dau_and_more.py @@ -0,0 +1,42 @@ +# Generated by Django 6.0.1 on 2026-04-05 20:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('datasnaxx', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='platform', + name='stat', + ), + migrations.AlterField( + model_name='platform', + name='dau', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='platform', + name='description', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='platform', + name='engage_rate', + field=models.FloatField(blank=True, null=True), + ), + migrations.AlterField( + model_name='platform', + name='market_cap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='platform', + name='mau', + field=models.IntegerField(blank=True, null=True), + ), + ] diff --git a/datasnaxx/migrations/0003_platform_date_updated_subscriber_open_rate_and_more.py b/datasnaxx/migrations/0003_platform_date_updated_subscriber_open_rate_and_more.py new file mode 100644 index 0000000..a022796 --- /dev/null +++ b/datasnaxx/migrations/0003_platform_date_updated_subscriber_open_rate_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 6.0.1 on 2026-04-05 22:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('datasnaxx', '0002_remove_platform_stat_alter_platform_dau_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='platform', + name='date_updated', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='subscriber', + name='open_rate', + field=models.SmallIntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='subscriber', + name='amount', + field=models.SmallIntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='subscriber', + name='created_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='subscriber', + name='reoccuring', + field=models.CharField(max_length=15), + ), + migrations.AlterField( + model_name='subscriber', + name='start_date', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/datasnaxx/migrations/__init__.py b/datasnaxx/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datasnaxx/models.py b/datasnaxx/models.py new file mode 100644 index 0000000..e9a5525 --- /dev/null +++ b/datasnaxx/models.py @@ -0,0 +1,60 @@ +from django.db import models +from django.core.files.storage import FileSystemStorage +from django.contrib.auth.models import User + + +class Subscriber(models.Model): + email = models.CharField() + open_rate = models.SmallIntegerField(null=True, blank=True) + subscribed = models.BooleanField(null=True, blank=True) + created_at = models.DateTimeField(null=True, blank=True) + label = models.CharField(max_length=15) + status = models.CharField(max_length=7) + start_date = models.DateTimeField(null=True, blank=True) + amount = models.SmallIntegerField(null=True, blank=True) + interval = models.CharField(max_length=7) + reoccuring = models.CharField(max_length=15) + tier = models.CharField(max_length=15) + + + class Meta: + unique_together = ("email", "created_at") + verbose_name_plural = "Subscribers" + ordering = ['created_at'] + + def __unicode__(self): + return "%s" % self.email + + def __str__(self): + return u'%s' % self.email + + +class DigiStat(models.Model): + name = models.CharField(max_length=15) + stat = models.IntegerField() + + class Meta: + verbose_name_plural = "DigiStats" + ordering = ['name'] + + def __unicode__(self): + return "%s" % self.email + + def __str__(self): + return u'%s' % self.email + + +class Platform(models.Model): + name = models.CharField(max_length=15) + description = models.TextField(null=True, blank=True) + mau = models.IntegerField(null=True, blank=True) + dau = models.IntegerField(null=True, blank=True) + engage_rate = models.FloatField(null=True, blank=True) + market_cap = models.IntegerField(null=True, blank=True) + date_updated = models.DateTimeField(null=True, blank=True) + + def __unicode__(self): + return "%s" % self.name + + def __str__(self): + return u'%s' % self.name \ No newline at end of file diff --git a/datasnaxx/serializers.py b/datasnaxx/serializers.py new file mode 100644 index 0000000..0d41802 --- /dev/null +++ b/datasnaxx/serializers.py @@ -0,0 +1,27 @@ +from rest_framework import serializers +from .models import * + +from django.db import models +from django.contrib.auth.models import User +from rest_framework.permissions import BasePermission + +############ +## Events ## +############ + + +class CompanySerializer(serializers.ModelSerializer): + class Meta: + model = Company + # fields = ('id', 'name', 'website', 'city', 'latitude', 'longitude', 'has_map') + fields = '__all__' + + +class ContractSerializer(serializers.ModelSerializer): + company = CompanySerializer(many=False) + # target_language = serializers.SerializerMethodField() + class Meta: + model = Contract + fields = '__all__' + depth = 2 +# fields = ('id', 'name',) \ No newline at end of file diff --git a/datasnaxx/tests.py b/datasnaxx/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/datasnaxx/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/datasnaxx/urls.py b/datasnaxx/urls.py new file mode 100644 index 0000000..af80173 --- /dev/null +++ b/datasnaxx/urls.py @@ -0,0 +1,26 @@ +"""ds_events URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.1/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path, include, re_path +from .views import * + +urlpatterns = [ + re_path(r'^contracts/', ContractAPIView.as_view(), name="get-contracts-limit"), + re_path(r'^contracts-all/', ContractAllAPIView.as_view(), name="get-contracts"), + re_path(r'^companies/', CompanyAPIView.as_view(), name="get-companies"), + # re_path(r'^events-token/', EventsTokenAPIView.as_view(), name="get-token-events"), + +] diff --git a/datasnaxx/views.py b/datasnaxx/views.py new file mode 100644 index 0000000..60cf381 --- /dev/null +++ b/datasnaxx/views.py @@ -0,0 +1,64 @@ +from django.shortcuts import render +from datetime import datetime, timedelta +import pytz, random + +from .models import * +from .serializers import * + +from django.db.models import Q +from django.db.models import Count + +from rest_framework import generics +from rest_framework.decorators import authentication_classes, permission_classes +from rest_framework.authentication import SessionAuthentication, BasicAuthentication + +from rest_framework.permissions import IsAuthenticated +# from durin.auth import TokenAuthentication + +# from durin.views import APIAccessTokenView + +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import filters + +from rest_framework.response import Response +from rest_framework_api_key.permissions import HasAPIKey + +td = timedelta(hours=7) +odt = datetime.now() - td + +# Create your views here. + +class ContractAPIView(generics.ListAPIView): + serializer_class = ContractSerializer + queryset = Contract.objects.all()[:25] + filter_backends = [DjangoFilterBackend, filters.SearchFilter] + filterset_fields = ['id',] + permission_classes = [HasAPIKey] + + +class ContractAllAPIView(generics.ListAPIView): + serializer_class = ContractSerializer + queryset = Contract.objects.all() + filter_backends = [DjangoFilterBackend, filters.SearchFilter] + search_fields = ['notice_id', 'original_contract_number', 'title', 'description', 'company__name', 'us_dept', 'us_dept_sub_tier', 'us_office', 'naics_code', 'prod_svc_code'] + filterset_fields = ['id',] + permission_classes = [HasAPIKey] + + +class CompanyAPIView(generics.ListAPIView): + serializer_class = CompanySerializer + queryset = Company.objects.all() + permission_classes = [HasAPIKey] + +# class PromoAPIView(generics.ListAPIView): +# serializer_class = PromoSerializer +# queryset = Promo.objects.filter(published=True) +# filterset_fields = ['organization__name', 'calendar__shortcode',] +# search_fields = ['organization__name', 'calendar__shortcode',] +# # permission_classes = [HasAPIKey] + +# def get_queryset(self): +# calendar = self.request.GET.get('calendar__shortcode') +# queryset = Promo.objects.filter(published=True, calendar__shortcode=calendar).order_by('?') +# return queryset + diff --git a/events/migrations/0051_alter_event_calendar_alter_promo_calendar.py b/events/migrations/0051_alter_event_calendar_alter_promo_calendar.py new file mode 100644 index 0000000..38cf5cb --- /dev/null +++ b/events/migrations/0051_alter_event_calendar_alter_promo_calendar.py @@ -0,0 +1,23 @@ +# Generated by Django 6.0.1 on 2026-04-05 20:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('events', '0050_official_bluesky_official_email_official_instagram_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='event', + name='calendar', + field=models.ManyToManyField(blank=True, related_name='events', to='events.calendar'), + ), + migrations.AlterField( + model_name='promo', + name='calendar', + field=models.ManyToManyField(blank=True, to='events.calendar'), + ), + ] diff --git a/events/models.py b/events/models.py index 62ddd10..ab5d881 100644 --- a/events/models.py +++ b/events/models.py @@ -35,7 +35,7 @@ class Scraper(models.Model): class Meta: verbose_name_plural = "Scrapers" - ordering = ['name',] + ordering = ['-last_ran', 'name',] def __unicode__(self): return "%s" % self.name @@ -192,7 +192,7 @@ class Event(models.Model): show_link = models.URLField(blank=True, null=True) show_date = models.DateTimeField() - calendar = models.ManyToManyField(Calendar, blank=True, null=True, related_name="events") + calendar = models.ManyToManyField(Calendar, blank=True, related_name="events") scraper = models.ForeignKey(Scraper, on_delete=models.CASCADE, null=True) venue = models.ForeignKey(Organization, on_delete=models.CASCADE) @@ -236,7 +236,7 @@ class Promo(models.Model): title = models.CharField(max_length=63) organization = models.ForeignKey(Organization, on_delete=models.CASCADE) - calendar = models.ManyToManyField(Calendar, blank=True, null=True) + calendar = models.ManyToManyField(Calendar, blank=True) promo_type = models.CharField(max_length=15, choices=PROMO_TYPE, default='Ar') promo_image = models.ImageField(upload_to="promo", blank=True) short_text = models.CharField(max_length=255,blank=True, null=True)