import os, sys
import json
import re
from datetime import datetime

sys.path.append('/var/www/digisnaxx.ado/scrapers')
import dtss
dtss.getReady()

from time import sleep
from pprint import pprint as ppr
import pytz

from selenium import webdriver as wd
from lxml import html

from contracts.models import Contract, Company, Paragraph, OriginalContract


print("\n+++++\n+++++\n+++++\nStarting Scrape\n+++++\n+++++\n+++++\n")

tz_str = " -0600 UTC"
DFORMAT = "%b %d, %Y %H:%M %p %z %Z"
D2FORMAT = "%b %d, %Y %z %Z"
D3FORMAT = "%b %d %Y %z %Z"

SAM_BASE = "https://sam.gov"
SAMLINK = "https://sam.gov/search/?index=opp&page=1&pageSize=25&sort=-modifiedDate&sfm%5Bstatus%5D%5Bis_active%5D=true&sfm%5Bstatus%5D%5Bis_inactive%5D=true&sfm%5BsimpleSearch%5D%5BkeywordRadio%5D=ALL&sfm%5BsimpleSearch%5D%5BkeywordTags%5D%5B0%5D%5Bvalue%5D="


base_site = "https://www.war.gov/News/Contracts/"


br = wd.Chrome()
br.get(base_site)
ps = html.fromstring(br.page_source)
# links = ps.xpath('.//*/p[@class="title"]/a/@href')
page_links = []

for site in range(2,3):
    nsite = base_site + "?Page=" + str(site)
    br.get(nsite)
    sleep(2)
    ps = html.fromstring(br.page_source)
    page_links = page_links + ps.xpath('.//*/p[@class="title"]/a/@href')


# for site in range(10,25):
#     nsite = base_site + "?Page=" + str(site)
#     br.get(nsite)
#     sleep(2)
#     ps = html.fromstring(br.page_source)
#     page_links = page_links + ps.xpath('.//*/p[@class="title"]/a/@href')


data = []
base_data = []

for link in page_links:
    br.get(link)
    ps = html.fromstring(br.page_source)
    paras = ps.xpath('.//*/p/text()')
    # print(len(paras), link)
    for para in paras:
        nlink = " ".join(link.split("-")[-3:])[:-1] + tz_str
        new_paragraph, created = Paragraph.objects.get_or_create(
            link = link,
            paragraph=para,
            date = datetime.strptime(nlink, D3FORMAT),
            )
        print(created, new_paragraph)

        contracts = re.findall(r'[A-Z0-9-]{12,}', para)
        if len(contracts) > 0:
            # print(contracts)
            for contract in contracts:
                i = {}
                i['para'] = new_paragraph.id
                i['contract'] = contract.replace("-", "")
                i['website'] = link
                new_contract, created = OriginalContract.objects.get_or_create(
                    para = new_paragraph,
                    number= i['contract'],
                    )
                print(created, new_contract)
                data.append(i)


with open('contract_list.json', 'w') as fp:
    json.dump(data, fp)


for d in data:
    website = SAMLINK + d['contract']
    print("\nGetting Website: ", website)
    br.get(website)
    sleep(2)
    links = []
    ps = html.fromstring(br.page_source)
    links = ps.xpath('.//*/h3/a[@class="usa-link ng-star-inserted"]/@href')
    for link in links:
        print("\n++++++++\n", d['website'], "\n++++++++++\n")
        br.get(SAM_BASE + link)
        print("\nContract link: ", SAM_BASE + link)
        sleep(4)
        ps = html.fromstring(br.page_source)
        i = {}
        
        
        i['og_contract'] = OriginalContract.objects.get(number=d['contract'], para__id=d['para'])
        try:
            i['pub_date_txt'] = ps.xpath('.//*[@id="solicitation"]/div/div/div[3]/div[4]/div[4]/div/h5/text()')[0].strip()
        except:
            i['pub_date_txt'] = "missed"
        try:
            i['title'] = ps.xpath('.//*/h1[@class="card-title"]/text()')[0].strip()
        except:
            i['title'] = ps.xpath('.//*/h1[@class="card-title"]/text()')
        try:
            i['notice_id'] = ps.xpath('.//*[@id="solicitation"]/div/div/div[3]/div[1]/div[2]/div/h5/text()')[0].strip()
        except:
            i['notice_id'] = "missed"

        contract, created = Contract.objects.get_or_create(
            title=i["title"],
            original_contract_number = i["og_contract"],
            notice_id=i["notice_id"],
            pub_date_txt = i["pub_date_txt"],
            contract_url = br.current_url
            )
        print('CONTRACT SUCCESS', created, contract)

        contract.inactive_date = ps.xpath('.//*[@id="solicitation"]/div/div/div[3]/div[3]/div[2]/div/h5/text()')[0].strip()
        contract.inactive_policy = ps.xpath('.//*[@id="solicitation"]/div/div/div[3]/div[3]/div[4]/div/h5/text()')[0].strip()
        # contract.pub_date_txt = ps.xpath('.//*[@id="solicitation"]/div/div/div[3]/div[4]/div[4]/div/h5/text()')[0].strip()
        contract.resp_date = ps.xpath('.//*[@id="solicitation"]/div/div/div[3]/div[4]/div[2]/div/h5/text()')[0].strip()
        contract.us_dept = ps.xpath('.//*[@id="solicitation"]/div/div/div[4]/div/div[1]/div[1]/div[2]/h5/text()')[0].strip()
        contract.us_dept_subtier = ps.xpath('.//*[@id="solicitation"]/div/div/div[4]/div/div[1]/div[2]/div[2]/h5/text()')[0].strip()
        contract.related_notice_id = ps.xpath('.//*[@id="solicitation"]/div/div/div[3]/div[1]/div[4]/div/h5/text()')[0].strip()
        contract.opp_type = ps.xpath('.//*[@id="solicitation"]/div/div/div[3]/div[2]/div[2]/div/h5/text()')[0].strip()
        contract.orig_set = ps.xpath('.//*[@id="class"]/div[2]/div[1]/div[2]/h5/text()')[0].strip()
        contract.prod_svc_code = ps.xpath('.//*[@id="class"]/div[2]/div[2]/div[2]/div/h5/text()')[0].strip()
        contract.naics_code = ps.xpath('.//*[@id="class"]/div[2]/div[2]/div[4]/div/h5/text()')[0].strip()
        contract.save()


        try:
            contract.us_office = ps.xpath('.//*[@id="solicitation"]/div/div/div[4]/div/div[2]/div/div[2]/h5/text()')[0].strip()
            contract.save()
        except:
            contract.us_office = "naan"
            contract.save()
        try:
            contract.line_num = ps.xpath('.//*[@id="solicitation"]/div/div/div[3]/div[2]/div[4]/div/h5/text()')[0].strip()
            contract.save()
        except:
            continue
        try:
            contract.major_cmd = ps.xpath('.//*[@id="solicitation"]/div/div/div[4]/div/div[1]/div[3]/div[2]/h5/text()')[0].strip()
            contract.save()
        except:
            contract.major_cmd = "naan"
            contract.save()
        try:
            i['awarded_name'] = ps.xpath('.//*[@id="award-details"]/div[2]/div/div[4]/div[2]/div/h5/text()')[0].strip()
        except:
            i['awarded_name'] = "naan"
        try:
            i['award_date'] = ps.xpath('.//*[@id="award-details"]/div[2]/div/div[1]/div[2]/div/h5/text()')[0].strip()
            contract.save()
        except:
            i['award_date'] = "naan"
            contract.save()
        try:
            contract.award_num = ps.xpath('.//*[@id="award-details"]/div[2]/div/div[2]/div[2]/div/h5/text()')[0].strip()
            contract.save()
        except:
            contract.award_num = "naan"
            contract.save()
        try:
            i['unq_entity_id'] = ps.xpath('.//*[@id="award-details"]/div[2]/div/div[3]/div[3]/div/h5/text()')[0].strip()
            contract.unq_entity_id = ps.xpath('.//*[@id="award-details"]/div[2]/div/div[3]/div[3]/div/h5/text()')[0].strip()
            contract.save()
        except:
            contract.unq_entity_id = "naan"
            contract.save()
        try:
            i['awarded_addr'] = ps.xpath('.//*[@id="award-details"]/div[2]/div/div[2]/div[3]/div[2]/h5/text()')[0].strip()
        except:
            i['awarded_addr'] = "naan"
        try:
            contract.contract_value = ps.xpath('.//*[@id="award-details"]/div[2]/div/div[3]/div[4]/div[3]/div/h5/text()')[0].strip()
            contract.save()
        except:
            contract.contract_value = "naan"
            contract.save()
        try:
            contract.description = ps.xpath('.//*[@id="desc"]/div[2]/div/div/p/text()')[0].strip()
            contract.save()
        except:
            contract.description = "naan"
            contract.save()

        base_data.append(i)

        print("\nappended info: ", i['title'])
        try:
            new_company, created = Company.objects.get_or_create(
                name = i["awarded_name"],
                address_complete=i["awarded_addr"],
                unq_entity_id = i["unq_entity_id"],
                )
            print("COMPANY CREATED: ", created, new_company)
            contract.company = new_company
            contract.save()
        except Exception as e:
            print("\nCompany error: ", e)
        try:
            if contract.pub_date_txt.split(" ")[-1:][0].strip() == "EST":
                print("TZ: ", contract.pub_date_txt.split(" ")[-1:][0].strip())
                ndate = " ".join(contract.pub_date_txt.split(" ")[:-1]) + " -0500 UTC"
                contract.pub_date = datetime.strptime(ndate, DFORMAT)
                contract.save()
            elif contract.pub_date_txt.split(" ")[-1:][0].strip() == "EDT":
                ndate = " ".join(contract.pub_date_txt.split(" ")[:-1]) + " -0500 UTC"
                contract.pub_date = datetime.strptime(ndate, DFORMAT)
                contract.save()
            elif contract.pub_date_txt.split(" ")[-1:][0].strip() == "CST":
                ndate = " ".join(contract.pub_date_txt.split(" ")[:-1]) + " -0600 UTC"
                contract.pub_date = datetime.strptime(ndate, DFORMAT)
                contract.save()
            elif contract.pub_date_txt.split(" ")[-1:][0].strip() == "CDT":
                ndate = " ".join(contract.pub_date_txt.split(" ")[:-1]) + " -0600 UTC"
                contract.pub_date = datetime.strptime(ndate, DFORMAT)
                contract.save()
            elif contract.pub_date_txt.split(" ")[-1:][0].strip() == "MST":
                ndate = " ".join(contract.pub_date_txt.split(" ")[:-1]) + " -0700 UTC"
                contract.pub_date = datetime.strptime(ndate, DFORMAT)
                contract.save()
            elif contract.pub_date_txt.split(" ")[-1:][0].strip() == "PST":
                ndate = " ".join(contract.pub_date_txt.split(" ")[:-1]) + " -0800 UTC"
                contract.pub_date = datetime.strptime(ndate, DFORMAT)
                contract.save()
        except Exception as e:
            print("Date Error: ", e)
            pass
        # try:
        #     contract.major_cmd = i["major_cmd"],
        #     contract.save()
        # except Exception as e:
        #     print("Major Cmd Error: ", e)
        #     pass
        try:
            time = " -0500 UTC"
            contract.award_date = datetime.strptime(i["award_date"] + time, D2FORMAT)
            contract.save()
        except Exception as e:
            print("Award Date Error: ", e)
            pass
        # try:
        #     contract.award_num = i["award_num"]
        #     contract.save()
        # except Exception as e:
        #     print("Award Number Error: ", e)
        #     pass
        # try:
        #     contract.contract_value = i["contract_value"]
        #     contract.save()
        # except Exception as e:
        #     print("Major Cmd Error: ", e)
        #     pass
        # try:
        #     contract.unq_entity_id = i["unq_entity_id"]
        #     contract.save()
        # except Exception as e:
        #     print("Major Cmd Error: ", e)
        #     pass
            

with open('final_results.json', 'w') as fp:
    json.dump(base_data, fp)