import ApiSCSE
import sys
import pprint
import os
import time

apiscse = ApiSCSE.ApiSCSE()
API_USER = 'USER_EMAIL'
API_KEY = 'API_credentials'
COUNTRY = 'Default'
KEYWORD_FILE = 'fortune-500.txt'
RESULTS_PER_KEYWORD = 100
LANGUAGE = 'English'
PRIORITY = 10
JOBNAME = 'Demo Job'
MODE = 'cli'

# Initialize API
if MODE == 'http':
    print('<pre>')
res = apiscse.initi(API_USER, API_KEY)
if not res:
    print("Error during API initialization:")
    print(apiscse.GetError())
    sys.exit(1)
else:
    print("Initialized")

# Get license and credits
license = apiscse.getLicense()
credits = apiscse.getCredits()

# Get job statuses
jobs = apiscse.getJobs()
jobs_unfinished = 0
jobs_finished = 0
jobs_running = 0

for job in jobs:
    if int(job['progress']) == 100:
        jobs_finished += 1
    elif int(job['start']) == 1:
        jobs_running += 1
    else:
        jobs_unfinished += 1

print(f'SCSE license active: {license["name"]}')
print(f'SCSE wallet: {credits["credit"]} Credits')
print(f'Jobs: {jobs_unfinished} unfinished jobs, {jobs_running} running jobs, {jobs_finished} finished jobs')

if float(credits['credit']) < 0.6:
    print('Insufficient Credits in wallet\n')
    sys.exit(1)

# Get or create job
job = apiscse.getJob(JOBNAME)
print(job)
if not job:
    print(f"Creating job {JOBNAME}: ")
    job = apiscse.createJob(JOBNAME, LANGUAGE, COUNTRY, RESULTS_PER_KEYWORD, PRIORITY)
    if job:
        print("success\n")
    else:
        print("failure\n")
        if apiscse.getLastResponse().has_exception:
            pprint.pprint(apiscse.getLastResponse().exceptions)
        sys.exit(1)

# Check if job needs to be started
if int(job['start']) == 0:
    if not os.path.exists(KEYWORD_FILE):
        print(f'File {KEYWORD_FILE} does not exist\n')
        sys.exit(1)

    with open(KEYWORD_FILE, 'r') as file:
        keywords = [line.strip() for line in file]
    res = apiscse.modify_job(JOBNAME, keywords, apiscse.MODE_REPLACE, LANGUAGE, COUNTRY, RESULTS_PER_KEYWORD, PRIORITY)
    if not res:
        print('Modification of Job failed\n')
        if apiscse.getLastResponse().has_exception:
            pprint.pprint(apiscse.getLastResponse().exceptions)
        sys.exit(0)

    print(f'Job starting cost: {res["credit_cost"]} Credits\n')

    res = apiscse.start_job(JOBNAME)
    if not res:
        started = False
        for exception in apiscse.getLastResponse().exceptions:
            if exception['error_id'] == 10008:
                started = True
                break
        if not started:
            print(f'Job {JOBNAME} could not be started\n')
            sys.exit(1)
        print(f'Job {JOBNAME} was already started\n')
    else:
        print(f'Job {JOBNAME} has been started\n')

# Monitor job progress
latest_page = 0
job = apiscse.getJob(JOBNAME)
original_estimated_hours = float(job['estimated_hours_left'])
start_time = time.time()
end_time = start_time + original_estimated_hours * 3600 * 3
once = 0
total_organic = 0
total_creative = 0

while not once or time.time() < end_time:
    print(f"Waiting for results, estimated time left: {float(job['estimated_hours_left']):.2f} hours ; Progress: {float(job['progress']):.2f}..\n")
    if MODE == 'http':
        sys.stdout.flush()
    time.sleep(5)
    job = apiscse.getJob(JOBNAME)
    total_pages = int(job['total_pages'])
    result_pages = int(job['pages'])

    while result_pages > latest_page:
        print(f'Downloading new result page {latest_page + 1}\n')
        results = apiscse.getResults(JOBNAME, latest_page + 1)
        for key in results:
            print(key)
        for result in results['keyword_results']:
            print(f'Keyword: {result["keyword"]} - Organic results: {result["count_organic"]}, Creative results: {result["count_creative"]}\n')
            total_organic += result['count_organic']
            total_creative += result['count_creative']
            print('Organic results:\n')
            for list1 in result['results_organic']:
                print(f'\tURL: {list1["url"]}')
                print(f'\tTitle: {list1["title"]}')
                print(f'\tDescription: {list1["description"]}')
                if list1['sitelinks']:
                    for sitelink in list1['sitelinks']:
                        print(f'\t\tURL: {sitelink["url"]}')
                        print(f'\t\tTitle: {sitelink["title"]}')
                        print(f'\t\tDescription: {sitelink["description"]}')
                print("\n")
            try:
                if result['creative'] > 0:
                    print('Advertisements:')
                    for list1 in result['results_creative']:
                        print(f'\tURL: {list1["url"]}')
                        print(f'\tTitle: {list1["title"]}')
                        print(f'\tDescription: {list1["description"]}')
                        if list1['sitelinks']:
                            for sitelink in list1['sitelinks']:
                                print(f'\t\tURL: {sitelink["url"]}')
                                print(f'\t\tTitle: {sitelink["title"]}')
                                print(f'\t\tDescription: {sitelink["description"]}')
                        print('\n')
            except KeyError:
                pass
        latest_page += 1
        if total_pages == latest_page:
            break
    once += 1

hours_passed = round((time.time() - start_time) / 3600, 2)
print('\n')
print(f'Scraping of {latest_page} pages finished successfully\n')
print(f'Received {total_organic} organic results and {total_creative} ads\n')
print(f'Estimated time: {original_estimated_hours} hours, Real time: {hours_passed} hours\n')
if hours_passed != 0.00 and time.time() >= end_time:
    print('ERROR: Timeout while waiting for results\n')