HEX
Server: Apache/2.4.52 (Ubuntu)
System: Linux spn-python 5.15.0-89-generic #99-Ubuntu SMP Mon Oct 30 20:42:41 UTC 2023 x86_64
User: arjun (1000)
PHP: 8.1.2-1ubuntu2.20
Disabled: NONE
Upload Files
File: //home/arjun/projects/buyercall_new/buyercall/buyercall/blueprints/form_leads/views.py
import logging as log
import traceback
from urllib.parse import quote_plus, urlparse, urlsplit
from contextlib import closing
from io import StringIO
import csv
from docx import Document
from operator import itemgetter
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
import datetime
from datetime import date
from flask_cors import cross_origin
from buyercall.lib.flask_mailplus import _try_renderer_template
import pytz
import redis
from flask import (
    Blueprint,
    jsonify,
    json,
    make_response,
    render_template,
    request,
    redirect,
    flash,
    url_for,
    send_file,
    current_app as app
)
from flask_login import login_required, current_user
from flask_babel import gettext as _
from buyercall.lib.util_dms import AmsLeadXml, ams_client, neo_client, NeoLead
from sqlalchemy import or_, and_, desc
from sqlalchemy.orm import load_only, joinedload
from sqlalchemy.sql.expression import func, case
from sqlalchemy.sql import text
from buyercall.extensions import db, csrf
from buyercall.lib.util_twilio import subaccount_client
from buyercall.lib.util_bandwidth import bw_client
from buyercall.lib.util_crypto import AESCipher
from buyercall.blueprints.user.decorators import role_required
from .models import (
    ExternalForm as Form, FormLead as Lead, FormLeadField as LeadField,
    ExternalFormField as FormField, FormLog, ExternalFormFieldDefinition
)
from buyercall.blueprints.activity.models import ActivityType, ActivityName, ActivityLogs
from ..widgets.models import Widget
from ..agents.models import Agent
from ..widgets.routing import (
    add_widget_lead, Routing, BandwidthRouting
)
import requests
from hashlib import sha1
import hmac
import re
from ..filters import isfloat, isint
import base64
import uuid

form_leads = Blueprint('form_leads', __name__, template_folder='templates')

CONDITIONAL_NOTIFICATION_DELAY_DEFAULT = 15

def sortByPosition(e):
  return e['position']


def valid_date(date_text, format):
    try:
        datetime.datetime.strptime(date_text, format)
        return True
    except ValueError:
        return False


def represents_int(s):
    try:
        int(s)
        return True
    except ValueError:
        return False


def append_error(current_message, error_message):
    if len(current_message) > 0:
        current_message = current_message + ' ' + error_message
    else:
        current_message = current_message + error_message

    return current_message


def encrypt_value(text):
    crypto_key = app.config['CRYPTO_SECRET_KEY']
    cipher = AESCipher(crypto_key)

    if text is not None:
        try:
            return cipher.encrypt(text)
        except TypeError:
            #log.error('Error encrypting field value: ' + str(text))
            return text
        except ValueError:
            #log.error('Error encrypting field value: ' + str(text))
            return text
    else:
        return text


def decrypt_value(text):
    crypto_key = app.config['CRYPTO_SECRET_KEY']
    cipher = AESCipher(crypto_key)

    if text is not None:
        try:
            return cipher.decrypt(text)
        except TypeError:
            #log.error('Error decrypting field value: ' + str(text))
            return text
        except ValueError:
            #log.error('Error decrypting field value: ' + str(text))
            return text
    else:
        return text


@form_leads.route('/forms', methods=['GET'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def forms_list():
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    if current_user.forms_access:
        forms = Form.query.all()
        account_forms = Form.query.filter(
            Form.partnership_account_id == partnership_account_id
        ).first()
        return render_template('forms.jinja2', forms=forms, account_forms=account_forms)
    else:
        flash(_('You do not have permission to do that.'), 'danger')
        return redirect(url_for('leads.call_leads'))


@form_leads.route('/forms/<int:form_id>/leads', methods=['GET'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def leads_list(form_id):
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    # Import the partnership account model
    from ..partnership.models import PartnershipAccount
    partnership_account = PartnershipAccount\
        .query\
        .filter(PartnershipAccount.id == partnership_account_id)\
        .first()

    if current_user.forms_access:
        form = Form.query.filter(
            Form.partnership_account_id == partnership_account_id,
            Form.id == form_id
        ).first()
        if not form:
            flash(_('Form with ID {} does not exist.').format(form_id))
            return redirect(url_for('form_leads.forms_list'))

        formname = form.display_name

        fields = FormField.query.filter(
            FormField.form_id == form_id,
            FormField.show_in_table == True  # noqa
        ).order_by('position')\
            .all()

        from ..user.models import UserExternalApiAutoPostTie
        service_provider_list = UserExternalApiAutoPostTie.get_allowed_service_providers(current_user.id, partnership_account_id)

        supervisor_user_business_type = None
        from buyercall.lib.supervisor_manager import current_supervisor_user
        if current_supervisor_user and current_supervisor_user.is_authenticated:
            partner_id = current_supervisor_user.partnership_id

            if partner_id:
                from ..partnership.models import Partnership
                supervisor_user_partnership = Partnership.query.filter(Partnership.id == partner_id).first()
                if supervisor_user_partnership and supervisor_user_partnership.business_type:
                    supervisor_user_business_type = supervisor_user_partnership.business_type
            elif current_supervisor_user.role == 'limitsysadmin':
                supervisor_user_business_type = 'automotive'

        return render_template('leads.jinja2',
                               business_type=partnership_account.business_type,
                               supervisor_user_business_type=supervisor_user_business_type,
                               fields=fields,
                               form_id=form_id,
                               formname=formname,
                               list=service_provider_list)
    else:
        flash(_('You do not have permission to do that.'), 'danger')
        return redirect(url_for('leads.call_leads'))


@form_leads.route('/forms/<int:id_>/notification_criteria', methods=['GET'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def form_notification_criteria(id_):
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    form = Form.query.filter(
        Form.partnership_account_id == partnership_account_id,
        Form.id == id_
    ).one()

    if form:
        return jsonify(
            conditions=form.conditional_criteria,
            is_conditional_notification=form.is_conditional_notification
        )
    else:
        return jsonify(
            conditions='',
            is_conditional_notification=False
        )


@form_leads.route('/forms/<int:id_>', methods=['GET'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def form_edit(id_):
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership
    external_provider_name = 'external service provider'

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    form = Form.query.filter(
        Form.partnership_account_id == partnership_account_id,
        Form.id == id_
    ).one()

    routings = Widget.query.filter(
        Widget.partnership_account_id == partnership_account_id,
        Widget.enabled == True  # noqa
    ).all()

    fields = FormField.query.filter(
        FormField.form_id == id_
    ).order_by(FormField.position).all()

    # Check to see if the user has credit prequalify enabled
    from ..partnership.models import PartnershipAccountCreditTie, PartnershipCreditTie
    prequalify_credentials = PartnershipAccountCreditTie.partner_account_seven_hundred_credit_info(
        partnership_account_id, 'prequalify')
    if not prequalify_credentials:
        prequalify_credentials = PartnershipAccountCreditTie.partner_account_finserv_credit_info(
            partnership_account_id, 'prequalify')
    # if not prequalify_credentials:
    #    partnership_id = current_user.partnership_id
    #    prequalify_credentials = PartnershipCreditTie.partner_finserv_credit_info(partnership_id, 'prequalify')

    from ..partnership.models import PartnershipAccount, ExternalApiServiceProvidersPartnershipAccountTie
    account = PartnershipAccount.query.filter(PartnershipAccount.id == current_user.partnership_account_id).first()
    account_name = re.sub('[^A-Za-z0-9]+', '', account.name).lower() + '@buyercall.com'
    check_service_provider = ExternalApiServiceProvidersPartnershipAccountTie.service_provider_name(
        partnership_account_id
    )

    if check_service_provider is not None and check_service_provider is not '':
        external_provider_name = check_service_provider

    # The auto email responds subject
    if form.send_auto_email_subject:
        auto_email_subject = form.send_auto_email_subject
    else:
        auto_email_subject = '#COMPANY# - Thank you for your submission'
    if form.send_auto_email_msg:
        auto_email_body = form.send_auto_email_msg
    else:
        body_string = '\n\nThank you for your submission\n---\n{0}\n---\n'\
            .format('We are in the process of reviewing your submission for the #FORM#. '
                    'Once we will be in contact as soon as possible')
        auto_email_body = _('Hi #NAME#,%(body)s Thanks,\n#COMPANY# support team'
                            '\n\n Please note this is an automated email notification. Do not reply to this email.',
                            body=body_string)

    # This section will handle some pre-stuff for the auto sms template
    from ..phonenumbers.models import Phone
    phone_numbers = Phone.query\
        .filter(Phone.partnership_account_id == partnership_account_id)\
        .filter(Phone.is_deactivated == '0').all()

    if phone_numbers:
        active_numbers = True
    else:
        active_numbers = False

    # Count how many active phone numbers there are for the user and update the total in total_numbers above
    # Also declare the phone numbers and the friendly name for the user and add them to a list above

    numbers = []
    for no in phone_numbers:
        number = no.phonenumber
        name = no.friendly_name
        inbound_id = no.id
        display_name = '{} - {}'.format(name, number)
        phone_info = [inbound_id, display_name]
        numbers.append(phone_info)
    # Look through the phone numbers for the current user and create a dictionary
    phone_dict = {}
    for number in numbers:
        phone_dict[number[0]] = number[1]
    log.info(phone_dict)

    if form.send_auto_sms_msg:
        auto_sms_body = form.send_auto_sms_msg
    else:
        auto_sms_body = _('#COMPANY#: Hi #NAME#, thanks for your form submission. We will review it and get back '
                          'to you asap.')
    if form.send_auto_sms_inbound_id:
        selected_inbound_id = form.send_auto_sms_inbound_id
    else:
        selected_inbound_id = ''

    from ..user.models import UserExternalApiAutoPostTie
    from ..form_leads.models import ExternalApiFormAutoPostTie

    ams_evo_allowed = False
    ams_evo_is_auto_send = False
    ams_evo_tie_exists = UserExternalApiAutoPostTie.get_service_provider_access_state(current_user.id, partnership_account_id, 'AMS Evolution')
    if ams_evo_tie_exists and ams_evo_tie_exists.is_allowed:
        ams_evo_allowed = True

    ams_evo_auto_send_tie = ExternalApiFormAutoPostTie.get_service_provider_send_state(form.id, partnership_account_id, 'AMS Evolution')
    if ams_evo_auto_send_tie and ams_evo_auto_send_tie.is_automatically_sent:
        ams_evo_is_auto_send = True

    ams_2000_allowed = False
    ams_2000_is_auto_send = False
    ams_2000_tie_exists = UserExternalApiAutoPostTie.get_service_provider_access_state(current_user.id, partnership_account_id, 'AMS 2000')
    if ams_2000_tie_exists and ams_2000_tie_exists.is_allowed:
        ams_2000_allowed = True

    ams_2000_auto_send_tie = ExternalApiFormAutoPostTie.get_service_provider_send_state(form.id, partnership_account_id, 'AMS 2000')
    if ams_2000_auto_send_tie and ams_2000_auto_send_tie.is_automatically_sent:
        ams_2000_is_auto_send = True

    neo_verify_allowed = False
    neo_verify_is_auto_send = False
    neo_verify_tie_exists = UserExternalApiAutoPostTie.get_service_provider_access_state(current_user.id, partnership_account_id, 'NEO Verify')
    if neo_verify_tie_exists and neo_verify_tie_exists.is_allowed:
        neo_verify_allowed = True

    neo_verify_auto_send_tie = ExternalApiFormAutoPostTie.get_service_provider_send_state(form.id, partnership_account_id, 'NEO Verify')
    if neo_verify_auto_send_tie and neo_verify_auto_send_tie.is_automatically_sent:
        neo_verify_is_auto_send = True

    fields = FormField.query.filter(
        FormField.form_id == id_
    ).order_by(FormField.position).all()

    selection_fields = []

    return render_template(
        'form_edit.jinja2',
        form=form,
        routings=routings,
        fields=fields,
        account_name=account_name,
        subject=auto_email_subject,
        body=auto_email_body,
        phone_options=phone_dict,
        auto_sms_body=auto_sms_body,
        selected_phone_number=selected_inbound_id,
        active_numbers=active_numbers,
        external_service_provider_name=external_provider_name,
        prequalify_credentials=prequalify_credentials,
        ams_2000_show=ams_2000_allowed,
        ams_2000_auto_send=ams_2000_is_auto_send,
        ams_evo_show=ams_evo_allowed,
        ams_evo_auto_send=ams_evo_is_auto_send,
        neo_verify_show=neo_verify_allowed,
        neo_verify_auto_send=neo_verify_is_auto_send
    )


# return data into jquery datatables
@form_leads.route('/forms/<int:id_>/form_logs/data')
@csrf.exempt
@login_required
def data(id_):
    order = int(request.args.get('order[0][column]', '-1'))
    direction = request.args.get('order[0][dir]', 'asc')
    offset = int(request.args.get('start', 0))
    limit = int(request.args.get('length', 99))
    partnership_account_id = current_user.partnership_account_id

    from buyercall.blueprints.partnership.models import ExternalApiServiceProviders

    all_logs = db.session.query(FormLog.id,
                            FormLog.posted_on,
                            FormLog.description,
                            ExternalApiServiceProviders.name,
                            FormLog.status,
                            FormLog.form_lead_id,
                            FormLog.external_api_service_provider_id)\
        .join(Lead) \
        .outerjoin(ExternalApiServiceProviders) \
        .filter(and_(FormLog.form_id == id_, Lead.partnership_account_id == partnership_account_id))

    columns = [
        FormLog.id,
        FormLog.posted_on,
        FormLog.description,
        'external_api_service_providers.name',
        FormLog.status,
        FormLog.form_lead_id
    ]
    sorted_logs = all_logs.with_entities(*columns)

    if order in range(len(columns)):
        order_pred = columns[order]
        if direction == 'desc':
            order_pred = desc(order_pred)
        sorted_logs = sorted_logs.order_by(order_pred)

    sorted_logs = sorted_logs.offset(offset).limit(limit)
    data = [
        {i: row[i] for i in range(len(row))
         } for row in sorted_logs.all()
        ]

    return jsonify(
        recordsFiltered=all_logs.count(),
        recordsTotal=all_logs.count(),
        data=data
    )


@form_leads.route('/forms/<int:id_>', methods=['POST'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def form_update(id_):
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    form = Form.query.filter(
        Form.partnership_account_id == partnership_account_id,
        Form.id == id_
    ).one()

    routing = Widget.query.options(load_only('id')).filter(
        Widget.partnership_account_id == partnership_account_id,
        Widget.id == (request.form.get('routing') or None)
    ).first()

    ams_2000_auto_send_post = request.form.get('ams_2000_auto_send', '')
    ams_evo_auto_send_post = request.form.get('ams_evo_auto_send', '')
    neo_verify_auto_send_post = request.form.get('neo_verify_auto_send', '')

    from ..user.models import UserExternalApiAutoPostTie
    from ..form_leads.models import ExternalApiFormAutoPostTie

    # check user permissions for ams evolution
    ams_evo_auto_send = False
    ams_evo_tie_exists = UserExternalApiAutoPostTie.get_service_provider_access_state(current_user.id,
                                                                                      partnership_account_id,
                                                                                      'AMS Evolution')
    if ams_evo_tie_exists and ams_evo_tie_exists.is_allowed:
        if ams_evo_auto_send_post == 'on':
            ams_evo_auto_send = True
        # set ams evolution auto send state
        ExternalApiFormAutoPostTie.set_service_provider_access_state(form.id,
                                                                     partnership_account_id,
                                                                     'AMS Evolution',
                                                                     ams_evo_auto_send)

    # check user permissions for ams 2000
    ams_2000_auto_send = False
    ams_2000_tie_exists = UserExternalApiAutoPostTie.get_service_provider_access_state(current_user.id,
                                                                                       partnership_account_id,
                                                                                       'AMS 2000')
    if ams_2000_tie_exists and ams_2000_tie_exists.is_allowed:
        if ams_2000_auto_send_post == 'on':
            ams_2000_auto_send = True
        # set ams 2000 auto send state
        ExternalApiFormAutoPostTie.set_service_provider_access_state(form.id, partnership_account_id, 'AMS 2000',
                                                                     ams_2000_auto_send)

    # check user permissions for neo verify
    neo_verify_auto_send = False
    neo_verify_tie_exists = UserExternalApiAutoPostTie.get_service_provider_access_state(current_user.id,
                                                                                      partnership_account_id,
                                                                                      'NEO Verify')
    if neo_verify_tie_exists and neo_verify_tie_exists.is_allowed:
        if neo_verify_auto_send_post == 'on':
            neo_verify_auto_send = True
        # set neo verify auto send state
        ExternalApiFormAutoPostTie.set_service_provider_access_state(form.id,
                                                                     partnership_account_id,
                                                                     'NEO Verify',
                                                                     neo_verify_auto_send)

    partial_email = request.form.get('partial_data_email', '')
    if partial_email == 'on':
        partial_email = True
    else:
        partial_email = False

    full_email = request.form.get('full_submit_email', '')
    if full_email == 'on':
        full_email = True
    else:
        full_email = False

    auto_email_respond = request.form.get('send_auto_email', '')
    if auto_email_respond == 'on':
        auto_email_respond = True
    else:
        auto_email_respond = False

    auto_sms_respond = request.form.get('send_auto_sms', '')
    if auto_sms_respond == 'on':
        auto_sms_respond = True
    else:
        auto_sms_respond = False

    auto_prequalify_lead = request.form.get('auto_prequalify_credit', '')
    if auto_prequalify_lead == 'on':
        auto_prequalify_lead = True
    else:
        auto_prequalify_lead = False

    conditional_notification = request.form.get('is_conditional', '')
    conditions = request.form.get('condition_criteria', '')

    form.display_name = request.form.get('display_name', '')
    form.routing = routing
    form.email_addresses = {'emails': request.form.get('recipient_emails', '')}
    form.adf_email_addresses = {'emails': request.form.get('form_adf_emails', '')}
    form.email_subject = request.form.get('email_subject', '')
    form.partial_data_email = partial_email
    form.full_submit_email = full_email
    form.send_auto_email = auto_email_respond
    form.send_auto_sms = auto_sms_respond
    form.auto_prequalify_credit = auto_prequalify_lead

    if conditional_notification in ['true', 'True'] and conditions:
        form.is_conditional_notification = True
    else:
        form.is_conditional_notification = False
    form.conditional_criteria = json.loads(conditions)

    # Check if the auto email respond checkbox is checked and if the
    # The subject field is empty populate it so that there's atleast
    # some content being sent
    if form.send_auto_email_subject == '' and auto_email_respond:
        form.send_auto_email_subject = '#COMPANY# -' + ' ' + 'Thank you for your submission'
    # Same as for the subject above, check if the email body is empty
    # and if so populate it with template content
    if form.send_auto_email_msg == '' and auto_email_respond:
        body_string = '\n\nThank you for your submission\n---\n{0}\n---\n' \
            .format('We are in the process of reviewing your submission for the #FORM#. '
                    'Once we will be in contact as soon as possible')
        auto_email_body = _('Hi #NAME#,%(body)s Thanks,\n#COMPANY# support team'
                            '\n\n Please note this is an automated email notification. Do not reply to this email.',
                            body=body_string)
        form.send_auto_email_msg = auto_email_body

    # Check if the auto sms respond checkbox is checked and if the
    # The inbound field is empty populate it so that there's atleast a number used
    if form.send_auto_sms_inbound_id is None and auto_sms_respond:
        from ..phonenumbers.models import Phone
        phone_number = Phone.query \
            .filter(Phone.partnership_account_id == current_user.partnership_account_id) \
            .filter(Phone.is_deactivated == '0').first()
        form.send_auto_sms_inbound_id = phone_number.id
    # Check if the auto sms respond checkbox is checked and if the
    # The sms body is empty populated it with template data
    if form.send_auto_sms_msg == '' and auto_sms_respond:
        form.send_auto_sms_msg = _('Hi #NAME#, thanks for your form submission. We will review it and get back to you '
                                   'asap. #COMPANY#')

    fields = FormField.query.filter(
        FormField.form_id == id_
    ).order_by(FormField.position).all()
    for field in fields:
        new_name = request.form.get('field-{}'.format(field.field_id))
        if new_name:
            field.display_name = new_name

    for field in fields:
        try:
            pos = int(request.form.get('field-pos-{}'.format(field.field_id), -1))
            if pos != -1:
                field.position = pos
        except Exception:
            pass

    try:
        db.session.commit()
        ActivityLogs.add_log(current_user.id, ActivityType.PAGE, ActivityName.EDIT, request, id_)
        flash(_('The form has been updated successfully.'), 'success')
        return redirect(url_for('form_leads.forms_list'))
    except Exception:
        log.error(traceback.format_exc())
        db.session.rollback()
        flash(_("Error saving form information."), 'danger')

    routings = Widget.query.filter(
        Widget.partnership_account_id == partnership_account_id,
        Widget.enabled == True  # noqa
    ).all()

    fields = sorted(fields, key=lambda f: f.position)

    return render_template(
        'form_edit.jinja2', form=form, routings=routings, fields=fields
    )


@form_leads.route('/api/forms', methods=['GET'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def forms_json():
    """Return server side data."""
    search = request.args.get('search[value]', '')
    order = int(request.args.get('order[0][column]', '-1'))
    direction = request.args.get('order[0][dir]', 'asc')
    offset = int(request.args.get('start', 0))
    limit = int(request.args.get('length', 99))
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    columns = [
        'id', 'display_name', 'lead_count', 'created_on', 'updated_on'
    ]

    total = Form.query.filter(
        Form.partnership_account_id == partnership_account_id
    )

    leads = Lead.query.filter(
        Lead.partnership_account_id == partnership_account_id
    ).with_entities(
        Lead.form_id,
        func.sum(1).label('lead_count')
    ).group_by(Lead.form_id).subquery()

    filtered = total
    if search:
        pattern = '%{}%'.format(search)
        filtered = total.filter(or_(
            Form.display_name.ilike(pattern),
        ))

    filtered = filtered.outerjoin(
        (leads, Form.id == leads.c.form_id)
    ).with_entities(
        Form.id,
        Form.display_name,
        case(
            [(leads.c.lead_count.is_(None), 0)],
            else_=leads.c.lead_count
        ).label('lead_count'),
        Form.created_on,
        Form.updated_on
    )

    sorted_ = filtered
    if order in range(len(columns)):
        order_pred = '{} {}'.format(columns[order], direction)
        sorted_ = sorted_.order_by(text(order_pred))
    sorted_ = sorted_.offset(offset).limit(limit)

    data = [
        {i: row[i] for i in range(len(row))} for row in sorted_.all()
    ]

    return jsonify(
        draw=request.args['draw'],
        recordsFiltered=filtered.count(),
        recordsTotal=total.count(),
        data=data
    )


@form_leads.route('/forms/csv', methods=['GET'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def forms_csv():
    """Return server side data."""
    search = request.args.get('search[value]', '')
    order = int(request.args.get('order[0][column]', '-1'))
    direction = request.args.get('order[0][dir]', 'asc')
    columns = [
        'id', 'display_name', 'lead_count', 'created_on', 'updated_on'
    ]
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    total = Form.query.filter(
        Form.partnership_account_id == partnership_account_id
    )

    leads = Lead.query.filter(
        Lead.partnership_account_id == partnership_account_id
    ).with_entities(
        Lead.form_id,
        func.sum(1).label('lead_count')
    ).group_by(Lead.form_id).subquery()

    filtered = total
    if search:
        pattern = '%{}%'.format(search)
        filtered = total.filter(or_(
            Form.display_name.ilike(pattern),
        ))

    filtered = filtered.outerjoin(
        (leads, Form.id == leads.c.form_id)
    ).with_entities(
        Form.id,
        Form.display_name,
        case(
            [(leads.c.lead_count.is_(None), 0)],
            else_=leads.c.lead_count
        ).label('lead_count'),
        Form.created_on,
        Form.updated_on
    )
    sorted_ = filtered
    if order in range(len(columns)):
        order_pred = '{} {}'.format(columns[order], direction)
        sorted_ = sorted_.order_by(text(order_pred))

    row_no = 0
    header = ['No', 'Friendly Name', 'Lead Count', 'Created On', 'Updated On']
    with closing(StringIO()) as out:
        writer = csv.writer(out)
        writer.writerow(header)
        for row in sorted_.all():
            csv_row = [x for x in row]
            row_no += 1
            csv_row[0] = row_no
            writer.writerow(csv_row)

        filename = 'Forms-{}.csv'.format(
            date.today().strftime('%Y-%m-%d')
        )

        resp = make_response(out.getvalue())
        resp.headers['Content-Type'] = 'text/csv'
        resp.headers['Content-Disposition'] = 'attachment; filename={}'.format(
            filename
        )
        ActivityLogs.add_log(current_user.id, ActivityType.DATA, ActivityName.DOWNLOAD, request, None, 'csv')
        return resp


@form_leads.route('/api/forms/<int:form_id>/leads', methods=['POST'])
@csrf.exempt
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def leads_json(form_id):
    """Return server side data."""
    search = request.form.get('search[value]', '')
    order = int(request.form.get('order[0][column]', '-1'))
    direction = request.form.get('order[0][dir]', 'asc')
    offset = int(request.form.get('start', 0))
    limit = int(request.form.get('length', 99))
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership
    encrypt_key = app.config['CRYPTO_SECRET_KEY']
    cipher = AESCipher(encrypt_key)

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    form = Form.query.filter(
        Form.partnership_account_id == partnership_account_id,
        Form.id == form_id
    ).first()
    if not form:
        return 'Form not found', 404

    fields = FormField.query.filter(
        FormField.form_id == form_id,
        FormField.show_in_table == True  # noqa
    ).order_by(FormField.position).all()

    columns = ['id']
    columns.extend([field.field_id for field in fields])
    columns.extend(['form_name', 'created_on', 'updated_on', 'credit_score'])

    total = Lead.query.filter(
        Lead.partnership_account_id == partnership_account_id,
        Lead.form_id == form_id
    )

    filtered = total
    if search:
        pattern = '%{}%'.format(search)
        filtered = filtered.filter(
            Lead.fields.any(decrypt_value(LeadField.field_value).ilike(pattern))
        )

    sorted_ = filtered
    if order in range(len(columns)):
        order_field = columns[order]

        if order_field in ['created_on', 'updated_on']:
            sorted_ = sorted_.order_by(text('%s %s' % (order_field, direction)))
        else:
            field_query = LeadField.query.filter(
                LeadField.field_id == order_field
            ).with_entities(
                LeadField.lead_id,
                decrypt_value(LeadField.field_value.label('sort_field'))
            ).subquery()
            sorted_ = sorted_.options(joinedload(Lead.fields)).outerjoin(
                (field_query, Lead.id == field_query.c.lead_id)
            )
            if direction == 'desc':
                sorted_ = sorted_.order_by(field_query.c.sort_field.desc())
            else:
                sorted_ = sorted_.order_by(field_query.c.sort_field)

    sorted_ = sorted_.offset(offset).limit(limit).all()

    from ..form_leads.models import ExternalApiFormLeadPostTie
    sorted_ids = [row.id for row in sorted_]
    lead_posts = ExternalApiFormLeadPostTie.existing_posts_for_leads(form_id, sorted_ids, partnership_account_id)

    data = []
    for lead in sorted_:
        row = {}
        lead_fields = {}
        service_provider_posts = []

        for field in lead.fields:
            lead_fields[field.field_id] = decrypt_value(field.field_value)

        for i, field in enumerate(fields):
            row[str(i + 1)] = lead_fields.get(field.field_id, '')
        n = len(fields)

        # Get the credit score for the leads in the form leads table
        from ..contacts.models import CreditReports
        credit_report = CreditReports.query \
            .filter(and_(CreditReports.contact_id == lead.contact_id,
                         CreditReports.partnership_account_id == partnership_account_id,
                         CreditReports.is_successful.is_(True))).order_by(CreditReports.created_on.desc()).first()

        if credit_report is not None:
            encrypt_key = app.config['CRYPTO_SECRET_KEY']
            cipher = AESCipher(encrypt_key)
            # Decrypt the credit score value
            credit_score = cipher.decrypt(credit_report.credit_score)
        else:
            credit_score = 'Unknown'

        if lead.id in lead_posts:
            service_provider_posts = lead_posts[lead.id]

        row.update({
            '0': lead.id,
            str(n + 1): form.display_name,
            str(n + 2): lead.created_on,
            str(n + 3): lead.updated_on,
            str(n + 4): credit_score,
            'contact_id': lead.contact_id,
            'external_api_lead_id': service_provider_posts
        })
        data.append(row)

    return jsonify(
        draw=request.form['draw'],
        recordsFiltered=filtered.with_entities(Lead.id).group_by(Lead.id).count(),
        recordsTotal=total.with_entities(Lead.id).count(),
        data=data
    )


@form_leads.route('/forms/<int:form_id>/leads/csv', methods=['GET'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def leads_csv(form_id):
    """Return server side data."""

    if current_user.role == 'admin' or current_user.role == 'sysadmin':
        date_from = str(request.args.get('df'))
        date_to = str(request.args.get('dt'))
        partnership_account_id = current_user.partnership_account_id
        partnership_account_group_viewing = current_user.is_viewing_partnership

        # set the date filters
        converted_date_from = datetime.datetime.strptime(date_from, "%m%d%Y").date()
        converted_date_to = datetime.datetime.strptime(date_to, "%m%d%Y").date()
        filter_by_date = and_(func.date(Lead.updated_on) >= converted_date_from, func.date(Lead.updated_on) <= converted_date_to)

        # Check if being viewed by super partner
        if partnership_account_group_viewing:
            partnership_account_id = current_user.get_user_viewing_partnership_account_id

        from ..partnership.models import PartnershipAccount
        partnership_account = PartnershipAccount \
            .query \
            .filter(PartnershipAccount.id == partnership_account_id) \
            .first()

        form = Form.query.filter(
            Form.partnership_account_id == partnership_account_id,
            Form.id == form_id
        ).first()
        if not form:
            return 'Form not found', 404

        fields = FormField.query.filter(
            FormField.form_id == form_id,
        ).order_by(FormField.position).all()

        columns = ['id']
        columns.extend([field.field_id for field in fields])
        columns.extend(['created_on', 'updated_on'])

        filtered = Lead\
            .query\
            .filter(Lead.partnership_account_id == partnership_account_id,
                    Lead.form_id == form_id)\
            .filter(filter_by_date)\
            .order_by(desc(Lead.updated_on))

        lead_results = filtered\
            .from_self()\
            .limit(2500)\
            .all()

        from ..contacts.models import Contact
        contact_results = filtered.join(Contact).with_entities(Contact.id, Contact.credit_score).all()
        contact_list = {}
        for row in contact_results:
            contact_list[row[0]] = row[1]

        row_no = 0
        header = ['No']
        header.extend([field.display_name for field in fields])

        if partnership_account.business_type == 'automotive':
            header.extend(['Form Name', 'Credit Score', 'Created On', 'Updated On'])
        else:
            header.extend(['Form Name', 'Created On', 'Updated On'])

        with closing(StringIO()) as out:
            writer = csv.writer(out)
            writer.writerow(header)
            lead_fields = {}

            for row in lead_results:
                row_no += 1
                credit_score = None

                for field in row.fields:
                    lead_fields[field.field_id] = decrypt_value(field.field_value)

                csv_row = [row_no]
                csv_row += [lead_fields.get(x.field_id, '') for x in fields]

                if partnership_account.business_type == 'automotive':
                    if contact_list[row.contact_id]:
                        credit_score = decrypt_value(contact_list[row.contact_id])

                    csv_row += [form.display_name, credit_score, row.created_on, row.updated_on]
                else:
                    csv_row += [form.display_name, row.created_on, row.updated_on]

                writer.writerow([x for x in csv_row])

            filename = 'Form Leads-{}.csv'.format(
                date.today().strftime('%Y-%m-%d')
            )

            resp = make_response(out.getvalue())
            resp.headers['Content-Type'] = 'text/csv'
            resp.headers['Content-Disposition'] = 'attachment; filename={}'.format(
                filename
            )
            ActivityLogs.add_log(current_user.id, ActivityType.DATA, ActivityName.DOWNLOAD, request, form_id)
            return resp
    else:
        return ('Access Denied', 403)


@form_leads.route('/form_leads', methods=['POST'])
@csrf.exempt
@cross_origin()
def add_leads():
    post_state = None

    def redirect_back():
        """ Redirect to the thankyou page after POST """
        new_path = request.form.get('redirect')
        response = make_response('', 303)
        response.headers['Location'] = new_path
        return response

    form_id = request.form.get('form-id')

    if not form_id:
        log.error("Cannot save lead: missing form-id field on HTML page")
        return redirect_back()
    from ..partnership.models import PartnershipAccount,Partnership
    form = Form.query.filter(Form.public_id == form_id).first()

    partnership_account_id = form.partnership_account_id
    partnership_account = PartnershipAccount.query.get(partnership_account_id) if partnership_account_id else None
    if partnership_account:
        partnership_id = partnership_account.partnership_id
    else:
        partnership_id = None

    try:
        partnership = Partnership.query.get(partnership_id) if partnership_id else None
        from buyercall.blueprints.sysadmin.models import RequestLog
        request_id = request.environ.get('HTTP_X_REQUEST_ID')
        update_data = {
            "partnership_id": partnership_id if partnership_id else None,
            'partnership_name': partnership.name if partnership else None,
            "partnership_account_id": partnership_account_id if partnership_account_id else None,
            'partnership_account_name': partnership_account.name if partnership_account else None,
            "resource": form.id if form else None,
            'user_id': None}

        RequestLog().update_record(request_id, update_data)

    except Exception as e:
        print(f"The exception in /form_leads POST is: {e}")
    # end save form-id to resource field of request-log data

    if not form:
        log.error("Cannot save lead: cannot find form '{}' in database".format(
            form_id
        ))
        return redirect_back()

    fields = FormField.query.filter(FormField.form_id == form.id).all()
    field_ids = [x.field_id for x in fields]

    if request.form.get('lead-id-field'):
        lead = Lead.query.filter(Lead.id == int(request.form.get('lead-id-field'))).first()
    else:
        lead = Lead()
        lead.form = form
        lead.partnership_account_id = form.partnership_account_id

    ## To comply with privacy rules we need to encrypt sensitive data in rest
    ## The following section will look for the ss numbers and dob dates fields in the form lead
    ## and encrypt the field in the database using AES. The decryption of values starts on line 647
    #encrypt_key = app.config['CRYPTO_SECRET_KEY']
    #cipher = AESCipher(encrypt_key)

    for (key, val) in request.form.items():
        if key not in ['form-id', 'redirect']:
            field = LeadField(key, val)
            encoded = ''

            if field.field_value is not None:
                encoded = encrypt_value(field.field_value)
            field.field_value = encoded

            ## All ss number sensitive fields should be either cumemberfield, snnfield, cosignerssnfield
            #if field.field_id == 'cumemberfield' or field.field_id == 'ssnfield':
            #    so_no = field.field_value
            #    # Encrypt the ssn number field and assign it to field value
            #    encoded = cipher.encrypt(so_no)
            #    field.field_value = encoded
            #if field.field_id == 'cosignerssnfield':
            #    co_so_no = field.field_value
            #    cipher = AESCipher(encrypt_key)
            #    # Encrypt the cosigner ssn number field and assign it to field value
            #    encoded = cipher.encrypt(co_so_no)
            #    field.field_value = encoded
            ## All dob date sensitive fields should be either datepicker and cosignerdateofbirthfield
            #if field.field_id == 'datepicker' or field.field_id == 'dobfield':
            #    dob_date = field.field_value
            #    cipher = AESCipher(encrypt_key)
            #    # Encrypt the dob field and assign it to field value
            #    encoded = cipher.encrypt(dob_date)
            #    field.field_value = encoded
            #if field.field_id == 'cosignerdateofbirthfield':
            #    co_dob_date = field.field_value
            #    cipher = AESCipher(encrypt_key)
            #    # Encrypt the cosigner dob field and assign it to field value
            #    encoded = cipher.encrypt(co_dob_date)
            #    field.field_value = encoded

            if lead.id:
                field.lead_id = lead.id
                field = db.session.merge(field)
            lead.fields.append(field)
            if key in field_ids:
                field_ids.remove(key)

    for key in field_ids:
        field = LeadField(key, '')
        if lead.id:
            field.lead_id = lead.id
            field = db.session.merge(field)
        lead.fields.append(field)

    db.session.add(lead)
    db.session.commit()

    # Add contact based on new form lead using celery
    from .form_tasks import add_form_contact
    form_lead_id = lead.id

    first_name = ''
    if 'firstnamefield' in request.form:
        first_name = request.form.get('firstnamefield', '')
    elif 'firstname' in request.form:
        first_name = request.form.get('firstname', '')

    last_name = ''
    if 'lastnamefield' in request.form:
        last_name = request.form.get('lastnamefield', '')
    elif 'lastname' in request.form:
        last_name = request.form.get('lastname', '')

    email_address = ''
    if 'emailfield' in request.form:
        email_address = request.form.get('emailfield', '')
    elif 'email' in request.form:
        email_address = request.form.get('email', '')

    phone_number = ''
    if 'phonefield' in request.form:
        phone_number = request.form.get('phonefield', '')
    elif 'phonenumber' in request.form:
        phone_number = request.form.get('phonenumber', '')

    address_1 = ''
    if 'streetfield' in request.form:
        address_1 = request.form.get('streetfield', '')
    elif 'address_1' in request.form:
        address_1 = request.form.get('address_1', '')

    address_2 = ''
    if 'address_2' in request.form:
        address_2 = request.form.get('address_2', '')

    city = ''
    if 'citytfield' in request.form:
        city = request.form.get('citytfield', '')
    elif 'cityfield' in request.form:
        city = request.form.get('cityfield', '')
    elif 'city' in request.form:
        city = request.form.get('city', '')

    state = ''
    if 'statefield' in request.form:
        state = request.form.get('statefield', '')
    elif 'state' in request.form:
        state = request.form.get('state', '')

    zip = ''
    if 'zipfield' in request.form:
        zip = request.form.get('zipfield', '')
    elif 'zip' in request.form:
        zip = request.form.get('zip', '')

    country = ''
    if 'countryfield' in request.form:
        country = request.form.get('countryfield', '')
    elif 'country' in request.form:
        country = request.form.get('country', '')

    rep = ''
    if 'repfield' in request.form:
        rep = request.form.get('repfield', '')
    elif 'representative' in request.form:
        rep = request.form.get('representative', '')

    birthday = ''
    if 'datepicker' in request.form:
        birthday = request.form.get('datepicker', '')
    elif 'birthday' in request.form:
        birthday = request.form.get('birthday', '')
    elif 'dobfield' in request.form:
        birthday = request.form.get('dobfield', '')

    campaign = None
    if 'campaign' in request.form:
        campaign = request.form.get('campaign', '')
    elif 'campaignfield' in request.form:
        campaign = request.form.get('campaignfield', '')

    # Vehicle of interest fields
    in_veh_year = None
    if 'vehicle_year' in request.form:
        in_veh_year = request.form.get('vehicle_year', None)
    elif 'vehicleyearfield' in request.form:
        in_veh_year = request.form.get('vehicleyearfield', None)
    if in_veh_year in ('NaN', 'undefined'):
        in_veh_year = None

    in_veh_make = ''
    if 'vehicle_make' in request.form:
        in_veh_make = request.form.get('vehicle_make', '')
    elif 'vehiclemakefield' in request.form:
        in_veh_make = request.form.get('vehiclemakefield', '')

    in_veh_model = ''
    if 'vehicle_model' in request.form:
        in_veh_model = request.form.get('vehicle_model', '')
    elif 'vehiclemodelfield' in request.form:
        in_veh_model = request.form.get('vehiclemodelfield', '')

    in_veh_vin = ''
    if 'vin' in request.form:
        in_veh_vin = request.form.get('vin', '')
    elif 'vinfield' in request.form:
        in_veh_vin = request.form.get('vinfield', '')

    in_veh_trim = ''
    if 'vehicle_trim' in request.form:
        in_veh_trim = request.form.get('vehicle_trim', '')
    elif 'vehicletrimfield' in request.form:
        in_veh_trim = request.form.get('vehicletrimfield', '')

    in_veh_stock = ''
    if 'stock_number' in request.form:
        in_veh_stock = request.form.get('stock_number', '')
    elif 'stockfield' in request.form:
        in_veh_stock = request.form.get('stockfield', '')

    in_veh_price = None
    if 'vehicle_purchase_price' in request.form:
        in_veh_price = request.form.get('vehicle_purchase_price', None)
    elif 'vehiclepurchasepricefield' in request.form:
        in_veh_price = request.form.get('vehiclepurchasepricefield', None)
    if in_veh_price in ('NaN', 'undefined'):
        in_veh_price = None

    in_veh_mileage = None
    if 'vehicle_mileage' in request.form:
        in_veh_mileage = request.form.get('vehicle_mileage', None)
    elif 'vehiclemileagefield' in request.form:
        in_veh_mileage = request.form.get('vehiclemileagefield', None)
    if in_veh_mileage in ('NaN', 'undefined'):
        in_veh_mileage = None

    in_veh_condition = ''
    if 'vehicle_condition' in request.form:
        in_veh_condition = request.form.get('vehicle_condition', '')
    elif 'vehicleconditionfield' in request.form:
        in_veh_condition = request.form.get('vehicleconditionfield', '')

    in_veh_url = ''
    if 'listing_url' in request.form:
        in_veh_url = request.form.get('listing_url', '')
    elif 'listingurlfield' in request.form:
        in_veh_url = request.form.get('listingurlfield', '')

    ti_veh_year = None
    if 'trade_in_vehicle_year' in request.form:
        ti_veh_year = request.form.get('trade_in_vehicle_year', None)
    elif 'tradeinvehicleyearfield' in request.form:
        ti_veh_year = request.form.get('tradeinvehicleyearfield', None)
    if ti_veh_year in ('NaN', 'undefined'):
        ti_veh_year = None

    ti_veh_make = ''
    if 'trade_in_vehicle_make' in request.form:
        ti_veh_make = request.form.get('trade_in_vehicle_make', '')
    elif 'tradeinvehiclemakefield' in request.form:
        ti_veh_make = request.form.get('tradeinvehiclemakefield', '')

    ti_veh_model = ''
    if 'trade_in_vehicle_model' in request.form:
        ti_veh_model = request.form.get('trade_in_vehicle_model', '')
    elif 'tradeinvehiclemodelfield' in request.form:
        ti_veh_model = request.form.get('tradeinvehiclemodelfield', '')

    ti_veh_vin = ''
    if 'trade_in_vehicle_vin' in request.form:
        ti_veh_vin = request.form.get('trade_in_vehicle_vin', '')
    elif 'tradeinvehiclevinfield' in request.form:
        ti_veh_vin = request.form.get('tradeinvehiclevinfield', '')

    ti_veh_value = None
    if 'trade_in_vehicle_value' in request.form:
        ti_veh_value = request.form.get('trade_in_vehicle_value', None)
    elif 'tradeinvehiclevaluefield' in request.form:
        ti_veh_value = request.form.get('tradeinvehiclevaluefield', None)
    if ti_veh_value in ('NaN', 'undefined'):
        ti_veh_value = None

    ti_veh_mileage = None
    if 'trade_in_vehicle_mileage' in request.form:
        ti_veh_mileage = request.form.get('trade_in_vehicle_mileage', None)
    elif 'tradeinvehiclemileagefield' in request.form:
        ti_veh_mileage = request.form.get('tradeinvehiclemileagefield', None)
    if ti_veh_mileage in ('NaN', 'undefined'):
        ti_veh_mileage = None

    ti_veh_condition = None
    if 'trade_in_vehicle_condition' in request.form:
        ti_veh_condition = request.form.get('trade_in_vehicle_condition', None)
    elif 'tradeinvehicleconditionfield' in request.form:
        ti_veh_condition = request.form.get('tradeinvehicleconditionfield', None)

    partnership_account_id = lead.partnership_account_id
    updated_on = lead.updated_on
    add_form_contact(form_lead_id,
                     first_name,
                     last_name,
                     email_address,
                     phone_number,
                     address_1,
                     address_2,
                     city,
                     state,
                     zip,
                     country,
                     rep,
                     in_veh_year,
                     in_veh_make,
                     in_veh_model,
                     in_veh_vin,
                     in_veh_trim,
                     in_veh_stock,
                     in_veh_price,
                     in_veh_mileage,
                     in_veh_condition,
                     in_veh_url,
                     ti_veh_year,
                     ti_veh_make,
                     ti_veh_model,
                     ti_veh_vin,
                     ti_veh_value,
                     ti_veh_mileage,
                     ti_veh_condition,
                     partnership_account_id,
                     updated_on,
                     birthday,
                     campaign=campaign)

    # Check to see which external providers need to be sent to.
    from ..partnership.models import ExternalApiServiceProviders, ExternalApiServiceProvidersPartnershipAccountTie
    from ..form_leads.models import ExternalApiFormAutoPostTie
    service_providers = (db.session.query(ExternalApiFormAutoPostTie.external_api_service_provider_id,
                                          ExternalApiServiceProviders.name)
                         .filter(ExternalApiServiceProvidersPartnershipAccountTie.active,
                                 ExternalApiServiceProvidersPartnershipAccountTie.external_api_service_provider_id ==
                                 ExternalApiFormAutoPostTie.external_api_service_provider_id,
                                 ExternalApiServiceProvidersPartnershipAccountTie.partnership_account_id ==
                                 partnership_account_id).filter(ExternalApiFormAutoPostTie.form_id == form.id,
                                                                ExternalApiFormAutoPostTie.is_automatically_sent)
                         .filter(ExternalApiServiceProvidersPartnershipAccountTie.external_api_service_provider_id ==
                                 ExternalApiServiceProviders.id)).all()

    # Conditional notifications set
    if form.is_conditional_notification:
        log.debug('Conditional notification detected.')
        state = 'partial'
        state_key = 'lead-' + str(lead.id) + '-form-state'
        post_key = 'lead-' + str(lead.id) + '-form-post'
        processed_key = 'lead-' + str(lead.id) + '-form-processed'
        conditional_notification_delay = app.config['CONDITIONAL_NOTIFICATION_PROCESSOR_DELAY']
        posted_request = request.form
        stored_posted_values = None
        posted_values = None
        stored_values = {}
        processed_state = 'incomplete'
        redis_db = redis.StrictRedis(
            host=app.config['REDIS_CONFIG_URL'],
            port=app.config['REDIS_CONFIG_PORT'],
            decode_responses=True
        )
        if conditional_notification_delay:
            delay = conditional_notification_delay * 60
        else:
            delay = CONDITIONAL_NOTIFICATION_DELAY_DEFAULT * 60

        if request.headers.get('partial-submit') == 'yes':
            from buyercall.blueprints.form_leads.form_tasks import check_conditional_notification
            check_conditional_notification.apply_async(args=[lead.id, form.id, form.conditional_criteria], countdown=delay)
        else:
            state = 'full'
            processed_state = 'complete'

        # Process posted values to redis dictionary store
        for field_id in posted_request:
            value = posted_request[field_id]
            if value:
                stored_values[field_id] = value

        stored_posted_values = redis_db.get(post_key)
        if stored_posted_values:
            posted_values = json.loads(stored_posted_values)
        if posted_values:
            for posted_value in posted_values:
                value = posted_values[posted_value]
                if value:
                    key = posted_value
                    stored_values[key] = value

        # Store the posted values, the state and the lead processed state
        redis_db.setex(post_key, delay + 60, json.dumps(stored_values))
        redis_db.setex(state_key, delay + 60, state)
        redis_db.setex(processed_key, delay + 60, processed_state)
        post_state = state

        if state == 'partial':
            # Check providers to see if API post need to be make to 3rd party
            if service_providers:
                for provider in service_providers:
                    if provider.name.lower() == 'neo verify':
                        from buyercall.blueprints.form_leads.form_tasks import form_external_api_request
                        form_external_api_request.delay(lead.id, lead.partnership_account_id,
                                                        provider.external_api_service_provider_id)
            return jsonify(lead=lead.id)
        else:
            process_conditional_notification(lead.id, form.id, form.conditional_criteria)
    else:
        # Standard notifications set
        if request.headers.get('partial-submit') == 'yes':
            if lead.email_sent is False and form.partial_data_email:
                from buyercall.blueprints.form_leads.form_tasks import form_lead_email_notifications
                form_lead_email_notifications.delay(lead.id)
                lead.email_sent = True
                log.debug('A partial form submit email notification has been sent')
                db.session.commit()

            # Check providers to see if API post need to be make to 3rd party
            if service_providers:
                for provider in service_providers:
                    if provider.name.lower() == 'neo verify':
                        from buyercall.blueprints.form_leads.form_tasks import form_external_api_request
                        form_external_api_request.delay(lead.id, lead.partnership_account_id,
                                                        provider.external_api_service_provider_id)
            return jsonify(lead=lead.id)

    if not form.is_conditional_notification or post_state == 'full':
        if lead.email_sent is False and form.full_submit_email:
            from buyercall.blueprints.form_leads.form_tasks import form_lead_email_notifications
            form_lead_email_notifications.delay(lead.id)
            lead.email_sent = True
            log.debug('A full form submit email notification has been sent')
            db.session.commit()

        # Check if email auto responder is turned on and send email
        if form.send_auto_email:
            from ..partnership.models import Partnership, PartnershipAccount
            account = PartnershipAccount.query.filter(
                PartnershipAccount.id == lead.partnership_account_id).first()
            account_name = account.name
            sender_domain = 'buyercall.com'
            partner = Partnership.query.filter(
                Partnership.id == account.partnership_id).first()
            if partner is not None:
                if partner.email_sender:
                    sender_domain = partner.email_sender.split("@")[1]
            account_email_address = re.sub('[^A-Za-z0-9]+', '', account.name).lower() + '@' + sender_domain

            from .form_tasks import send_email_auto_responds
            send_email_auto_responds.delay(email_address, first_name, form.display_name, account_name,
                                           account_email_address, form.send_auto_email_subject, form.send_auto_email_msg)
            log.info('An auto email responds message has been sent to lead id {}'.format(lead.id))

        # Check if sms auto responder is turned on and send text message
        if form.send_auto_sms:
            from ..partnership.models import PartnershipAccount
            account = PartnershipAccount.query.filter(
                PartnershipAccount.id == lead.partnership_account_id).first()
            account_name = account.name

            from .form_tasks import send_sms_auto_responds
            send_sms_auto_responds.delay(phone_number, first_name, form.display_name,
                                         account_name, form.send_auto_sms_inbound_id,
                                         form.send_auto_sms_msg)
            log.info('An auto sms responds message has been sent to lead id {}'.format(lead.id))

    # Try and retrieve the credit score for the lead
    from ..partnership.models import PartnershipAccountCreditTie
    credit_credentials = PartnershipAccountCreditTie.partner_account_seven_hundred_credit_info(
        partnership_account_id, 'prequalify')
    service_provider = '700Credit'
    if not credit_credentials:
        credit_credentials = PartnershipAccountCreditTie.partner_account_finserv_credit_info(
            partnership_account_id, 'prequalify')
        service_provider = 'Finserv'
    if not credit_credentials:
        from ..partnership.models import PartnershipAccount, PartnershipCreditTie
        partner_account = PartnershipAccount.query.filter(PartnershipAccount.id == partnership_account_id).first()
        try:
            credit_credentials = PartnershipCreditTie.partner_finserv_credit_info(
                partner_account.partnership_id, 'prequalify')
        except:
            log.error("Unable to set partnership id for lead {} and got error : {}".format(
                lead.id, traceback.format_exc()))
        service_provider = 'Finserv'
    try:
        if form.auto_prequalify_credit and credit_credentials:
            if credit_credentials.experian_enabled:
                bureau = 'experian'
            elif credit_credentials.transunion_enabled:
                bureau = 'transunion'
            elif credit_credentials.equifax_enabled:
                bureau = 'equifax'
            else:
                bureau = ''
            from buyercall.blueprints.form_leads.form_tasks import form_prequalify_credit_check
            form_prequalify_credit_check.delay(bureau, 'prequalify', lead.contact_id, partnership_account_id, first_name,
                                         last_name, address_1, city, state, zip, service_provider, birthday)
    except Exception:
        log.error("Cannot check credit on form lead {} and got error : {}".format(lead.id, traceback.format_exc()))
    try:
        if form.routing:
            widget_lead = add_widget_lead(
                form.routing,
                firstName=first_name,
                lastName=last_name,
                emailAddress=email_address,
                phoneNumber=phone_number
            )
            from ..partnership.models import PartnershipAccount
            account = PartnershipAccount.query.filter(
                PartnershipAccount.id == lead.partnership_account_id).first()
            # Is it a Bandwidth or a Twilio number?
            if form.routing.inbound.type == 'tracking':
                client = bw_client(account.partnership_id, 'voice')
                log.info('Calling Bandwidth number...')
                BandwidthRouting(client).call_lead(form.routing, widget_lead)
            else:
                log.info('Calling Twilio number...')
                partnership = account.partnership
                subscription = account.subscription
                if not subscription:
                    subscription = partnership.subscription
                subaccount_sid = subscription.twilio_subaccount_sid
                client = subaccount_client(subaccount_sid, account.partnership_id)
                r = Routing(client)
                r.call_lead(form.routing, widget_lead)
    except Exception:
        log.error("Cannot add or call lead: {}".format(traceback.format_exc()))

    if service_providers:
        for provider in service_providers:
            from buyercall.blueprints.form_leads.form_tasks import form_external_api_request
            form_external_api_request.delay(lead.id, lead.partnership_account_id, provider.external_api_service_provider_id)

    return redirect_back()


def process_conditional_notification(lead_id, form_id, conditions):
    try:
        log.error('x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x')
        log.error('Processing Conditional Notification Detected')
        log.error('x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x-x')

        redis_db = redis.StrictRedis(
            host=app.config['REDIS_CONFIG_URL'],
            port=app.config['REDIS_CONFIG_PORT'],
            decode_responses=True
        )
        state_key = 'lead-' + str(lead_id) + '-form-state'
        post_key = 'lead-' + str(lead_id) + '-form-post'
        lead_state = str(redis_db.get(state_key))

        stored_posted_values = redis_db.get(post_key)
        condition_criteria_list = []
        condition_criteria_and_present = False
        condition_criteria_and = True
        condition_criteria_or = []
        retrieved_posted_values = {}
        posted_values = None
        send_ok_emails = False

        log.error('lead_id: ' + str(lead_id) + ' | state: ' + str(lead_state))

        if stored_posted_values:
            posted_values = json.loads(stored_posted_values)

        if posted_values:
            for posted_value in posted_values:
                value = posted_values[posted_value]
                if value:
                    key = posted_value
                    retrieved_posted_values[key] = value

        log.error('Posted fields and values: ')
        log.error(str(retrieved_posted_values))

        if conditions and lead_state:
            log.error('Conditions: ')
            log.error(str(conditions))
            then_subject_line = None
            if 'okSubjectLine' in conditions:
                then_subject_line = conditions['okSubjectLine']
            else_subject_line = None
            if 'nokSubjectLine' in conditions:
                else_subject_line = conditions['nokSubjectLine']
            then_emails = None
            if 'okEmailRecipients' in conditions:
                then_emails = conditions['okEmailRecipients']
            else_emails = None
            if 'nokEmailRecipients' in conditions:
                else_emails = conditions['nokEmailRecipients']
            then_adf_emails = None
            if 'okAdfEmailRecipients' in conditions:
                then_adf_emails = conditions['okAdfEmailRecipients']
            else_adf_emails = None
            if 'nokAdfEmailRecipients' in conditions:
                else_adf_emails = conditions['nokAdfEmailRecipients']
            condition_criteria = []
            if 'criteria' in conditions:
                condition_criteria = conditions['criteria']

            if conditions and (then_emails or else_emails or then_adf_emails or else_adf_emails):
                for condition in condition_criteria:
                    condition_pos = None
                    if 'pos' in condition:
                        condition_pos = condition['pos']
                    condition_option = None
                    if 'type' in condition:
                        condition_option = condition['type']
                    condition_form_field = None
                    if 'form_field' in condition:
                        condition_form_field = condition['form_field']
                    condition_operation = None
                    if 'operation' in condition:
                        condition_operation = condition['operation']
                    condition_value = None
                    if 'value' in condition:
                        condition_value = condition['value']
                    condition_valid = False

                    if condition_option == 'form_field':
                        if condition_form_field in retrieved_posted_values:
                            posted_value = retrieved_posted_values.get(condition_form_field, '')
                            log.error('Processing form_field condition. Posted value: ' + str(posted_value)
                                      + ', condition_form_field: ' + str(condition_form_field))
                            if posted_value == condition_value:
                                condition_valid = True
                            log.error('Condition match: ' + str(condition_valid))
                    if condition_option == 'submit':
                        log.error('Processing submit condition. Submit condition: ' + str(condition_value)
                                  + ', Actual submissions: ' + str(lead_state))
                        if condition_value == 'all':
                            condition_valid = True
                        elif condition_value == 'partial' and lead_state == 'partial':
                            condition_valid = True
                        elif condition_value == 'full' and lead_state == 'full':
                            condition_valid = True
                        log.error('Condition match: ' + str(condition_valid))
                    if condition_operation in ['and', 'initial']:
                        condition_criteria_and_present = True
                        if condition_valid == False:
                            condition_criteria_and = False
                    if condition_operation == 'or':
                        condition_criteria_or.append(condition_valid)

                if condition_criteria_and_present:
                    if condition_criteria_and or (condition_criteria_or and True in condition_criteria_or):
                        send_ok_emails = True
                elif condition_criteria_or and True in condition_criteria_or:
                    send_ok_emails = True


                if send_ok_emails:
                    log.error('Then - subject line: ' + str(then_subject_line))
                    log.error('Then - emails: ' + str(then_emails) + ' - adf emails: ' + str(then_adf_emails))
                    if then_emails or then_adf_emails:
                        from buyercall.blueprints.form_leads.form_tasks import \
                            form_lead_email_notifications_with_options
                        form_lead_email_notifications_with_options.delay(lead_id, False, None, True, then_subject_line,
                                                                         then_emails, then_adf_emails)
                else:
                    log.error('Else - subject line: ' + str(else_subject_line))
                    log.error('Else - emails: ' + str(else_emails) + ' - adf emails: ' + str(else_adf_emails))
                    if else_emails or else_adf_emails:
                        from buyercall.blueprints.form_leads.form_tasks import \
                            form_lead_email_notifications_with_options
                        form_lead_email_notifications_with_options.delay(lead_id, False, None, True, else_subject_line,
                                                                         else_emails, else_adf_emails)
            else:
                log.error('No notification condition emails set for form: ' + str(form_id) + '. For lead: '
                          + str(lead_id) + '. No notifications will be sent.')
        else:
            log.error('No notification condition criteria set for form: ' + str(form_id) + '. For lead: '
                      + str(lead_id) + '. No notifications will be sent.')
    except Exception:
        log.error(traceback.format_exc())


@form_leads.route('/form_leads/add/<int:contact_id_>/<int:form_id_>', methods=['GET'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def form_leads_new(contact_id_, form_id_):
    try:
        lead_field_list = []
        partnership_account_id = current_user.partnership_account_id
        partnership_account_group_viewing = current_user.is_viewing_partnership

        # Check if being viewed by super partner
        if partnership_account_group_viewing:
            partnership_account_id = current_user.get_user_viewing_partnership_account_id

        # Import the partnership account model
        from ..partnership.models import PartnershipAccount
        partnership_account = PartnershipAccount \
            .query \
            .filter(PartnershipAccount.id == partnership_account_id) \
            .first()

        if partnership_account.business_type != 'automotive':
            flash(_(
                'You do not have permission to perform this action'),
                'danger')
            return redirect(url_for('contacts.contact_list'))

        from buyercall.blueprints.contacts.models import Contact
        contact = Contact\
            .query\
            .filter(Contact.id == contact_id_,
                    Contact.partnership_account_id == partnership_account_id)\
            .first()

        if contact is None:
            flash(_(
                'Lead does not exist'),
                'danger')
            return redirect(url_for('contacts.contact_list'))

        form_fields = FormField.query.filter(
            FormField.form_id == form_id_
        ).all()

        if form_fields is None:
            flash(_(
                'Form does not exist'),
                'danger')
            return redirect(url_for('contacts.contact_list'))

        if len(form_fields) == 0:
            flash(_(
                'Form fields do not exist'),
                'danger')
            return redirect(url_for('contacts.contact_list'))

        field_definitions = ExternalFormFieldDefinition\
            .query\
            .filter()\
            .order_by('position')\
            .all()

        agents_list = Agent.query \
            .filter((Agent.partnership_account_id == partnership_account_id),
                    Agent.is_deactivated.is_(False)) \
            .order_by('firstname')
        agents = []
        for a in agents_list:
            a_name = a.firstname + ' ' + a.lastname
            agents.append(a)

        for field_definition in field_definitions:
            found_field = None

            for field in form_fields:
                if field_definition.check_field_id_match(field.field_id,
                                                         field_definition.field_id,
                                                         field_definition.old_field_id):
                    found_field = field
                    current_field_value = ''

                    if field_definition.field_id == 'firstname':
                        current_field_value = contact.firstname
                    elif field_definition.field_id == 'lastname':
                        current_field_value = contact.lastname
                    elif field_definition.field_id == 'phonenumber':
                        current_field_value = contact.phonenumber_1
                    elif field_definition.field_id == 'email':
                        current_field_value = contact.email
                    elif field_definition.field_id == 'address_1':
                        current_field_value = contact.address_1
                    elif field_definition.field_id == 'address_2':
                        current_field_value = contact.address_2
                    elif field_definition.field_id == 'state':
                        current_field_value = contact.state
                    elif field_definition.field_id == 'zip':
                        current_field_value = contact.zip
                    elif field_definition.field_id == 'city':
                        current_field_value = contact.city
                    elif field_definition.field_id == 'country':
                        current_field_value = contact.country
                    elif field_definition.field_id == 'birthday':
                        current_field_value = contact.birthday
                    elif field_definition.field_id == 'representative':
                        current_field_value = contact.agent_assigned

                    new_lead_field = {'field_id': field_definition.field_id,
                                     'field_value': current_field_value,
                                     'display_name': field_definition.display_name
                                     }
                    lead_field_list.append(new_lead_field)
                    break
            if found_field:
                form_fields.remove(field)

        return render_template(
            'form_leads_new.jinja2',
            contact_id=contact_id_,
            form_id=form_id_,
            form_field_list=lead_field_list,
            agents=agents
        )
    except Exception:
        log.error(traceback.format_exc())
        log.info('The current user: {}, does not have access to the form'.format(current_user.email))
        flash(_(
            'You do not have access to this form lead, it is associated with a different account. '
            'Please login with a different account or contact support.'),
              'danger')
        return redirect(url_for('contacts.contact_list'))


@form_leads.route('/form_leads/add/<int:contact_id_>/<int:form_id_>', methods=['POST'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def form_leads_new_post(contact_id_, form_id_):
    try:
        new_lead_id = None
        new_lead_fields = []
        partnership_account_id = current_user.partnership_account_id
        partnership_account_group_viewing = current_user.is_viewing_partnership

        # Check if being viewed by super partner
        if partnership_account_group_viewing:
            partnership_account_id = current_user.get_user_viewing_partnership_account_id

        # Import the partnership account model
        from ..partnership.models import PartnershipAccount
        partnership_account = PartnershipAccount \
            .query \
            .filter(PartnershipAccount.id == partnership_account_id) \
            .first()

        if partnership_account.business_type != 'automotive':
            flash(_(
                'You do not have permission to perform this action'),
                'danger')
            return redirect(url_for('contacts.contact_list'))

        from ..contacts.models import Contact
        contact = Contact\
            .query\
            .filter(Contact.id == contact_id_)\
            .first()

        if contact is None:
            flash(_(
                'Lead does not exist'),
                'danger')
            return redirect(url_for('contacts.contact_list'))

        form_fields = FormField.query.filter(
            FormField.form_id == form_id_
        ).all()

        if form_fields is None or len(form_fields) == 0:
            flash(_(
                'Form does not exist'),
                'danger')
            return redirect(url_for('contacts.contact_list'))

        field_definitions = ExternalFormFieldDefinition\
            .query\
            .filter()\
            .order_by('position')\
            .all()

        for field_definition in field_definitions:
            found_field = None

            for field in form_fields:
                if field_definition.check_field_id_match(field.field_id,
                                                         field_definition.field_id,
                                                         field_definition.old_field_id):

                    found_field = field
                    new_field_id = field.field_id
                    new_field_value = request.form.get('field-' + field_definition.field_id, '')

                    if new_field_value:
                        field.field_value = new_field_value

                        if field.field_id == 'birthday' or field.field_id == 'datepicker' or field.field_id == 'dobfield':
                            contact.birthday = field.field_value
                        elif field.field_id == 'representative' or field.field_id == 'repfield':
                            contact.agent_assigned = field.field_value
                        elif field.field_id == 'ssn' or field.field_id == 'ssnfield' or field.field_id == 'cumemberfield' \
                                or field.field_id == 'co_applicant_ssn' or field.field_id == 'cosignerssnfield':
                            if len(field.field_value) > 9:
                                new_field_value = field.field_value.replace('-', '').replace(' ', '').replace('#', '')
                        elif field.field_id == 'firstname' or field.field_id == 'firstnamefield':
                            contact.firstname = field.field_value
                        elif field.field_id == 'lastname' or field.field_id == 'lastnamefield':
                            contact.lastname = field.field_value
                        elif field.field_id == 'email' or field.field_id == 'emailfield':
                            contact.email = field.field_value
                        elif field.field_id == 'address_1' or field.field_id == 'streetfield':
                            contact.address_1 = field.field_value
                        elif field.field_id == 'address_2':
                            contact.address_2 = field.field_value
                        elif field.field_id == 'city' or field.field_id == 'citytfield' or field.field_id == 'cityfield':
                            contact.city = field.field_value
                        elif field.field_id == 'state' or field.field_id == 'statefield':
                            contact.state = field.field_value
                        elif field.field_id == 'zip' or field.field_id == 'zipfield':
                            new_field_value = new_field_value.replace("-", "")

                            if len(new_field_value) > 5:
                                new_field_value = new_field_value[: 5]

                            contact.zip = new_field_value
                        elif field.field_id == 'country' or field.field_id == 'countryfield':
                            contact.country = field.field_value

                        new_field_value = encrypt_value(new_field_value)
                    else:
                        new_field_value = ''

                    new_lead_field = LeadField(field_id=new_field_id, field_value=new_field_value)
                    new_lead_fields.append(new_lead_field)
                    break
            if found_field:
                form_fields.remove(field)

        new_lead = Lead(contact_id=contact_id_,
                        email_sent=False,
                        form_id=form_id_,
                        partnership_account_id=partnership_account_id,
                        fields=new_lead_fields)

        db.session.add(new_lead)
        db.session.commit()

        new_lead_id = new_lead.id

    except Exception:
        log.error(traceback.format_exc())
        db.session.rollback()
        flash(_("Error saving form information."), 'danger')
        return redirect(url_for('form_leads.form_leads_new', contact_id_=contact_id_, form_id_=form_id_))

    flash(_('The form lead has been successfully saved.'), 'success')
    return redirect(url_for('form_leads.form_leads_edit', id_=new_lead_id))


@form_leads.route('/form_leads/<int:id_>', methods=['GET'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def form_leads_edit(id_):
    try:
        partnership_account_id = current_user.partnership_account_id
        partnership_account_group_viewing = current_user.is_viewing_partnership

        # Check if being viewed by super partner
        if partnership_account_group_viewing:
            partnership_account_id = current_user.get_user_viewing_partnership_account_id

        # Import the partnership account model
        from ..partnership.models import PartnershipAccount
        partnership_account = PartnershipAccount \
            .query \
            .filter(PartnershipAccount.id == partnership_account_id) \
            .first()

        field_definitions = ExternalFormFieldDefinition.query.filter().order_by('position').all()

        lead = Lead.query.filter(
            Lead.partnership_account_id == partnership_account_id,
            Lead.id == id_
        ).one()

        form_id = lead.form_id

        form_fields = FormField.query.filter(
            lead.form_id == FormField.form_id
        ).subquery()

        fields = LeadField.query.filter(
            LeadField.lead_id == lead.id
        ).outerjoin(
           (form_fields, LeadField.field_id == form_fields.c.field_id)
        ).order_by('position').with_entities(
            form_fields.c.display_name,
            LeadField.field_id,
            LeadField.field_value
        ).all()

        # Before decrypting a empty string is assign to the values in case the field is not used
        # in a form.
        # ss_number = ''
        # co_ss_number = ''
        # drivers_license = ''
        # co_drivers_license = ''
        # dob_date = ''
        # co_dob_date = ''
        agents = ''
        encrypt_key = app.config['CRYPTO_SECRET_KEY']
        cipher = AESCipher(encrypt_key)

        # In the next section the form lead fields are retrieved and each sensitive field is
        # retrieved using an if statement
        personal_field_list = []
        application_field_list = []
        address_field_list = []
        financial_field_list = []
        vehicle_field_list = []
        family_field_list = []
        employment_field_list = []
        miscellaneous_field_list = []

        for field_definition in field_definitions:
            for field in fields:
                if field_definition.check_field_id_match(field.field_id, field_definition.field_id, field_definition.old_field_id):
                    field_id = field.field_id
                    field_value = None
                    field_display_name = field_definition.display_name
                    field_position = field_definition.position

                    if field.field_id == 'representative' or field.field_id == 'repfield':
                        agents_list = Agent.query\
                            .filter((Agent.partnership_account_id == partnership_account_id),
                                    Agent.is_deactivated.is_(False))\
                            .order_by('firstname')
                        agents = []
                        for a in agents_list:
                            a_name = a.firstname + ' ' + a.lastname
                            if a_name != field.field_value:
                                agents.append(a)

                    if field.field_value:
                        try:
                            field_value = cipher.decrypt(field.field_value)
                        except TypeError:
                            log.error('Error decrypting field value for ' + field.field_id + '. Possibly an older form lead. Defaulting to: ' + field.field_value + '. Lead id: ' + str(lead.id))
                            field_value = field.field_value
                        except ValueError:
                            log.error('Error decrypting field value for ' + field.field_id + '. Possibly an older form lead. Defaulting to: ' + field.field_value + '. Lead id: ' + str(lead.id))
                            field_value = field.field_value
                    else:
                        field_value = ''

                    updated_field = {'field_id': field_id,
                                     'field_value': field_value,
                                     'display_name': field_display_name,
                                     'category': field_definition.category,
                                     'position': field_position
                                     }
                    if field_definition.category == 'personal':
                        personal_field_list.append(updated_field)
                    if field_definition.category == 'application':
                        application_field_list.append(updated_field)
                    elif field_definition.category == 'address':
                        address_field_list.append(updated_field)
                    elif field_definition.category == 'financial':
                        financial_field_list.append(updated_field)
                    elif field_definition.category == 'vehicle':
                        vehicle_field_list.append(updated_field)
                    elif field_definition.category == 'family':
                        family_field_list.append(updated_field)
                    elif field_definition.category == 'employment':
                        employment_field_list.append(updated_field)

                    fields.remove(field)
                    break

        if len(fields) > 0:
            for field in fields:
                field_id = field.field_id
                field_value = None
                field_display_name = field.field_id
                field_position = 0

                if field.field_value:
                    try:
                        field_value = cipher.decrypt(field.field_value)
                    except TypeError:
                        log.error(
                            'Error decrypting field value for ' + field.field_id + '. Possibly an older form lead. Defaulting to: ' + field.field_value + '. Lead id: ' + str(
                                lead.id))
                        field_value = field.field_value
                    except ValueError:
                        log.error(
                            'Error decrypting field value for ' + field.field_id + '. Possibly an older form lead. Defaulting to: ' + field.field_value + '. Lead id: ' + str(
                                lead.id))
                        field_value = field.field_value
                else:
                    field_value = ''

                updated_field = {'field_id': field_id,
                                 'field_value': field_value,
                                 'display_name': field_display_name,
                                 'category': 'miscellaneous',
                                 'position': field_position
                                 }

                miscellaneous_field_list.append(updated_field)

        from ..user.models import UserExternalApiAutoPostTie
        service_provider_list = UserExternalApiAutoPostTie\
            .get_allowed_service_providers_with_form_log_count(current_user.id,
                                                               partnership_account_id,
                                                               lead.id)

        supervisor_user_business_type = None
        from buyercall.lib.supervisor_manager import current_supervisor_user
        if current_supervisor_user and current_supervisor_user.is_authenticated:
            partner_id = current_supervisor_user.partnership_id

            if partner_id:
                from ..partnership.models import Partnership
                supervisor_user_partnership = Partnership.query.filter(Partnership.id == partner_id).first()
                if supervisor_user_partnership and supervisor_user_partnership.business_type:
                    supervisor_user_business_type = supervisor_user_partnership.business_type
            elif current_supervisor_user.role == 'limitsysadmin':
                supervisor_user_business_type = 'automotive'

        ActivityLogs.add_log(current_user.id, ActivityType.PAGE, ActivityName.VIEW, request, id_)

        return render_template(
            'form_leads_edit.jinja2',
            lead=lead,
            supervisor_user_business_type=supervisor_user_business_type,
            business_type=partnership_account.business_type,
            form_id=form_id,
            personal_fields=personal_field_list,
            application_fields=application_field_list,
            address_fields=address_field_list,
            financial_fields=financial_field_list,
            vehicle_fields=vehicle_field_list,
            family_fields=family_field_list,
            employment_fields=employment_field_list,
            miscellaneous_fields=miscellaneous_field_list,
            agents=agents,
            list=service_provider_list
        )
    except Exception:
        log.error(traceback.format_exc())
        log.info('The current user: {}, does not have access to the form'.format(current_user.email))
        flash(_(
            'You do not have access to this form lead, it is associated with a different account. '
            'Please login with a different account or contact support.'),
              'danger')
        return redirect(url_for('contacts.contact_list'))


@form_leads.route('/form_leads/<int:id_>', methods=['POST'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def form_leads_update(id_):
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    lead = Lead.query.filter(
        Lead.partnership_account_id == partnership_account_id,
        Lead.id == id_
    ).one()

    form_id = lead.form_id

    form_fields = FormField.query.filter(
        lead.form_id == FormField.form_id
    ).subquery()

    fields = LeadField.query.filter(
        LeadField.lead_id == id_
    ).outerjoin(
       (form_fields, LeadField.field_id == form_fields.c.field_id)
    ).order_by('position').all()

    encrypt_key = app.config['CRYPTO_SECRET_KEY']
    cipher = AESCipher(encrypt_key)
    # Get the contact associated with the form lead
    from ..contacts.models import Contact
    contact = Contact.query.filter(Contact.id == lead.contact_id).first()

    for field in fields:
        new_name = request.form.get('field-' + field.field_id, '')
        if new_name:
            field.field_value = new_name

            if field.field_id == 'birthday' or field.field_id == 'datepicker' or field.field_id == 'dobfield':
                contact.birthday = field.field_value
            elif field.field_id == 'representative' or field.field_id == 'repfield':
                contact.agent_assigned = field.field_value
            elif field.field_id == 'ssn' or field.field_id == 'ssnfield' or field.field_id == 'cumemberfield' \
                    or field.field_id == 'co_applicant_ssn' or field.field_id == 'cosignerssnfield':
                if len(field.field_value) > 9:
                    field.field_value = field.field_value.replace('-', '').replace(' ', '').replace('#', '')
            elif field.field_id == 'firstname' or field.field_id == 'firstnamefield':
                contact.firstname = field.field_value
            elif field.field_id == 'lastname' or field.field_id == 'lastnamefield':
                contact.lastname = field.field_value
            elif field.field_id == 'email' or field.field_id == 'emailfield':
                contact.email = field.field_value
            elif field.field_id == 'address_1' or field.field_id == 'streetfield':
                contact.address_1 = field.field_value
            elif field.field_id == 'address_2':
                contact.address_2 = field.field_value
            elif field.field_id == 'city' or field.field_id == 'citytfield' or field.field_id == 'cityfield':
                contact.city = field.field_value
            elif field.field_id == 'state' or field.field_id == 'statefield':
                contact.state = field.field_value
            elif field.field_id == 'zip' or field.field_id == 'zipfield':
                contact.zip = field.field_value
            elif field.field_id == 'country' or field.field_id == 'countryfield':
                contact.country = field.field_value

            field.field_value = encrypt_value(field.field_value)
        else:
            field.field_value = ''

    lead.updated_on = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
    contact.updated_on = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)

    try:
        db.session.commit()
        ActivityLogs.add_log(current_user.id, ActivityType.PAGE, ActivityName.EDIT, request, id_)
        flash(_('The form lead has been updated successfully.'), 'success')
        return redirect(url_for('form_leads.leads_list', form_id=form_id))
    except Exception:
        log.error(traceback.format_exc())
        db.session.rollback()
        flash(_("Error saving form information."), 'danger')

    return redirect(url_for('form_leads.form_leads_edit', id_=id_))


@form_leads.route('/form_leads/<int:lead_id>/leads/doc', methods=['POST'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def leads_doc(lead_id):
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    """Return server side data."""
    lead = Lead.query.filter(
        Lead.partnership_account_id == partnership_account_id,
        Lead.id == lead_id
    ).first()
    if not lead:
        return 404, 'Form not found'

    form_fields = FormField.query.filter(
        lead.form_id == FormField.form_id
    ).subquery()

    fields = LeadField.query.filter(
        LeadField.lead_id == lead_id
    ).outerjoin(
       (form_fields, LeadField.field_id == form_fields.c.field_id)
    ).order_by('position').with_entities(
        form_fields.c.display_name,LeadField.field_id,
        LeadField.field_value
    ).all()

    heading = request.form['heading']
    today = datetime.date.today()
    today_str = today.strftime('%b, %d %Y')
    notes = request.form['notes']
    footer = request.form['footer']
    document = Document()
    document.add_heading(heading, level=1)
    document_date = document.add_paragraph()
    document_date.add_run('date: ').bold = True
    document_date.add_run(today_str)
    document_date.alignment = WD_PARAGRAPH_ALIGNMENT.RIGHT
    document.add_heading('Form Lead Information:', level=2)
    document.add_paragraph()
    encrypt_key = app.config['CRYPTO_SECRET_KEY']
    cipher = AESCipher(encrypt_key)

    for field in fields:
        if field.field_id != 'lead-id-field' and field.field_id != 'lead-id':
            document_fields = document.add_paragraph()
            if field.display_name:
                document_fields.add_run("{}: ".format(field.display_name)).bold = True
            else:
                document_fields.add_run(str(field.field_id) + ": ").bold = True

            try:
                if field.field_value is not None:
                    field.field_value = cipher.decrypt(field.field_value)
            except TypeError:
                log.error('Error decrypting field value. Possibly an older form lead. Lead id: ' + str(lead.id))
            except ValueError:
                log.error('Error decrypting field value. Possibly an older form lead. Lead id: ' + str(lead.id))

            document_fields.add_run(field.field_value)

    document.add_heading('Additional Notes:',level=2)
    document.add_paragraph()
    document.add_paragraph(notes)
    document.add_paragraph()
    footer_text = document.add_paragraph()
    footer_text.add_run(footer).bold = True

    f = StringIO()
    document.save(f)
    f.seek(0)
    return send_file(f, as_attachment=True, attachment_filename='form_lead.docx')


# Function used to create PDF Doc for lead details
@form_leads.route('/form_leads/<int:lead_id>/leads/pdf')
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def leads_pdf(lead_id):
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership
    browser_used = request.user_agent.browser
    vehicle_info = False
    contact_vehicle_info = False
    contact_vehicle = None

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    # Return lead data based on lead_id
    lead = Lead.query.filter(
        Lead.partnership_account_id == partnership_account_id,
        Lead.id == lead_id
    ).first()
    if lead:
        try:
            # Return contact linked to lead
            from ..contacts.models import Contact, ContactVehicle
            contact = Contact\
                .query\
                .filter(Contact.id == lead.contact_id,
                        Contact.partnership_account_id == lead.partnership_account_id)\
                .first()
            if contact:
                contact_vehicle = ContactVehicle.query.filter(ContactVehicle.contact_id == lead.contact_id).first()

            # Return form used to capture lead
            form = Form.query.filter(Form.id == lead.form_id).first()

            # Return the predefined form fields
            form_fields = FormField.query.filter(
                lead.form_id == FormField.form_id
            ).subquery()

            # Return the form field values for the specific lead
            fields = LeadField.query.filter(
                LeadField.lead_id == lead_id
            ).outerjoin(
                (form_fields, LeadField.field_id == form_fields.c.field_id)
            ).order_by('position').with_entities(
                form_fields.c.display_name,
                LeadField.field_id,
                LeadField.field_value
            ).all()

            ss_number = ''
            dob_date = ''
            co_ssn = ''
            co_dob = ''
            decrypted_fields = []

            for field in fields:
                decrypted_field = LeadField(field.field_id, decrypt_value(field.field_value))

                if contact:
                    if contact.firstname and (decrypted_field.field_id == 'firstname' or decrypted_field.field_id == 'firstnamefield'):
                        decrypted_field.field_value = contact.firstname
                    elif contact.lastname and (decrypted_field.field_id == 'lastname' or decrypted_field.field_id == 'lastnamefield'):
                        decrypted_field.field_value = contact.lastname
                    elif contact.email and (decrypted_field.field_id == 'email' or decrypted_field.field_id == 'emailfield'):
                        decrypted_field.field_value = contact.email
                    elif contact.city and (decrypted_field.field_id == 'city' or decrypted_field.field_id == 'cityfield' or field.field_id == 'citytfield'):
                        decrypted_field.field_value = contact.city
                    elif contact.state and (decrypted_field.field_id == 'state' or decrypted_field.field_id == 'statefield'):
                        decrypted_field.field_value = contact.state
                    elif contact.zip and (decrypted_field.field_id == 'zip' or decrypted_field.field_id == 'zipfield'):
                        decrypted_field.field_value = contact.zip
                    elif contact.address_1 and (decrypted_field.field_id == 'address_1' or decrypted_field.field_id == 'streetfield'):
                        decrypted_field.field_value = contact.address_1
                    elif contact.address_2 and (decrypted_field.field_id == 'address_2'):
                        decrypted_field.field_value = contact.address_2

                decrypted_fields.append(decrypted_field)

                if field.field_id == 'cumemberfield' or field.field_id == 'ssnfield' or field.field_id == 'ssn':
                    ss_number = decrypted_field.field_value
                    if len(ss_number) > 16:
                        ss_number = 'unavailable'
                elif field.field_id == 'datepicker' or field.field_id == 'dobfield' or field.field_id == 'birthday':
                    dob_date = decrypted_field.field_value
                    if len(dob_date) > 16:
                        dob_date = 'unavailable'
                elif field.field_id == 'cosignerssnfield' or field.field_id == 'co_applicant_ssn':
                    co_ssn = decrypted_field.field_value
                    if len(co_ssn) > 16:
                        co_ssn = 'unavailable'
                elif field.field_id == 'cosignerdateofbirthfield' or field.field_id == 'co_applicant_birthday':
                    co_dob = decrypted_field.field_value
                    if len(co_dob) > 16:
                        co_dob = 'unavailable'

            finance_fields = LeadField.query \
                .filter(and_(LeadField.lead_id == lead_id,
                             LeadField.field_id.in_(['cosignerbankruptcyfield', 'co_applicant_bankruptcy_declaration',
                                                     'cosignerrepossessionfield', 'co_applicant_repossession_declaration',
                                                     'cosigneramountowedfield', 'co_applicant_load_amount_outstanding',
                                                     'cosignerbanknamefield', 'co_applicant_bank_name',
                                                     'cosignercheckingbalancefield', 'co_applicant_checking_balance'
                                                     'cosignersavingbalancefield', 'co_applicant_saving_balance',
                                                     'cosignerchildcarepaymentfield', 'co_applicant_child_care_payment',
                                                     'cosignerutilitiespaymentfield', 'co_applicant_utilities_payment',
                                                     'cosignerdownpaymentfield', 'co_applicant_down_payment_amount',
                                                     'cosignermaintenancepaymentfield', 'co_applicant_maintenance_payment']))).first()

            reference_fields = LeadField.query \
                .filter(and_(LeadField.lead_id == lead_id,
                             LeadField.field_id.in_(['firstreferencenamefield', 'first_reference_name']))).first()

            additional_reference_fields = LeadField.query \
                .filter(and_(LeadField.lead_id == lead_id,
                             LeadField.field_id.in_(['thirdreferencenamefield', 'third_reference_name']))).first()

            vehicle_fields = LeadField.query \
                .filter(and_(LeadField.lead_id == lead_id,
                             (or_(LeadField.field_id.in_(['vehiclemileagefield', 'vehicle_mileage']),
                                  LeadField.field_id.in_(['vehiclepurchasepricefield', 'vehicle_purchase_price']),
                                  LeadField.field_id.in_(['vehiclemakefield', 'vehicle_make']),
                                  LeadField.field_id.in_(['vinfield', 'vin']),
                                  LeadField.field_id.in_(['tradeinvehicletrimfield', 'trade_in_vehicle_trim']),
                                  LeadField.field_id.in_(['tradeinvehiclemakefield', 'trade_in_vehicle_make']),
                                  LeadField.field_id.in_(['listingurlfield', 'listing_url']),
                                  LeadField.field_id.in_(['tradeinvehiclevinfield', 'trade_in_vehicle_vin']))))).first()

            vehicle_seller_fields = LeadField.query \
                .filter(and_(LeadField.lead_id == lead_id,
                             (or_(LeadField.field_id.in_(['sellerbusinessnamefield', 'seller_business_name']),
                                  LeadField.field_id.in_(['sellercontactnamefield', 'seller_contact_name']))))).first()

            if finance_fields:
                fin_info = True
            else:
                fin_info = False

            if reference_fields:
                reference_info = True
            else:
                reference_info = False

            if additional_reference_fields:
                additional_reference_info = True
            else:
                additional_reference_info = False

            if contact_vehicle:
                contact_vehicle_info = True
            else:
                if vehicle_fields:
                    vehicle_info = True
                else:
                    vehicle_info = False

            if vehicle_seller_fields:
                vehicle_seller_info = True
            else:
                vehicle_seller_info = False

            # Return the partnership account name
            from ..partnership.models import PartnershipAccount
            account = PartnershipAccount.query.filter(PartnershipAccount.id == lead.partnership_account_id).first()

            # Return current day's date
            today = datetime.date.today()
            today_str = today.strftime('%b, %d %Y')

            # Retrieve the credit score for the form lead
            # Get the credit score for the leads in the form leads table
            from buyercall.blueprints.contacts.models import CreditReports
            credit_report = CreditReports.query \
                .filter(and_(CreditReports.contact_id == lead.contact_id,
                             CreditReports.partnership_account_id == partnership_account_id,
                             CreditReports.is_successful.is_(True))).order_by(CreditReports.created_on.desc()).first()

            if credit_report:
                credit_score = decrypt_value(credit_report.credit_score)
            else:
                credit_score = 'Unknown'

            from .form_tasks import generate_pdf
            ActivityLogs.add_log(current_user.id, ActivityType.DATA, ActivityName.PDF, request, lead_id)
            return generate_pdf(lead_id, account.name, form.display_name, decrypted_fields, today_str, ss_number, dob_date,
                                co_ssn, co_dob, fin_info, reference_info, vehicle_info, contact_vehicle_info,
                                vehicle_seller_info, credit_score, browser_used, lead.contact_id, contact_vehicle,
                                additional_reference_info)
        except:
            log.error(f'Error generating pdf for id {lead_id} : {traceback.format_exc()}')
            flash(_('Something went wrong trying to retrieve the lead. Please contact Support for assistance.'),
                  'danger')
            return redirect(url_for('contacts.contact_list'))

    else:
        log.error('Unable to detect the form lead id: {}'.format(lead_id))
        flash(_('Something went wrong trying to retrieve the lead. Please contact Support for assistance.'), 'danger')
        return redirect(url_for('contacts.contact_list'))


@form_leads.route('/api/form_leads/<int:id_>/autopay', methods=['POST'])
@login_required
@role_required('admin', 'partner')
def form_leads_autopay_request_action(id_):
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    return form_leads_autopay_request(id_, partnership_account_id)


def form_leads_autopay_request(lead, lead_partnership_id = None):

    requiredFields = ['loantypefield', # APPLICATION OBJECT
                     #'applicationSource'
                      'firstnamefield', 'lastnamefield', 'emailfield', 'privacypolicyfield', #APPLICANT OBJECT
                      'phonefield', # PHONE OBJECT
                      'streetfield', 'zipfield', #'current', # ADDRESS OBJECT
                      'incomefield', #'payFrequency', 'current', # EMPLOYMENT OBJECT
                      'vehicleyearfield', 'mileagefield' # 'payofffield', 'grossAmt'  VEHICLE OBJECT
                      ]

    form_fields = FormField.query.filter(
        lead.form_id == FormField.form_id
    ).subquery()

    lead_fields = LeadField.query.filter(
        LeadField.lead_id == lead.id
    ).outerjoin(
        (form_fields, LeadField.field_id == form_fields.c.field_id)
    ).with_entities(
        form_fields.c.display_name,
        LeadField.field_id,
        LeadField.field_value
    ).all()

    form_fields_list = list(name.field_id for name in lead_fields)

    mapped_fields = {}
    for u in lead_fields:
        mapped_fields[u.field_id] = u.field_value

    # checking if all required fields present
    has_all_fields_vals = list(field in form_fields_list and mapped_fields[field] != '' for field in requiredFields)
    has_all_fields = all(has_all_fields_vals)
    if not has_all_fields:
        missing_fields = []
        for idx, val in enumerate(has_all_fields_vals):
            if not has_all_fields_vals[idx]:
                missing_fields.append(next((x for x in lead_fields if x.field_id == requiredFields[idx]), '').display_name)
        return 'Missing required field(s): ' + ', '.join(missing_fields), 400

    if not isfloat(mapped_fields.get('incomefield')):
        return 'Can not parse required field(s): ' + next((x for x in lead_fields if x.field_id == 'incomefield'), '').display_name, 400

    if not isint(mapped_fields.get('vehicleyearfield')):
        return 'Can not parse required field(s): ' + next((x for x in lead_fields if x.field_id == 'vehicleyearfield'), '').display_name, 400

    if not isint(mapped_fields.get('mileagefield')):
        return 'Can not parse required field(s): ' + next((x for x in lead_fields if x.field_id == 'mileagefield'), '').display_name, 400

    # Decrypt some fields for the API
    encrypt_key = app.config['CRYPTO_SECRET_KEY']
    cipher = AESCipher(encrypt_key)
    if mapped_fields.get('datepicker'):
        dob_date =  mapped_fields.get('datepicker', '')
        decrypt_dob = cipher.decrypt(dob_date)
    if mapped_fields.get('cumemberfield'):
        ss_no = mapped_fields.get('cumemberfield', '')
        decrypt_ss = cipher.decrypt(ss_no)

    # forming request body
    application = {}

    application['applicationSource'] = app.config['AUTOPAY_APPSOURCE']
    application['loanType'] = mapped_fields['loantypefield'].replace("Refinance Auto", "Refinance").replace('Lease Purchase', 'Purchase')

    application['applicant'] = {
        'firstName': mapped_fields['firstnamefield'],
        'lastName': mapped_fields['lastnamefield'],
        'emailAddress': mapped_fields['emailfield'],
        'agreeHard': True if mapped_fields.get('privacypolicyfield', 'No').lower() == 'yes' else False,
        'dob': decrypt_dob.replace("/", "-"),
        'ssn': re.sub(r'[-\s.]', "", decrypt_ss),
        'phones': [{'phoneNumber': mapped_fields['phonefield']}],
        'addresses': [{'street1': mapped_fields['streetfield'], 'zip': mapped_fields['zipfield'], 'current': True}],
        'employments': [{
                            'salaryAmount': float(mapped_fields.get('incomefield')) if isfloat(mapped_fields.get('incomefield')) else None,
                            'payFrequency': 'Monthly',
                            'companyName': mapped_fields.get('employerfield', ''),
                            'jobTitle': mapped_fields.get('titlefield'),
                            'current': True
                         }],
        'otherIncome': {
            'incomeSource': 'None' if mapped_fields.get('additionalincomesourcefield', '') == '' else mapped_fields.get('additionalincomesourcefield'),
            'incomeAmount': float(mapped_fields.get('additionalincomefield')) if isfloat(mapped_fields.get('additionalincomefield')) else None,
            'payFrequency': 'Monthly' # ask Herman!
        },
        'identification': {
            'type': 'Driver License',
            'state': mapped_fields.get('driverlicensetatefield', ''),
            'number': mapped_fields.get('driverlicensefield', '')
        }
    }

    grossAmt = mapped_fields.get('grossAmt', '')

    application['vehicle'] = {
        'vin': mapped_fields.get('vinfield'),
        'new': True if mapped_fields.get('vehicletypefield', 'No').lower() == 'new' else False,
        'grossAmt': float(grossAmt) if isfloat(grossAmt) else None,
        'payoffAmt': float(mapped_fields.get('payofffield')) if isfloat(mapped_fields.get('payofffield')) else None,
        'makeName': mapped_fields.get('vehiclemakefield'),
        'modelName': mapped_fields.get('vehiclemodelfield'),
        'vehicleYear': int(mapped_fields['vehicleyearfield']) if isint(mapped_fields.get('vehicleyearfield')) else None,
        'mileage': int(mapped_fields['mileagefield']) if isint(mapped_fields.get('mileagefield')) else None
    }

    if mapped_fields.get('applicationtypefield', '').lower() == 'joint':
        application['coApplicant'] = {
            'firstName': mapped_fields.get('cosignerfirstnamefield', ''),
            'lastName': mapped_fields.get('cosignerlastnamefield', ''),
            'emailAddress': mapped_fields['cosigneremailfield'],
            'agreeHard': True if mapped_fields.get('privacypolicyfield', 'No').lower() == 'yes' else False,
            'dob': mapped_fields.get('cosignerdateofbirthfield', '').replace("/", "-"),
            'ssn': re.sub(r'[-\s.]', "", mapped_fields.get('cosignerssnfield', '')),
            'phones': [{'phoneNumber': mapped_fields['cosignerphonefield']}],
            'addresses': None,
            'employments': [{
                'salaryAmount': float(mapped_fields.get('cosignermonthlyincomefield')) if isfloat(
                    mapped_fields.get('cosignermonthlyincomefield')) else None,
                'payFrequency': 'Monthly',
                'companyName': mapped_fields.get('cosigneremployerfield', ''),
                'jobTitle': mapped_fields.get('cosignerpositionfield'),
                'current': True
            }]
        }

    # forming API request to autopay
    source_key = app.config['AUTOPAY_SOURCE_KEY']
    secret_key = app.config['AUTOPAY_SECRET_KEY']

    loanType = application['loanType']
    emailAddress = next((x for x in lead_fields if x.field_id == 'emailfield'), '').field_value
    hashed = hmac.new(secret_key, b"{}{}".format(loanType, emailAddress), sha1)
    signature = hashed.hexdigest()

    url = app.config['AUTOPAY_API_URL'].format(source_key, signature)
    headers = {'content-type': 'application/json'}
    response = requests.post(url, data=json.dumps(application), headers=headers)

    if response.status_code == 200:
        result = json.loads(response.content.decode())
        lead.external_api_lead_id = result['autopayNumber']
        db.session.commit()

    return response.content.decode(), response.status_code


def send_notifications(lead, error_occurred=False, external_provider_name='', is_conditional=False,
                       conditional_subject_line=None, conditional_emails=None, conditional_adf_emails=None):

    from ..widgets.models import split_emails
    from ..partnership.models import Partnership, PartnershipAccount

    partner_account = PartnershipAccount.query.filter(PartnershipAccount.id == lead.partnership_account_id).first()
    partner = Partnership.query.filter(Partnership.id == partner_account.partnership_id).first()
    
    lead_notifications = Form.query.filter(Form.id == lead.form_id).first()
    lead_fields = LeadField.query.filter(LeadField.lead_id == lead.id).all()

    ad_source = ''
    ad_medium = ''
    ad_campaign = ''
    ad_content = ''
    for lead_field in lead_fields:
        location = None
        full_vehicle = None
        if lead_field.field_id == 'firstname' or lead_field.field_id == 'firstnamefield':
            firstname = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'lastname' or lead_field.field_id == 'lastnamefield':
            lastname = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'phone' or lead_field.field_id == 'phonefield' \
                or lead_field.field_id == 'phonenumber':
            phone = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'email' or lead_field.field_id == 'emailfield':
            email = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'vehicle_of_interest' or lead_field.field_id == 'vehiclefield':
            full_vehicle = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'location' or lead_field.field_id == 'locationfield':
            location = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'vehicle_year' or lead_field.field_id == 'vehicleyearfield':
            vehicle_year = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'vehicle_make' or lead_field.field_id == 'vehiclemakefield':
            vehicle_make = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'vehicle_model' or lead_field.field_id == 'vehiclemodelfield':
            vehicle_model = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'vehicle_trim' or lead_field.field_id == 'vehicletrimfield':
            vehicle_trim = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'vin' or lead_field.field_id == 'vinfield':
            vehicle_vin = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'stock_number' or lead_field.field_id == 'stockfield':
            vehicle_stock = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'address_1' or lead_field.field_id == 'streetfield':
            street = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'city' or lead_field.field_id == 'cityfield':
            city = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'state' or lead_field.field_id == 'statefield':
            state = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'zip' or lead_field.field_id == 'zipfield':
            zip = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'ad_source' or lead_field.field_id == 'adsourcefield':
            ad_source = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'ad_medium' or lead_field.field_id == 'admediumfield':
            ad_medium = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'ad_campaign' or lead_field.field_id == 'adcampaignfield':
            ad_campaign = decrypt_value(lead_field.field_value)

        if lead_field.field_id == 'ad_content' or lead_field.field_id == 'adcontentfield':
            ad_content = decrypt_value(lead_field.field_value)

    # LEAD INFORMATION FOR THE EMAIL
    ctx = vars(lead)

    try:
        if firstname:
            ctx['firstname'] = firstname
        else:
            ctx['firstname'] = '(Verify through the link below)'
            log.info('The firstname field is empty or non existing')
    except Exception:
        log.info('The firstname field was not used')
    try:
        if lastname:
            ctx['lastname'] = lastname
        else:
            ctx['lastname'] = ''
    except Exception:
        log.info('The lastname field was not used')
    try:
        if phone:
            ctx['phonenumber'] = phone
        else:
            ctx['phonenumber'] = ''
    except Exception:
        log.info('The phone field was not used')
    try:
        if email:
            ctx['email'] = email
        else:
            ctx['email'] = ''
    except Exception:
        log.info('The email field was not used')

    ctx['form_name'] = lead_notifications.display_name
    eastern = pytz.timezone('US/Eastern')
    ctx['created_on'] = lead.created_on.astimezone(eastern).strftime('%c')
    ctx['updated_on'] = lead.updated_on.astimezone(eastern).strftime('%c')
    ctx['partner_logo'] = partner.logo
    ctx['company'] = partner.name

    if partner.partner_url:
        ctx['lead_url'] = partner.partner_url + '/form_leads/' + str(lead.id)
        ctx['lead_contact_url'] = partner.partner_url + '/contacts/contact/' + str(lead.contact_id)
    else:
        ctx['lead_url'] = url_for('form_leads.form_leads_edit', id_=lead.id, _external=True)
        ctx['lead_contact_url'] = url_for('contacts.contact_lead_page', id=lead.contact_id, _external=True)

    ctx['lead_log_url'] = url_for('form_leads.form_edit', id_=lead.form_id, _external=True) + '?tab=logs'

    if error_occurred:
        subject_text = 'BuyerCall Automated Lead Export Error'
    else:
        if is_conditional:
            subject_text = conditional_subject_line
            ctx['form_name'] = subject_text
            if not subject_text:
                subject_text = 'Lead Notification'
        else:
            if lead_notifications.email_subject != '':
                subject_text= lead_notifications.email_subject
            else:
                subject_text = 'New BuyerCall Form Lead Notification'

    # SPLIT THE STRING OF EMAIL ADDRESSES AND ADD A , OR ; FOR FORMATTING AND ADD IT TO EMAIL LIST
    if is_conditional:
            emails = []
            emails.extend(split_emails(conditional_emails))
    else:
        if 'emails' in lead_notifications.email_addresses:
            emails = []
            emails.extend(split_emails(lead_notifications.email_addresses['emails']))
        else:
            emails = ''
    if emails:
        try:
            if error_occurred:
                if external_provider_name is None or '':
                    external_provider_name = 'the external service provider'

                ctx['external_provider'] = external_provider_name
                email_template = 'mail/failed_form_lead_post'
            else:
                email_template = 'mail/new_form_lead'

            # Render html template for email
            html_email_template = _try_renderer_template(email_template, ext='html', **ctx)
            # New SES emailing
            from buyercall.lib.util_ses_email import send_ses_email
            send_ses_email(recipients=emails,
                           p_id=partner.id,
                           subject=partner.name + ' - ' + subject_text,
                           html=html_email_template
                           )

        except Exception:
            log.error(traceback.format_exc())
    else:
        log.info('No email addresses have been set for this form id: {}'.format(lead.id))

    # ADF SPECIFIC FIELDS FOR ADF EMAIL
    ctx['vendor_name'] = partner_account.name
    ctx['form_lead_id'] = lead.id
    if is_conditional:
        ctx['form_type'] = lead_notifications.display_name
    else:
        ctx['form_type'] = subject_text

    # THE FULL VEHICLE FIELD
    try:
        if full_vehicle:
            ctx['full_vehicle'] = full_vehicle
        else:
            ctx['full_vehicle'] = ''
    except Exception:
        log.info('The vehicle field was not used')

    #THE LOCATION FIELD
    try:
        if location:
            ctx['location'] = location
        else:
            ctx['location'] = ''
    except Exception:
        log.info('The location field was not used')

    #THE LEAD COMMENTS FIELD
    try:
        lead_contact_url = ''

        if full_vehicle and location == '':
            ctx['lead_comments'] = 'The lead is interested in:' + ' ' + full_vehicle
        elif full_vehicle and location:
            ctx['lead_comments'] = 'The lead is interested in:' + ' ' + full_vehicle + ' ' + 'at location:' + ' ' + location
        else:
            ctx['lead_comments'] = ''

        if lead is not None and lead.contact_id is not None:
            if partner.partner_url:
                lead_contact_url = partner.partner_url + '/contacts/edit/' + str(lead.contact_id)
            else:
                lead_contact_url = url_for('contacts.contact_edit', id=lead.contact_id, _external=True)
            lead_contact_url = 'Access lead in BuyerCall: ' + lead_contact_url

            if ctx['lead_comments'] is not None and len(ctx['lead_comments']) > 1:
                ctx['lead_comments'] = ctx['lead_comments'] + '. ' + lead_contact_url
            else:
                ctx['lead_comments'] = lead_contact_url
    except Exception:
        log.info('The lead comments field was not used due to no full or location or both field missing')

    # THE VEHICLE YEAR FIELD
    try:
        if vehicle_year:
            ctx['vehicle_year'] = vehicle_year
        else:
            ctx['vehicle_year'] = ''
    except Exception:
        log.info('The vehicle year field was not used')

    # THE VEHICLE MODEL FIELD
    try:
        if vehicle_model:
            ctx['vehicle_model'] = vehicle_model
        else:
            ctx['vehicle_model'] = ''
    except Exception:
        log.info('The vehicle model field was not used')

    # THE VEHICLE MAKE FIELD
    try:
        if vehicle_make:
            ctx['vehicle_make'] = vehicle_make
        else:
            ctx['vehicle_make'] = ''
    except Exception:
        log.info('The vehicle make field was not used')

    # THE VEHICLE TRIM FIELD
    try:
        if vehicle_trim:
            ctx['vehicle_trim'] = vehicle_trim
        else:
            ctx['vehicle_trim'] = ''
    except Exception:
        log.info('The vehicle trim field was not used')

    # THE VEHICLE VIN FIELD
    try:
        if vehicle_vin:
            ctx['vehicle_vin'] = vehicle_vin
        else:
            ctx['vehicle_vin'] = ''
    except Exception:
        log.info('The vehicle vin field was not used')

    # THE VEHICLE STOCK FIELD
    try:
        if vehicle_stock:
            ctx['vehicle_stock'] = vehicle_stock
        else:
            ctx['vehicle_stock'] = ''
    except Exception:
        log.info('The vehicle stock field was not used')

    # THE STREET FIELD
    try:
        if street:
            ctx['address'] = street
        else:
            ctx['address'] = ''
    except Exception:
        log.info('The street field was not used')

    # THE CITY FIELD
    try:
        if city:
            ctx['city'] = city
        else:
            ctx['city'] = ''
    except Exception:
        log.info('The city field was not used')

    # THE STATE FIELD
    try:
        if state:
            ctx['state'] = state
        else:
            ctx['state'] = ''
    except Exception:
        log.info('The state field was not used')

    # THE ZIP FIELD
    try:
        if zip:
            ctx['postal_code'] = zip
        else:
            ctx['postal_code'] = ''
    except Exception:
        log.info('The postal code field was not used')

    # Check for Ad fields
    ctx['ad_source'] = ''
    ctx['ad_medium'] = ''
    ctx['ad_campaign'] = ''
    ctx['ad_content'] = ''
    ctx['ad_info_exists'] = False
    if ad_source or ad_medium or ad_campaign or ad_content:
        ctx['ad_info_exists'] = True
        ctx['ad_source'] = ad_source
        ctx['ad_medium'] = ad_medium
        ctx['ad_campaign'] = ad_campaign
        ctx['ad_content'] = ad_content

    # SPLIT THE STRING OF ADF EMAIL ADDRESSES AND ADD A , OR ; FOR FORMATTING AND ADD IT TO ADF EMAIL LIST
    if not error_occurred:
        adf_emails = []
        if is_conditional:
                adf_emails.extend(split_emails(conditional_adf_emails))
        else:
            if 'emails' in lead_notifications.adf_email_addresses:
                adf_emails.extend(split_emails(lead_notifications.adf_email_addresses['emails']))
            else:
                adf_emails = ''
        if adf_emails is not None and len(adf_emails) > 0:
            try:
                adf_subject = _(partner.name + ' - ' + 'ADF Form Lead')

                if is_conditional:
                    if conditional_subject_line:
                        adf_subject = partner.name + ' - ' + conditional_subject_line
                    if not adf_subject:
                        adf_subject = partner.name + ' - ' + 'Lead Notification'
                # Check for campaign info on contact record
                ctx['campaign_name'] = ''
                ctx['campaign_exists'] = False
                from buyercall.blueprints.contacts.models import Contact, Campaigns
                contact = Contact.query.filter(
                    Contact.id == lead.contact_id,
                    Contact.partnership_account_id == lead.partnership_account_id) \
                    .first()
                if contact and contact.campaign_id:
                    campaign = Campaigns.query.filter(
                        Campaigns.id == contact.campaign_id) \
                        .first()
                    if campaign:
                        ctx['campaign_exists'] = True
                        ctx['campaign_name'] = campaign.display_name

                # Render adf txt template for email
                adf_email_template = _try_renderer_template('mail/adf_form_lead', **ctx)
                # New SES emailing
                from buyercall.lib.util_ses_email import send_ses_email
                send_ses_email(recipients=adf_emails,
                               p_id=partner.id,
                               subject=adf_subject,
                               text=adf_email_template)
            except Exception:
                log.error(traceback.format_exc())
        else:
            log.info('No ADF email addresses have been set for this form id: {}'.format(lead.id))


# This function get's used for saving the email template in the modal
# on the form edit page used for when sending auto respond emails for form submissions
@form_leads.route('/forms/auto_email_response/<int:form_id>', methods=['POST'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def auto_email_response(form_id):
    # Get the external form that needs updating with the email template data
    form = db.session.query(Form) \
        .filter(Form.id == form_id).first()
    # Retrieve the field data from the modal
    subject = request.form.get('auto_email_subject')
    body = request.form.get('auto_email_body')

    log.info('The subject is: {}'.format(subject))
    # Set the db data to the field values on the modal form
    form.send_auto_email_subject = subject
    form.send_auto_email_msg = body
    # Save the field data to the form table
    db.session.commit()

    flash(_('The auto respond email template was saved successfully.'), 'success')
    return redirect(url_for('form_leads.form_edit', id_=form.id))


# This function get's used for saving the sms message template in the modal
# on the form edit page used for when sending auto sms messages for form submissions
@form_leads.route('/forms/auto_sms_response/<int:form_id>', methods=['POST'])
@login_required
@role_required('agent', 'admin', 'partner', 'sysadmin')
def auto_sms_response(form_id):
    # Get the external form that needs updating with the email template data
    form = db.session.query(Form) \
        .filter(Form.id == form_id).first()
    # Retrieve the field data from the modal
    inbound_id = request.form.get('phoneOptions')
    message_body = request.form.get('auto_sms_body')

    # Set the db data to the field values on the modal form
    form.send_auto_sms_inbound_id = inbound_id
    form.send_auto_sms_msg = message_body
    # Save the field data to the form table
    db.session.commit()

    flash(_('The auto text message reply template was saved successfully.'), 'success')
    return redirect(url_for('form_leads.form_edit', id_=form.id))


@form_leads.route('/api/form_leads/<int:id_>/external_api_provider/<int:service_provider_id_>', methods=['POST'])
@login_required
@role_required('admin', 'partner', 'agent')
def form_leads_external_provider_api_action(id_, service_provider_id_):
    partnership_account_id = current_user.partnership_account_id
    partnership_account_group_viewing = current_user.is_viewing_partnership

    # Check if being viewed by super partner
    if partnership_account_group_viewing:
        partnership_account_id = current_user.get_user_viewing_partnership_account_id

    return determine_external_api_request(id_, partnership_account_id, service_provider_id_)


def determine_external_api_request(lead_id, partnership_account_id, service_provider_id):
    from ..partnership.models import ExternalApiServiceProvidersPartnershipAccountTie, ExternalApiServiceProviders,PartnershipAccount,Partnership
    from ..sysadmin.models import RequestLog
    from buyercall.blueprints.sysadmin.utilities.ip_api import IpApi

    partnership_account = PartnershipAccount.query.get(partnership_account_id) if partnership_account_id else None
    partnership_id = partnership_account.partnership_id if partnership_account else None
    partnership = Partnership.query.get(partnership_id) if partnership_id else None
    partnership_provider_tie = ExternalApiServiceProvidersPartnershipAccountTie\
        .query.filter(ExternalApiServiceProvidersPartnershipAccountTie.partnership_account_id == partnership_account_id,
                ExternalApiServiceProvidersPartnershipAccountTie.external_api_service_provider_id == service_provider_id,
                ExternalApiServiceProvidersPartnershipAccountTie.active == True)\
        .first()

    if partnership_provider_tie:
        formatted_message = ''
        external_provider = ExternalApiServiceProviders.query \
            .filter(ExternalApiServiceProviders.id == partnership_provider_tie.external_api_service_provider_id) \
            .first()

        if external_provider:
            provider_name = external_provider.name
            request_id = str(uuid.uuid4())
            # fetch ip-address
            import socket
            try:
                hostname = socket.gethostname()
                ip_address = socket.gethostbyname(hostname)
            except Exception as e:
                try:
                    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                    s.connect(("127.0.0.1", 1))  
                    ip_address = s.getsockname()[0]
                    s.close()
                except Exception as e:
                    print(f"Error getting local IP: {e}")
                    ip_address = None

            request_data = {
                'request_id': request_id,
                "request_type": provider_name,
                'partnership_id': partnership.id if partnership else None,
                'partnership_account_id': partnership_account.id if partnership_account else None,
                'partnership_name': partnership.name if partnership else None,
                'partnership_account_name': partnership_account.name if partnership_account else None,
                'remote_ip_address': ip_address,
            }
            RequestLog().create_record(request_data)
            if provider_name and len(provider_name) > 0:
                provider_name = provider_name.lower()
                external_lead_id = ''

                lead = Lead.query.filter(
                    Lead.partnership_account_id == partnership_account_id,
                    Lead.id == lead_id
                ).one_or_none()

                if lead is None:
                    FormLog.create(partnership_account_id,
                                   lead_id,
                                   400,
                                   formatted_message,
                                   external_provider.id,
                                   external_provider.name)
                    return 'Lead does not exist.', 400

                if provider_name == 'autopay':
                    log.debug(
                        'External API request for provider ' + provider_name +
                        ' detected, for partnership account ID ' + str(partnership_account_id) +
                        ' for lead ID ' + str(lead_id))

                    message, code = form_leads_autopay_request(lead, partnership_account_id)
                    FormLog.create(partnership_account_id,
                                   lead_id,
                                   code,
                                   formatted_message,
                                   external_provider.id,
                                   external_provider.name)

                    if code != 200:
                        send_notifications(lead, True, external_provider.name)

                    return jsonify(message=message), code
                elif provider_name.lower() == 'ams 2000':
                    log.debug(
                        'External API request for provider ' + provider_name +
                        ' detected, for partnership account ID ' + str(partnership_account_id) +
                        ' for lead ID ' + str(lead_id))

                    code, message, external_lead_id = form_leads_ams_2000_request(lead,
                                                                                  partnership_account_id,
                                                                                  partnership_provider_tie,request_id)
                    if code != 200:
                        send_notifications(lead, True, external_provider.name)
                    else:
                        # Create lead post record
                        from ..form_leads.models import ExternalApiFormLeadPostTie
                        ExternalApiFormLeadPostTie.set_external_api_form_lead_post(lead.form_id,
                                                                                   lead.id,
                                                                                   external_provider.id,
                                                                                   external_lead_id)
                        if external_lead_id is not None and external_lead_id != '':
                            message = 'The submission was successful and returned customer id; ' + external_lead_id + '.'
                        else:
                            message = 'The submission was successful.'

                    # Create form post log
                    FormLog.create(partnership_account_id,
                                   lead_id,
                                   code,
                                   message,
                                   external_provider.id,
                                   external_provider.name)

                    return jsonify(message=message), code
                elif provider_name.lower() == 'ams evolution':
                    log.debug(
                        'External API request for provider ' + provider_name +
                        ' detected, for partnership account ID ' + str(partnership_account_id) +
                        ' for lead ID ' + str(lead_id))

                    # Create lead post record
                    code, message, external_lead_id = form_leads_ams_evolution_request(lead,
                                                                                       partnership_account_id,
                                                                                       partnership_provider_tie, request_id)

                    if code != 200:
                        send_notifications(lead, True, external_provider.name)
                    else:
                        #If successful post, save the External Provider Lead ID
                        from ..form_leads.models import ExternalApiFormLeadPostTie
                        ExternalApiFormLeadPostTie.set_external_api_form_lead_post(lead.form_id,
                                                                                   lead.id,
                                                                                   external_provider.id,
                                                                                   external_lead_id)
                        if external_lead_id is not None and external_lead_id != '':
                            message = 'The submission was successful and returned customer id; ' + external_lead_id + '.'
                        else:
                            message = 'The submission was successful.'

                    # Create form post log
                    FormLog.create(partnership_account_id,
                                   lead_id,
                                   code,
                                   message,
                                   external_provider.id,
                                   external_provider.name)

                    return jsonify(message=message), code
                elif provider_name.lower() == 'neo verify':
                    log.debug(
                        'External API request for provider ' + provider_name +
                        ' detected, for partnership account ID ' + str(partnership_account_id) +
                        ' for lead ID ' + str(lead_id))

                    if not lead.external_api_lead_id:
                        # Create lead post record
                        code, message, external_lead_id = form_leads_neo_verify_request(lead,
                                                                                        partnership_account_id,
                                                                                        partnership_provider_tie,
                                                                                        request_id)
                        # Add the neo id to the form lead
                        lead.external_api_lead_id = external_lead_id
                        db.session.commit()
                    else:
                        # Create lead post record
                        code, message, external_lead_id = form_leads_neo_verify_request(lead,
                                                                                        partnership_account_id,
                                                                                        partnership_provider_tie,
                                                                                        request_id,
                                                                                        neo_id=lead.external_api_lead_id)

                    if code not in (201, 202):
                        send_notifications(lead, True, external_provider.name)
                    else:
                        #If successful post, save the External Provider Lead ID
                        from ..form_leads.models import ExternalApiFormLeadPostTie
                        ExternalApiFormLeadPostTie.set_external_api_form_lead_post(lead.form_id,
                                                                                   lead.id,
                                                                                   external_provider.id,
                                                                                   external_lead_id)
                        if external_lead_id is not None and external_lead_id != '':
                            message = 'The submission was successful and returned customer id; ' + external_lead_id + '.'
                        else:
                            message = 'The submission was successful.'

                    # Create form post log
                    FormLog.create(partnership_account_id,
                                   lead_id,
                                   code,
                                   message,
                                   external_provider.id,
                                   external_provider.name)

                    return jsonify(message=message), code
                else:
                    log.error('External API enabled but no provider name match for ' + provider_name +
                              ', partnership account ID: ' + str(partnership_account_id) + '.')
            else:
                log.error('External API enabled but no provider detected, partnership account ID: ' + str(
                    partnership_account_id))
                return None


def form_leads_ams_2000_request(lead, partnership_account_id, external_provider_tie, request_id):
    encrypt_key = app.config['CRYPTO_SECRET_KEY']
    external_lead_id = ''
    from buyercall.blueprints.sysadmin.models import RequestLog
    RequestLog().update_record(
        request_id, {"request_type": "ams-2000"})

    if (external_provider_tie.url is None or external_provider_tie.url == '') or \
            (external_provider_tie.client_id is None or external_provider_tie.client_id == '') or \
            (external_provider_tie.secret is None or external_provider_tie.secret == '') or \
            (external_provider_tie.username is None or external_provider_tie.username == '') or \
            (external_provider_tie.password is None or external_provider_tie.password == '') or \
            (external_provider_tie.token_url is None or external_provider_tie.token_url == ''):
        log.error('External API enabled but no API details found, partnership account ID: ' + str(partnership_account_id) + '.')
        update_data = {
            "error": 'API details incomplete',
            'response_code': 400,
            "status": "failed",
            }
        RequestLog().update_record(request_id, update_data)
        return 400, 'API details incomplete.', None
    else:
        cipher = AESCipher(encrypt_key)
        api_username = cipher.decrypt(external_provider_tie.username)
        api_password = cipher.decrypt(external_provider_tie.password)
        api_secret = cipher.decrypt(external_provider_tie.secret)

        message, code, token_data = form_leads_ams_2000_token_request(external_provider_tie, api_username, api_password, api_secret, request_id)

        if code == 200:
            if token_data is not None and token_data != '':
                code, message, external_lead_id = form_leads_ams_2000_data_request(token_data, lead, external_provider_tie)

        return code, message, external_lead_id


def form_leads_ams_2000_token_request(external_provider, api_username, api_password, api_secret, request_id):
   
    result_token = ''
    message = 'Error retrieving token'
    update_data = {
        'current_url': external_provider.token_url,
        "path_info": external_provider.token_url,
        "response_text": message,
        'method':"POST",
        "response_code":400
        }
    from buyercall.blueprints.sysadmin.models import RequestLog
    RequestLog().update_record(request_id, update_data)
    handshake = {
        "client_id": external_provider.client_id,
        "client_secret": api_secret,
        "contentType": "application/x-www-form-urlencoded",
        "grant_type": "password",
        "password": api_password,
        "username": api_username
    }

    token_headers = {'content-type': 'application/x-www-form-urlencoded'}
    token_response = requests.post(external_provider.token_url,
                                   data=handshake,
                                   headers=token_headers)

    if token_response.status_code == 200:
        result = json.loads(token_response.content.decode())

        if result is not None and 'access_token' in result:
            result_token = result['access_token']
            message = 'Token successfully retrieved.'
    else:
        error_text = json.loads(token_response.content.decode())

        if 'viewErrors' in error_text:
            message = error_text['viewErrors']
        elif 'message' in error_text:
            message = error_text['message']

    status_code = token_response.status_code
    update_data = {
        "response_code": status_code,
        "response_text": message,
        "status": "success" if status_code < 400 else "failed"
        }
    from buyercall.blueprints.sysadmin.models import RequestLog
    RequestLog().update_record(request_id, update_data)
    return message, status_code, result_token


def generate_ams_2000_request_body(lead):
    error_message = ''
    data_body = {
        'CustomerSSN': '000000000',
        'CustomerFirstName': 'unavailable',
        'CustomerLastName': 'unavailable'
    }
    lead_fields = LeadField\
        .query\
        .filter(LeadField.lead_id == lead.id)\
        .all()

    from .automaster_fields import automaster_fields, automaster_field_details

    if lead_fields:
        field_definition_list = ExternalFormFieldDefinition.query.all()

        try:
            for field in lead_fields:
                field_list = ExternalFormFieldDefinition.get_all_related_fields(field.field_id, field_definition_list)

                if field_list:
                    for check_field in field_list:
                        if check_field in automaster_fields:
                            new_key = automaster_fields[check_field]
                            new_value = decrypt_value(field.field_value)

                            if new_key in automaster_field_details:
                                value_details = automaster_field_details[new_key]

                                if value_details is not None and value_details is not '':
                                    value_details = value_details.lower()

                                    if value_details == 'decimal':
                                        new_value = new_value
                                    elif value_details == 'datetime':
                                        if (new_key =='CoApplicantDateOfBirth' or new_key == 'CustomerDateOfBirth') and new_value is not None and new_value != '':
                                            #log.error(new_key + ' - Encrypted value ' + new_value)
                                            #new_value = decrypt_value(new_value)
                                            log.error(new_key + ' - Decrypted value ' + new_value)
                                    else:
                                        if represents_int(value_details):
                                            string_length = int(value_details)

                                            if new_value is None:
                                                new_value = ''
                                            elif new_value is not None:
                                                if (new_key == 'CustomerSSN' or new_key == 'CoApplicantSSN') and new_value is not None and new_value != '':
                                                    #log.error(new_key + ' - Encrypted value ' + new_value)
                                                    #new_value = decrypt_value(new_value)
                                                    log.error(new_key + ' - Decrypted value ' + new_value)

                                                if len(new_value) > string_length:
                                                    new_value = new_value[0:string_length]

                                data_body[new_key] = new_value
                            else:
                                data_body[new_key] = new_value
                            break

            if ('CustomerSSN' in data_body and data_body['CustomerSSN'] == '') or ('CustomerSSN' not in data_body):
                error_message = append_error(error_message, 'Applicant SSN not provided.')

            if ('Lot' in data_body and data_body['Lot'] == '') or ('Lot' not in data_body):
                error_message = append_error(error_message, 'Lot ID not provided.')

            if ('CustomerDLState' in data_body and data_body['CustomerDLState'] == '') or ('CustomerDLState' not in data_body):
                if 'CustomerAddressState' in data_body:
                    if data_body['CustomerAddressState'] != '':
                        data_body['CustomerDLState'] = data_body['CustomerAddressState']
                    else:
                        error_message = append_error(error_message, 'Customer drivers license state not provided.')
                else:
                    error_message = append_error(error_message, 'Customer drivers license state not provided.')

            if 'CoApplicantSSN' in data_body and data_body['CoApplicantSSN'] == '':
                log.info('No co-applicant SSN found. No co-applicant validation will be performed.')
            elif 'CoApplicantSSN' in data_body or 'CoApplicantFirstName' in data_body \
                    or 'CoApplicantLastName' in data_body:

                if 'CoApplicantSSN' in data_body and data_body['CoApplicantSSN'] == '':
                    error_message = append_error(error_message, 'Co-applicant SSN not provided.')

                if 'CoApplicantFirstName' in data_body and data_body['CoApplicantFirstName'] == '':
                    data_body['CoApplicantFirstName'] = 'unavailable'

                if 'CoApplicantLastName' in data_body and data_body['CoApplicantLastName'] == '':
                    data_body['CoApplicantLastName'] = 'unavailable'

                # If no drivers license state can be found for the co-app, then app drives license state used
                if ('CoApplicantDLState' in data_body
                    and (data_body['CoApplicantDLState'] == '' or data_body['CoApplicantDLState'] is None)
                ) or ('CoApplicantDLState' not in data_body):
                    if 'CoApplicantState' in data_body:
                        if data_body['CoApplicantState'] != '':
                            data_body['CoApplicantDLState'] = data_body['CoApplicantState']
                        elif data_body['CustomerDLState'] != '':
                            data_body['CoApplicantDLState'] = data_body['CustomerDLState']
                        else:
                            error_message = append_error(
                                error_message,
                                'Co-applicant drivers license state not provided.'
                            )
                    else:
                        error_message = append_error(error_message, 'Co-applicant drivers license state not provided.')

                if ('CoApplicantState' in data_body
                    and (data_body['CoApplicantState'] == '' or data_body['CoApplicantState'] is None)
                ) or ('CoApplicantState' not in data_body):
                    if 'CoApplicantDLState' in data_body:
                        if data_body['CoApplicantDLState'] != '':
                            data_body['CoApplicantState'] = data_body['CoApplicantDLState']
                        elif data_body['CustomerDLState'] != '':
                            data_body['CoApplicantState'] = data_body['CustomerDLState']
                        else:
                            error_message = append_error(error_message, 'Co-applicant state not provided.')
                    else:
                        error_message = append_error(error_message, 'Co-applicant state not provided.')
        except Exception:
            error_message = append_error(error_message, traceback.format_exc())
    else:
        log.error('No lead form leads found.')
        error_message = append_error(error_message, 'No lead found')

    return {
        'body': data_body,
        'error': error_message
    }


def form_leads_ams_2000_data_request(token_result, lead, external_provider):
    data_body = None
    data_generated = generate_ams_2000_request_body(lead)
    message = 'Error posting form data.'
    external_lead_id = ''

    if data_generated is not None and len(data_generated['error']) == 0:
        data_body = data_generated['body']

    if data_body is not None:
        data_headers = {'content-type': 'application/x-www-form-urlencoded',
                        'Authorization': 'Bearer ' + token_result}
        response = requests.post(external_provider.url,
                                 data=data_body,
                                 headers=data_headers)

        if response.status_code == 200:
            response.encoding = 'utf-8'
            external_lead_id = response.text.replace('"', '')

            if external_lead_id is not None and external_lead_id is not '':
                log.error('result id: ' + str(external_lead_id))
                lead.external_api_lead_id = str(external_lead_id)
                db.session.commit()
            else:
                log.error('No DealerAppsID found. Response: ' + str(external_lead_id))
        else:
            try:
                error_text = json.loads(response.content.decode())
                log.info('The AMS error is:{}'.format(error_text))

                if 'viewErrors' in error_text:
                    message = error_text['viewErrors']
                elif 'message' in error_text:
                    message = error_text['message']
            except Exception as e:
                log.info('The error is {}'.format(e))

        return response.status_code, message, external_lead_id
    else:
        log.error('Error building automaster request body. Error: ' + data_generated['error'])

        return 400, data_generated['error'], external_lead_id

    return None


def generate_ams_evolution_request_body(lead):
    error_message = ''
    co_applicant_present = False
    down_payment_present = False
    lead_fields = LeadField \
        .query \
        .filter(LeadField.lead_id == lead.id) \
        .all()

    required_applicant_fields = ['Birthday', 'Email', 'Firstname', 'Lastname', 'State', 'Zip', 'Home Phone']

    if lead_fields:
        field_definition_list = ExternalFormFieldDefinition.query.all()
        xml_lead_doc = AmsLeadXml()

        for field in lead_fields:
            field_list = ExternalFormFieldDefinition.get_all_related_fields(field.field_id, field_definition_list)
            field_value = decrypt_value(field.field_value)

            # Buyer fields
            if 'address_1' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.address_1(field_value)
            if 'address_2' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.address_2(field_value)
            if 'birthday' in field_list:
                if len(field_value) > 0:
                    try:
                        required_format = '%Y-%m-%d'
                        field_value = field_value.replace('/', '-')

                        if valid_date(field_value, required_format):
                            final_field_value = field_value
                        else:
                            final_field_value = datetime.datetime.strptime(field_value, '%m-%d-%Y').strftime('%Y-%m-%d')

                        xml_lead_doc.birthday(final_field_value)
                        required_applicant_fields.remove('Birthday')
                    except:
                        error_message = append_error(error_message, 'Invalid Application Birthday ' + str(field_value) + ', must be in YYYY-mm-dd format.')
            if 'mobile_phone' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.mobile_phone(field_value)
            if 'city' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.city(field_value)
            if 'driver_license' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.drivers_license(field_value)
            if 'email' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.email(field_value)
                    required_applicant_fields.remove('Email')
            if 'firstname' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.firstname(field_value)
                    required_applicant_fields.remove('Firstname')
            if 'phonenumber' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.phonenumber(field_value)
                    required_applicant_fields.remove('Home Phone')
            if 'lastname' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.lastname(field_value)
                    required_applicant_fields.remove('Lastname')
            if 'middlename' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.middlename(field_value)
            if 'state' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.state(field_value)
                    required_applicant_fields.remove('State')
            if 'work_phone' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.work_phone(field_value)
            if 'zip' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.zip(field_value)
                    required_applicant_fields.remove('Zip')
            if 'ssn' in field_list:
                if len(field_value) > 0:
                    formatted_ssn = field_value.replace('-', '').replace(' ', '').replace('/', '')
                    xml_lead_doc.ssn(formatted_ssn)

            # Co-buyer fields
            if 'co_applicant_address_1' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_address_1(field_value)
            if 'co_applicant_address_2' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_address_2(field_value)
            if 'co_applicant_birthday' in field_list:
                if len(field_value) > 0:
                    try:
                        field_value = field_value.replace('/', '-')
                        final_field_value = datetime.datetime.strptime(field_value, '%m-%d-%Y').strftime('%Y-%m-%d')
                        xml_lead_doc.co_applicant_birthday(final_field_value)
                    except:
                        error_message = append_error(error_message, 'Invalid Co-Applicant Birthday ' + str(field_value) + ', must be in YYYY-mm-dd format.')
            if 'co_applicant_mobilephone' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_mobile_phone(field_value)
            if 'co_applicant_city' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_city(field_value)
            if 'co_applicant_driver_license' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_driver_license(field_value)
            if 'co_applicant_email' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_email(field_value)
            if 'co_applicant_firstname' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_firstname(field_value)
                    co_applicant_present = True
            if 'co_applicant_phonenumber' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_phonenumber(field_value)
            if 'co_applicant_lastname' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_lastname(field_value)
                    co_applicant_present = True
            if 'co_applicant_middlename' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_middlename(field_value)
            if 'co_applicant_state' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_state(field_value)
            if 'co_applicant_work_phone' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_work_phone(field_value)
            if 'co_applicant_zip' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.co_applicant_zip(field_value)
            if 'co_applicant_ssn' in field_list:
                if len(field_value) > 0:
                    formatted_ssn = field_value.replace('-', '').replace(' ', '').replace('/', '')
                    xml_lead_doc.co_applicant_ssn(formatted_ssn)

            ## Integration Mapping
            #if 'dealer_lot_id' in field_list:
            #    if len(field_value) > 0:
            #        xml_lead_doc.dealer_lot_id(field_value)
            #        required_applicant_fields.remove('Dealer Lot ID')

            # Vehicle
            if 'vehicle_make' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.vehicle_make(field_value)
            if 'vehicle_model' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.vehicle_model(field_value)
            if 'vehicle_year' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.vehicle_year(field_value)
            if 'vin' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.vin(field_value)
            if 'stock_number' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.stock_number(field_value)

            # Cash Down
            if 'down_payment_amount' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.down_payment_amount(field_value)
                    down_payment_present = True

            # Sales person
            if 'representative' in field_list:
                if len(field_value) > 0:
                    xml_lead_doc.representative(field_value)

        if not down_payment_present:
            xml_lead_doc.down_payment_amount('0.000')

        if len(required_applicant_fields) > 0:
            cur_count = 0
            count = len(required_applicant_fields)
            message = 'Required fields are missing: '

            for field in required_applicant_fields:
                message += field
                cur_count += 1

                if cur_count == count:
                    message += '.'
                else:
                    message += ', '

            error_message = append_error(error_message, message)

        #if co_applicant_present == True and len(required_co_applicant_fields):
        #    cur_count = 0
        #    count = len(required_co_applicant_fields)
        #    message = 'Required co-applicant fields are missing: '
        #
        #    for field in required_co_applicant_fields:
        #        message += field
        #        cur_count += 1
        #
        #        if cur_count == count:
        #            message += '.'
        #        else:
        #            message += ', '
        #
        #    error_message = append_error(error_message, message)

        return {
            'xml': xml_lead_doc,
            'error': error_message
        }


def form_leads_ams_evolution_request(lead, partnership_account_id, external_provider_tie, request_id):
    from buyercall.blueprints.sysadmin.models import RequestLog
    # request_id = request.environ.get('HTTP_X_REQUEST_ID')
    # updating the request log with the request_type
    data = {"request_type": "ams_evolution", "method": "POST"}
    RequestLog().update_record(request_id, data)
    try:
        client = ams_client(partnership_account_id, "AMS Evolution",request_id)
    except:
        log.error('Error initiating client: ' + traceback.format_exc())
        return 400, 'Error initiating client.', ''

    data_generated = generate_ams_evolution_request_body(lead)

    if len(data_generated['error']) > 0 or data_generated['xml'] is None:
        return 400, data_generated['error'], ''
    try:
        xml_lead_doc = data_generated['xml']
        xml_lead_doc.username(decrypt_value(external_provider_tie.username))
        xml_lead_doc.password(decrypt_value(external_provider_tie.password))
        xml_lead_doc.client_id(decrypt_value(external_provider_tie.client_id))

        # Unable to find direct/official support for CDATA in Elemtree
        cdata_pre = '<Payload><![CDATA['
        cdata_post = ']]></Payload>'
        post_request = f"{cdata_pre}{xml_lead_doc}{cdata_post}"

        code, message, external_lead_id = client.post(post_request)

        return code, message, external_lead_id
    except:
        log.error('Error posting form lead. ' + traceback.format_exc())
        return 400, 'Error posting form lead. ', ''


def generate_neo_verify_request_body(lead, neo_id=None):
    error_message = ''
    lead_fields = LeadField \
        .query \
        .filter(LeadField.lead_id == lead.id) \
        .all()
    new_lead = None
    processed = False
    built = False

    if lead_fields:
        field_definition_list = ExternalFormFieldDefinition.query.all()
        if neo_id:
            new_lead = NeoLead(lead_fields, field_definition_list, neo_id)
        else:
            new_lead = NeoLead(lead_fields, field_definition_list)

        processed, error_message = new_lead.process_lead()

        if processed is True:
            built, error_message = new_lead.build_lead_object()

    if new_lead and processed is True and built is True:
        return {
            'obj': new_lead.get_lead_object(),
            'error': error_message
        }
    else:
        return {
            'obj': new_lead,
            'error': 'Error processing lead.'
        }


def form_leads_neo_verify_request(lead, partnership_account_id, external_provider_tie,request_id=None, neo_id=None):

    from buyercall.blueprints.sysadmin.models import RequestLog
    RequestLog().update_record(
        request_id, {"request_type": "neo"})
    try:
        client = neo_client(partnership_account_id, "NEO Verify",request_id)
    except:
        log.error('Error initiating client: ' + traceback.format_exc())
        return 400, 'Error initiating client.', ''

    if neo_id:
        data_generated = generate_neo_verify_request_body(lead, neo_id)

        if len(data_generated['error']) > 0 or data_generated['obj'] is None:
            return 400, data_generated['error'], ''
        try:
            code, message, external_lead_id = client.put(data_generated['obj'])
            return code, message, external_lead_id
        except:
            log.error('Error posting form lead. ' + traceback.format_exc())
            return 400, 'Error posting form lead. ', ''
    else:
        data_generated = generate_neo_verify_request_body(lead)

        if len(data_generated['error']) > 0 or data_generated['obj'] is None:
            return 400, data_generated['error'], ''
        try:
            code, message, external_lead_id = client.post(data_generated['obj'])
            return code, message, external_lead_id
        except:
            log.error('Error posting form lead. ' + traceback.format_exc())
            return 400, 'Error posting form lead. ', ''


@form_leads.route('/form_leads/<int:pa_id>/generate_leads/pdf')
@login_required
@role_required('sysadmin')
def generate_lead_pdfs(pa_id):
    list_max_size = 250

    # Import the partnership account model
    from ..partnership.models import PartnershipAccount
    account = PartnershipAccount\
        .query\
        .filter(PartnershipAccount.id == pa_id)\
        .first()

    if not account:
        return jsonify(
            account_name="",
            form_lead_count=0
        )

    form_leads = [fl.id for fl in db.session.query(Lead.id).filter(Lead.partnership_account_id == pa_id).all()]

    if form_leads:
        #Break the form leads list up into chunks of "list_max_size"
        form_lead_lists = [form_leads[i * list_max_size:(i + 1) * list_max_size] for i in range((len(form_leads) + list_max_size - 1) // list_max_size)]

        for form_list_part in form_lead_lists:
            from .form_tasks import generate_partnership_account_form_lead_pdfs
            generate_partnership_account_form_lead_pdfs.delay(account.id, account.name, form_list_part)

        return jsonify(
            account_name=account.name,
            form_lead_count=len(form_leads)
        )
    else:
        return jsonify(
            account_name=account.name,
            form_lead_count=0
        )