policymap/api/endpoints/get_sponsored.py

from flask import Blueprint, jsonify
from app import get_driver, neo4j_logger
import requests
import json
import os

bp = Blueprint('get_sponsored', __name__)

CACHE_FILE = 'cache.json'

def load_cache():
    if os.path.exists(CACHE_FILE):
        with open(CACHE_FILE, 'r') as f:
            return json.load(f)
    else:
        neo4j_logger.info(f"Cache file {CACHE_FILE} does not exist. Initializing empty cache.")
        return {}

def save_cache(cache_data):
    with open(CACHE_FILE, 'w') as f:
        json.dump(cache_data, f)
    neo4j_logger.info(f"Saved data to cache file: {CACHE_FILE}")

@bp.route('/get_sponsored')
def get_sponsored():
    # Load bioguideIds and legislation from the same cache
    cache = load_cache()
    if 'bioguideIds' not in cache or len(cache['bioguideIds']) == 0:
        return jsonify({"message": "No bioguideIds found in cache"}), 404

    # Print the number of items found in the cache initially
    initial_bioguideIds_count = len(cache['bioguideIds'])
    print(f"Initial bioguideIds count: {initial_bioguideIds_count}")
    processed_legislation_count = 0

    while 'bioguideIds' in cache and len(cache['bioguideIds']) > 0:
        # Step 1: Retrieve a sponsor from the cache
        current_bioguideId = cache['bioguideIds'].pop(0)
        if current_bioguideId is None:
            continue
        print(f"Processing bioguideId: {current_bioguideId}")

        congress_api_url = f"https://api.congress.gov/v3/member/{current_bioguideId}/sponsored-legislation"

        # Include API key in headers (if required)
        api_key = os.getenv('CONGRESS_API_KEY')
        if not api_key:
            neo4j_logger.error("Congress API key not found in environment variables")
            continue

        headers = {
            'X-API-KEY': api_key
        }

        # Step 2: Fetch sponsored legislation for the member
        response = requests.get(congress_api_url, headers=headers)
        print(f"Response Status Code: {response.status_code}")
        print(f"Response Text: {response.text}")

        if response.status_code != 200:
            neo4j_logger.error(f"Failed to fetch sponsored legislation for bioguideId {current_bioguideId}: Status Code {response.status_code}, Response: {response.text}")
            continue

        response_data = response.json()

        # Ensure the response contains 'sponsoredLegislation' key
        if 'sponsoredLegislation' not in response_data:
            neo4j_logger.error(f"Missing 'sponsoredLegislation' key in response for bioguideId {current_bioguideId}: {response_data}")
            continue

        sponsored_legislation = response_data['sponsoredLegislation']

        # Step 3: Store each piece of legislation in the cache along with the sponsor bioguideId
        for legislation in sponsored_legislation:
            if 'number' not in legislation:
                neo4j_logger.error(f"Missing 'number' field in legislation data for bioguideId {current_bioguideId}: {legislation}")
                continue

            key = f"legislation_{legislation['number']}"
            print(f"About to write to cache: {key} - {legislation}")

            if key not in cache:
                # Ensure all nested dictionaries are handled properly
                latest_action = legislation.get('latestAction')

                # Check if latest_action is None before calling .get()
                if latest_action is None:
                    latest_action = {}

                policy_area = legislation.get('policyArea', {})

                legislation_info = {
                    'bioguideId': current_bioguideId,
                    **legislation,
                    'latestAction': {
                        'actionDate': latest_action.get('actionDate'),
                        'text': latest_action.get('text')
                    },
                    'policyArea': {
                        'name': policy_area.get('name')
                    }
                }

                cache[key] = legislation_info
                processed_legislation_count += 1

                # Save the cache immediately after writing each entry
                save_cache(cache)

                # Print the updated legislation cache for debugging
                print(f"Updated legislation cache: {json.dumps(cache, indent=2)}")

        # Step 4: Delete the sponsor from the bioguideIds list (already done by popping)
        neo4j_logger.info(f"Processed sponsored legislation for bioguideId {current_bioguideId}")

        # Print the number of items left in the bioguide cache
        remaining_bioguideIds_count = len(cache['bioguideIds'])
        print(f"Remaining bioguideIds count after processing {current_bioguideId}: {remaining_bioguideIds_count}")

    # Save the cache again to ensure all changes are persisted
    save_cache(cache)

    # Print the total number of legislation items stored and overall items added to the cache
    print(f"Total processed legislation count: {processed_legislation_count}")
    print(f"Overall items in cache: {len(cache)}")

    return jsonify({
        "message": "Sponsored legislation processed successfully",
        "processed_legislation_count": processed_legislation_count
    }), 200