from flask import Blueprint, jsonify import requests import os import json from dotenv import load_dotenv from app import get_driver, neo4j_logger load_dotenv() bp = Blueprint('ingest_bills', __name__) @bp.route('/ingest_bills', methods=['POST']) def ingest_bills(): api_key = os.getenv('CONGRESS_API_KEY') if not api_key: neo4j_logger.error("Congress API key is missing in environment variables") return jsonify({"error": "Congress API key is missing"}), 500 congress_number = 117 start_offset = 0 page_size = 250 bills_url_template = f"https://api.congress.gov/v3/bill/{congress_number}?offset={{start_offset}}&limit={page_size}&api_key={api_key}" while True: url = bills_url_template.format(start_offset=start_offset) response = requests.get(url, timeout=10) # Add a timeout to prevent hanging requests if response.status_code != 200: neo4j_logger.error(f"Failed to fetch bills data: {response.status_code}, {response.text}") return jsonify({"error": "Failed to fetch bills data"}), 500 bills_data = response.json() # Check for pagination details total_results = int(bills_data.get('totalResults', 0)) if not bills_data.get('bills'): break # Process and create Bill nodes in Neo4j driver = get_driver() with driver.session() as session: for bill in bills_data['bills']: try: bill_properties = {} for key, value in bill.items(): if isinstance(value, dict): # Flatten nested dictionaries for sub_key, sub_value in value.items(): bill_properties[f"{key}_{sub_key}"] = sub_value or None else: bill_properties[key] = value or None # Ensure there is a unique identifier (bill_id) if not bill_properties.get('bill_id'): # Construct a bill_id based on available fields, e.g., bill_type and number bill_id = f"{bill_properties['billType']}_{bill_properties['number']}" bill_properties['bill_id'] = bill_id query_to_execute = session.write_transaction(create_bill_node, bill_properties) print(f"Executing query: {query_to_execute}") except Exception as e: neo4j_logger.error(f"Error creating Bill node: {e}") # Update offset for pagination start_offset += page_size if start_offset >= total_results: break return jsonify({"message": "Bill ingestion completed successfully"}), 200 def create_bill_node(tx, properties): query = """ MERGE (b:Bill {bill_id: $bill_id}) SET b += $properties RETURN b """ result = tx.run(query, bill_id=properties.get('bill_id'), properties=properties) return result.summary().query