78 lines
3.0 KiB
Python
Executable File
78 lines
3.0 KiB
Python
Executable File
from flask import Blueprint, jsonify
|
|
import requests
|
|
import os
|
|
import json
|
|
from dotenv import load_dotenv
|
|
from app import get_driver, neo4j_logger
|
|
|
|
load_dotenv()
|
|
bp = Blueprint('ingest_bills', __name__)
|
|
|
|
@bp.route('/ingest_bills', methods=['POST'])
|
|
def ingest_bills():
|
|
api_key = os.getenv('CONGRESS_API_KEY')
|
|
if not api_key:
|
|
neo4j_logger.error("Congress API key is missing in environment variables")
|
|
return jsonify({"error": "Congress API key is missing"}), 500
|
|
|
|
congress_number = 117
|
|
start_offset = 0
|
|
page_size = 250
|
|
bills_url_template = f"https://api.congress.gov/v3/bill/{congress_number}?offset={{start_offset}}&limit={page_size}&api_key={api_key}"
|
|
|
|
while True:
|
|
url = bills_url_template.format(start_offset=start_offset)
|
|
response = requests.get(url, timeout=10) # Add a timeout to prevent hanging requests
|
|
if response.status_code != 200:
|
|
neo4j_logger.error(f"Failed to fetch bills data: {response.status_code}, {response.text}")
|
|
return jsonify({"error": "Failed to fetch bills data"}), 500
|
|
|
|
bills_data = response.json()
|
|
|
|
# Check for pagination details
|
|
total_results = int(bills_data.get('totalResults', 0))
|
|
if not bills_data.get('bills'):
|
|
break
|
|
|
|
# Process and create Bill nodes in Neo4j
|
|
driver = get_driver()
|
|
with driver.session() as session:
|
|
for bill in bills_data['bills']:
|
|
try:
|
|
bill_properties = {}
|
|
for key, value in bill.items():
|
|
if isinstance(value, dict):
|
|
# Flatten nested dictionaries
|
|
for sub_key, sub_value in value.items():
|
|
bill_properties[f"{key}_{sub_key}"] = sub_value or None
|
|
else:
|
|
bill_properties[key] = value or None
|
|
|
|
# Ensure there is a unique identifier (bill_id)
|
|
if not bill_properties.get('bill_id'):
|
|
# Construct a bill_id based on available fields, e.g., bill_type and number
|
|
bill_id = f"{bill_properties['billType']}_{bill_properties['number']}"
|
|
bill_properties['bill_id'] = bill_id
|
|
|
|
query_to_execute = session.write_transaction(create_bill_node, bill_properties)
|
|
print(f"Executing query: {query_to_execute}")
|
|
except Exception as e:
|
|
neo4j_logger.error(f"Error creating Bill node: {e}")
|
|
|
|
# Update offset for pagination
|
|
start_offset += page_size
|
|
if start_offset >= total_results:
|
|
break
|
|
|
|
return jsonify({"message": "Bill ingestion completed successfully"}), 200
|
|
|
|
|
|
def create_bill_node(tx, properties):
|
|
query = """
|
|
MERGE (b:Bill {bill_id: $bill_id})
|
|
SET b += $properties
|
|
RETURN b
|
|
"""
|
|
result = tx.run(query, bill_id=properties.get('bill_id'), properties=properties)
|
|
return result.summary().query
|