diff --git a/add_bill_type.py b/add_bill_type.py new file mode 100644 index 0000000..673678d --- /dev/null +++ b/add_bill_type.py @@ -0,0 +1,68 @@ +import os +import requests +from dotenv import load_dotenv +from neo4j import GraphDatabase + +# Load environment variables from .env file +load_dotenv() + +# Neo4j connection details +NEO4J_URI = os.getenv('NEO4J_URI') +NEO4J_USER = os.getenv('NEO4J_USER') +NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD') + +# Congress API key +CONGRESS_API_KEY = os.getenv('CONGRESS_API_KEY') + +# Initialize Neo4j driver +driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD)) + +def fetch_bills(offset): + url = f"https://api.congress.gov/v3/bill/117/sres?offset={offset}&api_key={CONGRESS_API_KEY}" + print(f"Fetching data from {url}") + response = requests.get(url) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Failed to fetch data: {response.status_code}") + +def flatten_json(y): + out = {} + def flatten(x, name=''): + if type(x) is dict: + for a in x: + flatten(x[a], name + a + '_') + elif type(x) is list: + i = 0 + for a in x: + flatten(a, name + str(i) + '_') + i += 1 + else: + out[name[:-1]] = x + flatten(y) + return out + +def create_bill_node(tx, bill_data): + flat_bill_data = flatten_json(bill_data) + print(f"Creating Bill node with properties: {flat_bill_data}") + query = "CREATE (b:Bill $properties)" + tx.run(query, properties=flat_bill_data) + +def main(): + offset = 0 + while True: + try: + bills_data = fetch_bills(offset) + if not bills_data or len(bills_data['bills']) == 0: + break + with driver.session() as session: + for bill in bills_data['bills']: + session.write_transaction(create_bill_node, bill) + offset += 250 + except Exception as e: + print(f"Error: {e}") + break + +if __name__ == "__main__": + main() + driver.close() diff --git a/api/endpoints/person_network.py b/api/endpoints/person_network.py index 84cdda6..3f008d8 100644 --- a/api/endpoints/person_network.py +++ b/api/endpoints/person_network.py @@ -1,6 +1,8 @@ # endpoints/person_network.py from flask import Blueprint, jsonify +import neo4j +from neo4j import GraphDatabase from app import get_driver, neo4j_logger bp = Blueprint('network', __name__) @@ -30,6 +32,7 @@ def person_network(): if person_id not in nodes: properties = {key: value for key, value in person_node.items()} properties['id'] = person_id + properties['labels'] = list(person_node.labels) # Include labels separately nodes[person_id] = { **properties, "group": 1 # Group for Person @@ -40,6 +43,7 @@ def person_network(): if connected_id not in nodes: properties = {key: value for key, value in connected_node.items()} properties['id'] = connected_id + properties['labels'] = list(connected_node.labels) # Include labels separately nodes[connected_id] = { **properties, "group": 2 # Group for other nodes (e.g., Organization, Position) @@ -56,5 +60,5 @@ def person_network(): return jsonify({"nodes": list(nodes.values()), "links": links}) except Exception as e: - neo4j_logger.error(f"Error interacting with Neo4j: {e}") - return jsonify({"error": "An error occurred while interacting with the database"}), 500 + print(f"Error interacting with Neo4j: {e}") + return jsonify({"error": f"An error occurred while interacting with the database: {str(e)}"}), 500 diff --git a/api/templates/render_network.html b/api/templates/render_network.html index aaf9e1b..cc56d23 100644 --- a/api/templates/render_network.html +++ b/api/templates/render_network.html @@ -66,6 +66,11 @@ // Fetch network data from Flask endpoint d3.json('/person_network').then(function(data) { + if (data.error) { + console.error(data.error); + return; + } + const nodesData = data.nodes; const linksData = data.links; @@ -110,7 +115,8 @@ } else if (['Legislation', 'Bill', 'Law'].some(label => d.labels.includes(label))) { return d.title || 'Title'; } - return 'Node'; // Fallback for other types + // Fallback for other types + return Object.keys(d).filter(key => key !== 'labels').map(key => d[key]).join(', ') || 'Node'; }) .attr("dx", 10) // Offset from node .attr("dy", 4); // Offset from node diff --git a/update_bills.py b/update_bills.py new file mode 100644 index 0000000..6eda2fc --- /dev/null +++ b/update_bills.py @@ -0,0 +1,99 @@ +import sys +from neo4j import GraphDatabase +from dotenv import load_dotenv +import os +import requests +import json +from flatten_json import flatten + +# Global variable to store the list of bill numbers +bill_numbers = [] + +def search_bills(bill_type): + # Load environment variables from .env file + load_dotenv() + + # Get connection information from environment variables + uri = os.getenv('NEO4J_URI') + user = os.getenv('NEO4J_USER') + password = os.getenv('NEO4J_PASSWORD') + + # Connect to Neo4j database + driver = GraphDatabase.driver(uri, auth=(user, password)) + + try: + with driver.session() as session: + # Query to find nodes with label 'Bill' and property 'type' matching the provided value + query = "MATCH (b:Bill) WHERE b.type = $bill_type RETURN b.number" + + # Execute the query + result = session.run(query, bill_type=bill_type) + + # Collect the list of bill numbers + global bill_numbers + bill_numbers = [record["b.number"] for record in result] + finally: + # Close the driver connection + driver.close() + +def get_bill_details(congress, bill_type, bill_number): + url = f"https://api.congress.gov/v3/bill/{congress}/{bill_type.lower()}/{bill_number}?format=json&api_key={os.getenv('CONGRESS_API_KEY')}" + response = requests.get(url) + if response.status_code == 200: + return response.json() + else: + print(f"Failed to fetch bill details for {bill_number}: {response.status_code}") + print(f"Response Text: {response.text}") + return None + +def flatten_json(y): + out = {} + + def flatten(x, name=''): + if type(x) is dict: + for a in x: + flatten(x[a], name + a + '.') + elif type(x) is list: + i = 0 + for a in x: + flatten(a, name + str(i) + '.') + i += 1 + else: + out[name[:-1]] = x + + flatten(y) + return {k.replace('bill.', ''): v for k, v in out.items()} + +def update_bill_node(driver, bill_number, properties): + with driver.session() as session: + # Remove existing properties + query_remove_properties = f"MATCH (b:Bill {{number: $bill_number}}) SET b += {{}}" + session.run(query_remove_properties, bill_number=bill_number) + + # Add new properties + query_add_properties = f"MATCH (b:Bill {{number: $bill_number}}) SET b += $properties RETURN b" + session.run(query_add_properties, bill_number=bill_number, properties=properties) + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python search_bills.py ") + sys.exit(1) + + congress = sys.argv[1] + bill_type = sys.argv[2] + + search_bills(bill_type) + + # Connect to Neo4j database + driver = GraphDatabase.driver(os.getenv('NEO4J_URI'), auth=(os.getenv('NEO4J_USER'), os.getenv('NEO4J_PASSWORD'))) + + for bill_number in bill_numbers: + print(f"Fetching details for bill number {bill_number}...") + bill_details = get_bill_details(congress, bill_type, bill_number) + if bill_details: + flattened_properties = flatten_json(bill_details) + update_bill_node(driver, bill_number, flattened_properties) + print(f"Updated bill node with properties from JSON response for bill number {bill_number}") + + # Close the driver connection + driver.close()