add bills and extended properties

This commit is contained in:
Moses Rolston 2025-03-14 03:35:23 -07:00
parent 7ae10745ef
commit c1e456aedc
4 changed files with 180 additions and 3 deletions

68
add_bill_type.py Normal file
View File

@ -0,0 +1,68 @@
import os
import requests
from dotenv import load_dotenv
from neo4j import GraphDatabase
# Load environment variables from .env file
load_dotenv()
# Neo4j connection details
NEO4J_URI = os.getenv('NEO4J_URI')
NEO4J_USER = os.getenv('NEO4J_USER')
NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
# Congress API key
CONGRESS_API_KEY = os.getenv('CONGRESS_API_KEY')
# Initialize Neo4j driver
driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
def fetch_bills(offset):
url = f"https://api.congress.gov/v3/bill/117/sres?offset={offset}&api_key={CONGRESS_API_KEY}"
print(f"Fetching data from {url}")
response = requests.get(url)
if response.status_code == 200:
return response.json()
else:
raise Exception(f"Failed to fetch data: {response.status_code}")
def flatten_json(y):
out = {}
def flatten(x, name=''):
if type(x) is dict:
for a in x:
flatten(x[a], name + a + '_')
elif type(x) is list:
i = 0
for a in x:
flatten(a, name + str(i) + '_')
i += 1
else:
out[name[:-1]] = x
flatten(y)
return out
def create_bill_node(tx, bill_data):
flat_bill_data = flatten_json(bill_data)
print(f"Creating Bill node with properties: {flat_bill_data}")
query = "CREATE (b:Bill $properties)"
tx.run(query, properties=flat_bill_data)
def main():
offset = 0
while True:
try:
bills_data = fetch_bills(offset)
if not bills_data or len(bills_data['bills']) == 0:
break
with driver.session() as session:
for bill in bills_data['bills']:
session.write_transaction(create_bill_node, bill)
offset += 250
except Exception as e:
print(f"Error: {e}")
break
if __name__ == "__main__":
main()
driver.close()

View File

@ -1,6 +1,8 @@
# endpoints/person_network.py
from flask import Blueprint, jsonify
import neo4j
from neo4j import GraphDatabase
from app import get_driver, neo4j_logger
bp = Blueprint('network', __name__)
@ -30,6 +32,7 @@ def person_network():
if person_id not in nodes:
properties = {key: value for key, value in person_node.items()}
properties['id'] = person_id
properties['labels'] = list(person_node.labels) # Include labels separately
nodes[person_id] = {
**properties,
"group": 1 # Group for Person
@ -40,6 +43,7 @@ def person_network():
if connected_id not in nodes:
properties = {key: value for key, value in connected_node.items()}
properties['id'] = connected_id
properties['labels'] = list(connected_node.labels) # Include labels separately
nodes[connected_id] = {
**properties,
"group": 2 # Group for other nodes (e.g., Organization, Position)
@ -56,5 +60,5 @@ def person_network():
return jsonify({"nodes": list(nodes.values()), "links": links})
except Exception as e:
neo4j_logger.error(f"Error interacting with Neo4j: {e}")
return jsonify({"error": "An error occurred while interacting with the database"}), 500
print(f"Error interacting with Neo4j: {e}")
return jsonify({"error": f"An error occurred while interacting with the database: {str(e)}"}), 500

View File

@ -66,6 +66,11 @@
// Fetch network data from Flask endpoint
d3.json('/person_network').then(function(data) {
if (data.error) {
console.error(data.error);
return;
}
const nodesData = data.nodes;
const linksData = data.links;
@ -110,7 +115,8 @@
} else if (['Legislation', 'Bill', 'Law'].some(label => d.labels.includes(label))) {
return d.title || 'Title';
}
return 'Node'; // Fallback for other types
// Fallback for other types
return Object.keys(d).filter(key => key !== 'labels').map(key => d[key]).join(', ') || 'Node';
})
.attr("dx", 10) // Offset from node
.attr("dy", 4); // Offset from node

99
update_bills.py Normal file
View File

@ -0,0 +1,99 @@
import sys
from neo4j import GraphDatabase
from dotenv import load_dotenv
import os
import requests
import json
from flatten_json import flatten
# Global variable to store the list of bill numbers
bill_numbers = []
def search_bills(bill_type):
# Load environment variables from .env file
load_dotenv()
# Get connection information from environment variables
uri = os.getenv('NEO4J_URI')
user = os.getenv('NEO4J_USER')
password = os.getenv('NEO4J_PASSWORD')
# Connect to Neo4j database
driver = GraphDatabase.driver(uri, auth=(user, password))
try:
with driver.session() as session:
# Query to find nodes with label 'Bill' and property 'type' matching the provided value
query = "MATCH (b:Bill) WHERE b.type = $bill_type RETURN b.number"
# Execute the query
result = session.run(query, bill_type=bill_type)
# Collect the list of bill numbers
global bill_numbers
bill_numbers = [record["b.number"] for record in result]
finally:
# Close the driver connection
driver.close()
def get_bill_details(congress, bill_type, bill_number):
url = f"https://api.congress.gov/v3/bill/{congress}/{bill_type.lower()}/{bill_number}?format=json&api_key={os.getenv('CONGRESS_API_KEY')}"
response = requests.get(url)
if response.status_code == 200:
return response.json()
else:
print(f"Failed to fetch bill details for {bill_number}: {response.status_code}")
print(f"Response Text: {response.text}")
return None
def flatten_json(y):
out = {}
def flatten(x, name=''):
if type(x) is dict:
for a in x:
flatten(x[a], name + a + '.')
elif type(x) is list:
i = 0
for a in x:
flatten(a, name + str(i) + '.')
i += 1
else:
out[name[:-1]] = x
flatten(y)
return {k.replace('bill.', ''): v for k, v in out.items()}
def update_bill_node(driver, bill_number, properties):
with driver.session() as session:
# Remove existing properties
query_remove_properties = f"MATCH (b:Bill {{number: $bill_number}}) SET b += {{}}"
session.run(query_remove_properties, bill_number=bill_number)
# Add new properties
query_add_properties = f"MATCH (b:Bill {{number: $bill_number}}) SET b += $properties RETURN b"
session.run(query_add_properties, bill_number=bill_number, properties=properties)
if __name__ == "__main__":
if len(sys.argv) != 3:
print("Usage: python search_bills.py <congress> <bill_type>")
sys.exit(1)
congress = sys.argv[1]
bill_type = sys.argv[2]
search_bills(bill_type)
# Connect to Neo4j database
driver = GraphDatabase.driver(os.getenv('NEO4J_URI'), auth=(os.getenv('NEO4J_USER'), os.getenv('NEO4J_PASSWORD')))
for bill_number in bill_numbers:
print(f"Fetching details for bill number {bill_number}...")
bill_details = get_bill_details(congress, bill_type, bill_number)
if bill_details:
flattened_properties = flatten_json(bill_details)
update_bill_node(driver, bill_number, flattened_properties)
print(f"Updated bill node with properties from JSON response for bill number {bill_number}")
# Close the driver connection
driver.close()