add bills and extended properties
This commit is contained in:
parent
7ae10745ef
commit
c1e456aedc
68
add_bill_type.py
Normal file
68
add_bill_type.py
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
import os
|
||||||
|
import requests
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from neo4j import GraphDatabase
|
||||||
|
|
||||||
|
# Load environment variables from .env file
|
||||||
|
load_dotenv()
|
||||||
|
|
||||||
|
# Neo4j connection details
|
||||||
|
NEO4J_URI = os.getenv('NEO4J_URI')
|
||||||
|
NEO4J_USER = os.getenv('NEO4J_USER')
|
||||||
|
NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
|
||||||
|
|
||||||
|
# Congress API key
|
||||||
|
CONGRESS_API_KEY = os.getenv('CONGRESS_API_KEY')
|
||||||
|
|
||||||
|
# Initialize Neo4j driver
|
||||||
|
driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
|
||||||
|
|
||||||
|
def fetch_bills(offset):
|
||||||
|
url = f"https://api.congress.gov/v3/bill/117/sres?offset={offset}&api_key={CONGRESS_API_KEY}"
|
||||||
|
print(f"Fetching data from {url}")
|
||||||
|
response = requests.get(url)
|
||||||
|
if response.status_code == 200:
|
||||||
|
return response.json()
|
||||||
|
else:
|
||||||
|
raise Exception(f"Failed to fetch data: {response.status_code}")
|
||||||
|
|
||||||
|
def flatten_json(y):
|
||||||
|
out = {}
|
||||||
|
def flatten(x, name=''):
|
||||||
|
if type(x) is dict:
|
||||||
|
for a in x:
|
||||||
|
flatten(x[a], name + a + '_')
|
||||||
|
elif type(x) is list:
|
||||||
|
i = 0
|
||||||
|
for a in x:
|
||||||
|
flatten(a, name + str(i) + '_')
|
||||||
|
i += 1
|
||||||
|
else:
|
||||||
|
out[name[:-1]] = x
|
||||||
|
flatten(y)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def create_bill_node(tx, bill_data):
|
||||||
|
flat_bill_data = flatten_json(bill_data)
|
||||||
|
print(f"Creating Bill node with properties: {flat_bill_data}")
|
||||||
|
query = "CREATE (b:Bill $properties)"
|
||||||
|
tx.run(query, properties=flat_bill_data)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
offset = 0
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
bills_data = fetch_bills(offset)
|
||||||
|
if not bills_data or len(bills_data['bills']) == 0:
|
||||||
|
break
|
||||||
|
with driver.session() as session:
|
||||||
|
for bill in bills_data['bills']:
|
||||||
|
session.write_transaction(create_bill_node, bill)
|
||||||
|
offset += 250
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error: {e}")
|
||||||
|
break
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
driver.close()
|
@ -1,6 +1,8 @@
|
|||||||
# endpoints/person_network.py
|
# endpoints/person_network.py
|
||||||
|
|
||||||
from flask import Blueprint, jsonify
|
from flask import Blueprint, jsonify
|
||||||
|
import neo4j
|
||||||
|
from neo4j import GraphDatabase
|
||||||
from app import get_driver, neo4j_logger
|
from app import get_driver, neo4j_logger
|
||||||
|
|
||||||
bp = Blueprint('network', __name__)
|
bp = Blueprint('network', __name__)
|
||||||
@ -30,6 +32,7 @@ def person_network():
|
|||||||
if person_id not in nodes:
|
if person_id not in nodes:
|
||||||
properties = {key: value for key, value in person_node.items()}
|
properties = {key: value for key, value in person_node.items()}
|
||||||
properties['id'] = person_id
|
properties['id'] = person_id
|
||||||
|
properties['labels'] = list(person_node.labels) # Include labels separately
|
||||||
nodes[person_id] = {
|
nodes[person_id] = {
|
||||||
**properties,
|
**properties,
|
||||||
"group": 1 # Group for Person
|
"group": 1 # Group for Person
|
||||||
@ -40,6 +43,7 @@ def person_network():
|
|||||||
if connected_id not in nodes:
|
if connected_id not in nodes:
|
||||||
properties = {key: value for key, value in connected_node.items()}
|
properties = {key: value for key, value in connected_node.items()}
|
||||||
properties['id'] = connected_id
|
properties['id'] = connected_id
|
||||||
|
properties['labels'] = list(connected_node.labels) # Include labels separately
|
||||||
nodes[connected_id] = {
|
nodes[connected_id] = {
|
||||||
**properties,
|
**properties,
|
||||||
"group": 2 # Group for other nodes (e.g., Organization, Position)
|
"group": 2 # Group for other nodes (e.g., Organization, Position)
|
||||||
@ -56,5 +60,5 @@ def person_network():
|
|||||||
return jsonify({"nodes": list(nodes.values()), "links": links})
|
return jsonify({"nodes": list(nodes.values()), "links": links})
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
neo4j_logger.error(f"Error interacting with Neo4j: {e}")
|
print(f"Error interacting with Neo4j: {e}")
|
||||||
return jsonify({"error": "An error occurred while interacting with the database"}), 500
|
return jsonify({"error": f"An error occurred while interacting with the database: {str(e)}"}), 500
|
||||||
|
@ -66,6 +66,11 @@
|
|||||||
|
|
||||||
// Fetch network data from Flask endpoint
|
// Fetch network data from Flask endpoint
|
||||||
d3.json('/person_network').then(function(data) {
|
d3.json('/person_network').then(function(data) {
|
||||||
|
if (data.error) {
|
||||||
|
console.error(data.error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const nodesData = data.nodes;
|
const nodesData = data.nodes;
|
||||||
const linksData = data.links;
|
const linksData = data.links;
|
||||||
|
|
||||||
@ -110,7 +115,8 @@
|
|||||||
} else if (['Legislation', 'Bill', 'Law'].some(label => d.labels.includes(label))) {
|
} else if (['Legislation', 'Bill', 'Law'].some(label => d.labels.includes(label))) {
|
||||||
return d.title || 'Title';
|
return d.title || 'Title';
|
||||||
}
|
}
|
||||||
return 'Node'; // Fallback for other types
|
// Fallback for other types
|
||||||
|
return Object.keys(d).filter(key => key !== 'labels').map(key => d[key]).join(', ') || 'Node';
|
||||||
})
|
})
|
||||||
.attr("dx", 10) // Offset from node
|
.attr("dx", 10) // Offset from node
|
||||||
.attr("dy", 4); // Offset from node
|
.attr("dy", 4); // Offset from node
|
||||||
|
99
update_bills.py
Normal file
99
update_bills.py
Normal file
@ -0,0 +1,99 @@
|
|||||||
|
import sys
|
||||||
|
from neo4j import GraphDatabase
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
from flatten_json import flatten
|
||||||
|
|
||||||
|
# Global variable to store the list of bill numbers
|
||||||
|
bill_numbers = []
|
||||||
|
|
||||||
|
def search_bills(bill_type):
|
||||||
|
# Load environment variables from .env file
|
||||||
|
load_dotenv()
|
||||||
|
|
||||||
|
# Get connection information from environment variables
|
||||||
|
uri = os.getenv('NEO4J_URI')
|
||||||
|
user = os.getenv('NEO4J_USER')
|
||||||
|
password = os.getenv('NEO4J_PASSWORD')
|
||||||
|
|
||||||
|
# Connect to Neo4j database
|
||||||
|
driver = GraphDatabase.driver(uri, auth=(user, password))
|
||||||
|
|
||||||
|
try:
|
||||||
|
with driver.session() as session:
|
||||||
|
# Query to find nodes with label 'Bill' and property 'type' matching the provided value
|
||||||
|
query = "MATCH (b:Bill) WHERE b.type = $bill_type RETURN b.number"
|
||||||
|
|
||||||
|
# Execute the query
|
||||||
|
result = session.run(query, bill_type=bill_type)
|
||||||
|
|
||||||
|
# Collect the list of bill numbers
|
||||||
|
global bill_numbers
|
||||||
|
bill_numbers = [record["b.number"] for record in result]
|
||||||
|
finally:
|
||||||
|
# Close the driver connection
|
||||||
|
driver.close()
|
||||||
|
|
||||||
|
def get_bill_details(congress, bill_type, bill_number):
|
||||||
|
url = f"https://api.congress.gov/v3/bill/{congress}/{bill_type.lower()}/{bill_number}?format=json&api_key={os.getenv('CONGRESS_API_KEY')}"
|
||||||
|
response = requests.get(url)
|
||||||
|
if response.status_code == 200:
|
||||||
|
return response.json()
|
||||||
|
else:
|
||||||
|
print(f"Failed to fetch bill details for {bill_number}: {response.status_code}")
|
||||||
|
print(f"Response Text: {response.text}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def flatten_json(y):
|
||||||
|
out = {}
|
||||||
|
|
||||||
|
def flatten(x, name=''):
|
||||||
|
if type(x) is dict:
|
||||||
|
for a in x:
|
||||||
|
flatten(x[a], name + a + '.')
|
||||||
|
elif type(x) is list:
|
||||||
|
i = 0
|
||||||
|
for a in x:
|
||||||
|
flatten(a, name + str(i) + '.')
|
||||||
|
i += 1
|
||||||
|
else:
|
||||||
|
out[name[:-1]] = x
|
||||||
|
|
||||||
|
flatten(y)
|
||||||
|
return {k.replace('bill.', ''): v for k, v in out.items()}
|
||||||
|
|
||||||
|
def update_bill_node(driver, bill_number, properties):
|
||||||
|
with driver.session() as session:
|
||||||
|
# Remove existing properties
|
||||||
|
query_remove_properties = f"MATCH (b:Bill {{number: $bill_number}}) SET b += {{}}"
|
||||||
|
session.run(query_remove_properties, bill_number=bill_number)
|
||||||
|
|
||||||
|
# Add new properties
|
||||||
|
query_add_properties = f"MATCH (b:Bill {{number: $bill_number}}) SET b += $properties RETURN b"
|
||||||
|
session.run(query_add_properties, bill_number=bill_number, properties=properties)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) != 3:
|
||||||
|
print("Usage: python search_bills.py <congress> <bill_type>")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
congress = sys.argv[1]
|
||||||
|
bill_type = sys.argv[2]
|
||||||
|
|
||||||
|
search_bills(bill_type)
|
||||||
|
|
||||||
|
# Connect to Neo4j database
|
||||||
|
driver = GraphDatabase.driver(os.getenv('NEO4J_URI'), auth=(os.getenv('NEO4J_USER'), os.getenv('NEO4J_PASSWORD')))
|
||||||
|
|
||||||
|
for bill_number in bill_numbers:
|
||||||
|
print(f"Fetching details for bill number {bill_number}...")
|
||||||
|
bill_details = get_bill_details(congress, bill_type, bill_number)
|
||||||
|
if bill_details:
|
||||||
|
flattened_properties = flatten_json(bill_details)
|
||||||
|
update_bill_node(driver, bill_number, flattened_properties)
|
||||||
|
print(f"Updated bill node with properties from JSON response for bill number {bill_number}")
|
||||||
|
|
||||||
|
# Close the driver connection
|
||||||
|
driver.close()
|
Loading…
Reference in New Issue
Block a user