add bills and extended properties
This commit is contained in:
parent
7ae10745ef
commit
c1e456aedc
68
add_bill_type.py
Normal file
68
add_bill_type.py
Normal file
@ -0,0 +1,68 @@
|
||||
import os
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
from neo4j import GraphDatabase
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
# Neo4j connection details
|
||||
NEO4J_URI = os.getenv('NEO4J_URI')
|
||||
NEO4J_USER = os.getenv('NEO4J_USER')
|
||||
NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
|
||||
|
||||
# Congress API key
|
||||
CONGRESS_API_KEY = os.getenv('CONGRESS_API_KEY')
|
||||
|
||||
# Initialize Neo4j driver
|
||||
driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
|
||||
|
||||
def fetch_bills(offset):
|
||||
url = f"https://api.congress.gov/v3/bill/117/sres?offset={offset}&api_key={CONGRESS_API_KEY}"
|
||||
print(f"Fetching data from {url}")
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
else:
|
||||
raise Exception(f"Failed to fetch data: {response.status_code}")
|
||||
|
||||
def flatten_json(y):
|
||||
out = {}
|
||||
def flatten(x, name=''):
|
||||
if type(x) is dict:
|
||||
for a in x:
|
||||
flatten(x[a], name + a + '_')
|
||||
elif type(x) is list:
|
||||
i = 0
|
||||
for a in x:
|
||||
flatten(a, name + str(i) + '_')
|
||||
i += 1
|
||||
else:
|
||||
out[name[:-1]] = x
|
||||
flatten(y)
|
||||
return out
|
||||
|
||||
def create_bill_node(tx, bill_data):
|
||||
flat_bill_data = flatten_json(bill_data)
|
||||
print(f"Creating Bill node with properties: {flat_bill_data}")
|
||||
query = "CREATE (b:Bill $properties)"
|
||||
tx.run(query, properties=flat_bill_data)
|
||||
|
||||
def main():
|
||||
offset = 0
|
||||
while True:
|
||||
try:
|
||||
bills_data = fetch_bills(offset)
|
||||
if not bills_data or len(bills_data['bills']) == 0:
|
||||
break
|
||||
with driver.session() as session:
|
||||
for bill in bills_data['bills']:
|
||||
session.write_transaction(create_bill_node, bill)
|
||||
offset += 250
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
break
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
driver.close()
|
||||
@ -1,6 +1,8 @@
|
||||
# endpoints/person_network.py
|
||||
|
||||
from flask import Blueprint, jsonify
|
||||
import neo4j
|
||||
from neo4j import GraphDatabase
|
||||
from app import get_driver, neo4j_logger
|
||||
|
||||
bp = Blueprint('network', __name__)
|
||||
@ -30,6 +32,7 @@ def person_network():
|
||||
if person_id not in nodes:
|
||||
properties = {key: value for key, value in person_node.items()}
|
||||
properties['id'] = person_id
|
||||
properties['labels'] = list(person_node.labels) # Include labels separately
|
||||
nodes[person_id] = {
|
||||
**properties,
|
||||
"group": 1 # Group for Person
|
||||
@ -40,6 +43,7 @@ def person_network():
|
||||
if connected_id not in nodes:
|
||||
properties = {key: value for key, value in connected_node.items()}
|
||||
properties['id'] = connected_id
|
||||
properties['labels'] = list(connected_node.labels) # Include labels separately
|
||||
nodes[connected_id] = {
|
||||
**properties,
|
||||
"group": 2 # Group for other nodes (e.g., Organization, Position)
|
||||
@ -56,5 +60,5 @@ def person_network():
|
||||
return jsonify({"nodes": list(nodes.values()), "links": links})
|
||||
|
||||
except Exception as e:
|
||||
neo4j_logger.error(f"Error interacting with Neo4j: {e}")
|
||||
return jsonify({"error": "An error occurred while interacting with the database"}), 500
|
||||
print(f"Error interacting with Neo4j: {e}")
|
||||
return jsonify({"error": f"An error occurred while interacting with the database: {str(e)}"}), 500
|
||||
|
||||
@ -66,6 +66,11 @@
|
||||
|
||||
// Fetch network data from Flask endpoint
|
||||
d3.json('/person_network').then(function(data) {
|
||||
if (data.error) {
|
||||
console.error(data.error);
|
||||
return;
|
||||
}
|
||||
|
||||
const nodesData = data.nodes;
|
||||
const linksData = data.links;
|
||||
|
||||
@ -110,7 +115,8 @@
|
||||
} else if (['Legislation', 'Bill', 'Law'].some(label => d.labels.includes(label))) {
|
||||
return d.title || 'Title';
|
||||
}
|
||||
return 'Node'; // Fallback for other types
|
||||
// Fallback for other types
|
||||
return Object.keys(d).filter(key => key !== 'labels').map(key => d[key]).join(', ') || 'Node';
|
||||
})
|
||||
.attr("dx", 10) // Offset from node
|
||||
.attr("dy", 4); // Offset from node
|
||||
|
||||
99
update_bills.py
Normal file
99
update_bills.py
Normal file
@ -0,0 +1,99 @@
|
||||
import sys
|
||||
from neo4j import GraphDatabase
|
||||
from dotenv import load_dotenv
|
||||
import os
|
||||
import requests
|
||||
import json
|
||||
from flatten_json import flatten
|
||||
|
||||
# Global variable to store the list of bill numbers
|
||||
bill_numbers = []
|
||||
|
||||
def search_bills(bill_type):
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
# Get connection information from environment variables
|
||||
uri = os.getenv('NEO4J_URI')
|
||||
user = os.getenv('NEO4J_USER')
|
||||
password = os.getenv('NEO4J_PASSWORD')
|
||||
|
||||
# Connect to Neo4j database
|
||||
driver = GraphDatabase.driver(uri, auth=(user, password))
|
||||
|
||||
try:
|
||||
with driver.session() as session:
|
||||
# Query to find nodes with label 'Bill' and property 'type' matching the provided value
|
||||
query = "MATCH (b:Bill) WHERE b.type = $bill_type RETURN b.number"
|
||||
|
||||
# Execute the query
|
||||
result = session.run(query, bill_type=bill_type)
|
||||
|
||||
# Collect the list of bill numbers
|
||||
global bill_numbers
|
||||
bill_numbers = [record["b.number"] for record in result]
|
||||
finally:
|
||||
# Close the driver connection
|
||||
driver.close()
|
||||
|
||||
def get_bill_details(congress, bill_type, bill_number):
|
||||
url = f"https://api.congress.gov/v3/bill/{congress}/{bill_type.lower()}/{bill_number}?format=json&api_key={os.getenv('CONGRESS_API_KEY')}"
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
else:
|
||||
print(f"Failed to fetch bill details for {bill_number}: {response.status_code}")
|
||||
print(f"Response Text: {response.text}")
|
||||
return None
|
||||
|
||||
def flatten_json(y):
|
||||
out = {}
|
||||
|
||||
def flatten(x, name=''):
|
||||
if type(x) is dict:
|
||||
for a in x:
|
||||
flatten(x[a], name + a + '.')
|
||||
elif type(x) is list:
|
||||
i = 0
|
||||
for a in x:
|
||||
flatten(a, name + str(i) + '.')
|
||||
i += 1
|
||||
else:
|
||||
out[name[:-1]] = x
|
||||
|
||||
flatten(y)
|
||||
return {k.replace('bill.', ''): v for k, v in out.items()}
|
||||
|
||||
def update_bill_node(driver, bill_number, properties):
|
||||
with driver.session() as session:
|
||||
# Remove existing properties
|
||||
query_remove_properties = f"MATCH (b:Bill {{number: $bill_number}}) SET b += {{}}"
|
||||
session.run(query_remove_properties, bill_number=bill_number)
|
||||
|
||||
# Add new properties
|
||||
query_add_properties = f"MATCH (b:Bill {{number: $bill_number}}) SET b += $properties RETURN b"
|
||||
session.run(query_add_properties, bill_number=bill_number, properties=properties)
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("Usage: python search_bills.py <congress> <bill_type>")
|
||||
sys.exit(1)
|
||||
|
||||
congress = sys.argv[1]
|
||||
bill_type = sys.argv[2]
|
||||
|
||||
search_bills(bill_type)
|
||||
|
||||
# Connect to Neo4j database
|
||||
driver = GraphDatabase.driver(os.getenv('NEO4J_URI'), auth=(os.getenv('NEO4J_USER'), os.getenv('NEO4J_PASSWORD')))
|
||||
|
||||
for bill_number in bill_numbers:
|
||||
print(f"Fetching details for bill number {bill_number}...")
|
||||
bill_details = get_bill_details(congress, bill_type, bill_number)
|
||||
if bill_details:
|
||||
flattened_properties = flatten_json(bill_details)
|
||||
update_bill_node(driver, bill_number, flattened_properties)
|
||||
print(f"Updated bill node with properties from JSON response for bill number {bill_number}")
|
||||
|
||||
# Close the driver connection
|
||||
driver.close()
|
||||
Loading…
Reference in New Issue
Block a user