105 lines
4.0 KiB
Python
105 lines
4.0 KiB
Python
import requests
|
|
from neo4j import GraphDatabase
|
|
from dotenv import load_dotenv
|
|
import os
|
|
|
|
# Load environment variables from .env file
|
|
load_dotenv()
|
|
|
|
# Retrieve API key and Neo4j connection details from environment variables
|
|
CONGRESS_API_KEY = os.getenv('CONGRESS_API_KEY')
|
|
if not CONGRESS_API_KEY:
|
|
raise ValueError("API key not found in .env file")
|
|
|
|
NEO4J_URI = os.getenv('NEO4J_URI')
|
|
NEO4J_USER = os.getenv('NEO4J_USER')
|
|
NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
|
|
if not NEO4J_URI or not NEO4J_USER or not NEO4J_PASSWORD:
|
|
raise ValueError("Neo4j connection details not found in .env file")
|
|
|
|
BASE_URL_MEMBERSHIP = 'https://api.congress.gov/v3/member'
|
|
BASE_URL_COSPONSORED_LEGISLATION = 'https://api.congress.gov/v3/member/{}/cosponsored-legislation'
|
|
|
|
# Function to get all person nodes with bioguideId from Neo4j
|
|
def get_person_nodes(driver):
|
|
with driver.session() as session:
|
|
result = session.run("MATCH (p:Person) RETURN p.bioguideId")
|
|
return [record['p.bioguideId'] for record in result]
|
|
|
|
# Function to fetch cosponsored legislation for a given bioguideId
|
|
def fetch_cosponsored_legislation(bioguideId, api_key):
|
|
url = BASE_URL_COSPONSORED_LEGISLATION.format(bioguideId)
|
|
params = {
|
|
'api_key': api_key,
|
|
'limit': 250 # Maximum limit allowed is 250
|
|
}
|
|
response = requests.get(url, params=params)
|
|
if response.status_code == 200:
|
|
return response.json().get('legislations', [])
|
|
else:
|
|
print(f"Failed to retrieve cosponsored legislation for {bioguideId}: {response.status_code}")
|
|
return []
|
|
|
|
# Function to create or update a person node in Neo4j using MERGE
|
|
def merge_person_node(driver, bioguideId):
|
|
with driver.session() as session:
|
|
query = """
|
|
MERGE (p:Person {bioguideId: $bioguideId})
|
|
RETURN p.bioguideId
|
|
"""
|
|
result = session.run(query, bioguideId=bioguideId)
|
|
return result.single()[0]
|
|
|
|
# Function to create a legislation node in Neo4j and return its label
|
|
def create_legislation_node(driver, legislation):
|
|
with driver.session() as session:
|
|
number = legislation.get('number')
|
|
properties = {str(key): value for key, value in legislation.items() if value is not None and value != ""}
|
|
query = f"""
|
|
MERGE (l:Legislation {{id: $id}})
|
|
SET l += $properties
|
|
RETURN labels(l)
|
|
"""
|
|
result = session.run(query, id=legislation.get('id'), properties=properties)
|
|
return result.single()[0]
|
|
|
|
# Function to create a cosponsored relationship between a person and legislation node in Neo4j
|
|
def create_cosponsored_relationship(driver, bioguideId, number):
|
|
with driver.session() as session:
|
|
query = """
|
|
MATCH (p:Person {bioguideId: $bioguideId})
|
|
MATCH (l)
|
|
WHERE id(l) IN [id(n) WHERE n:Legislation AND n.id = $number]
|
|
MERGE (p)-[:COSPONSORED]->(l)
|
|
"""
|
|
session.run(query, bioguideId=bioguideId, number=legislation.get('id'))
|
|
|
|
# Function to process each person node and fetch cosponsored legislation
|
|
def process_person_nodes(driver):
|
|
person_nodes = get_person_nodes(driver)
|
|
for bioguideId in person_nodes:
|
|
print(f"Processing member with bioguideId: {bioguideId}")
|
|
|
|
# Ensure the person node exists in Neo4j
|
|
merge_person_node(driver, bioguideId)
|
|
|
|
# Fetch cosponsored legislation
|
|
legislations = fetch_cosponsored_legislation(bioguideId, CONGRESS_API_KEY)
|
|
for legislation in legislations:
|
|
print(f"Processing legislation: {legislation.get('number')}")
|
|
|
|
# Create or update the legislation node
|
|
create_legislation_node(driver, legislation)
|
|
|
|
# Create a cosponsored relationship from the person to the legislation
|
|
create_cosponsored_relationship(driver, bioguideId, legislation)
|
|
|
|
# Main execution block
|
|
if __name__ == "__main__":
|
|
driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
|
|
|
|
try:
|
|
process_person_nodes(driver)
|
|
finally:
|
|
driver.close()
|