checkpoint before converting to api

This commit is contained in:
Moses Rolston 2025-03-06 14:29:01 -08:00
parent 721a565e22
commit b9d22bede1
11 changed files with 717 additions and 7 deletions

2
.gitignore vendored
View File

@ -248,3 +248,5 @@ inc/
/pm_to_blib /pm_to_blib
/*.zip /*.zip
# Data Files
/*.csv

64
add_bill_node.py Normal file
View File

@ -0,0 +1,64 @@
import csv
from neo4j import GraphDatabase
from dotenv import load_dotenv
import os
# Load environment variables from .env file
load_dotenv()
# Retrieve Neo4j connection details from environment variables
NEO4J_URI = os.getenv('NEO4J_URI')
NEO4J_USER = os.getenv('NEO4J_USER')
NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
if not NEO4J_URI or not NEO4J_USER or not NEO4J_PASSWORD:
raise ValueError("Neo4j connection details not found in .env file")
TEST_FLAG = False
# Function to create or update a bill node in Neo4j using MERGE
def merge_bill_node(driver, bill):
with driver.session() as session:
query = """
MERGE (b:Bill {id: $id})
ON CREATE SET b += $properties
ON MATCH SET b += $properties
RETURN b
"""
# Filter out empty or None values and ensure keys are strings
properties = {str(key): value for key, value in bill.items() if value is not None and value != ""}
if TEST_FLAG:
print(f"MERGE (b:Bill {{id: '{bill['number']}'}}) ON CREATE SET b += ${properties} ON MATCH SET b += ${properties}")
else:
result = session.run(query, id=bill['number'], properties=properties)
return result.single()
# Read the CSV file and create/update nodes in Neo4j
def read_csv_and_merge_nodes(file_path):
driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
try:
with open(file_path, mode='r', newline='', encoding='utf-8') as csvfile:
reader = csv.DictReader(csvfile, delimiter='^')
for row in reader:
bill = {key: value.strip() if isinstance(value, str) else value for key, value in row.items()}
# Debugging print statement to check keys and values
print(f"Processing row with keys: {bill.keys()} and values: {bill.values()}")
result = merge_bill_node(driver, bill)
if not TEST_FLAG:
print(f"Created/Updated node: {result}")
finally:
if not TEST_FLAG:
driver.close()
# Path to the CSV file
csv_file_path = 'bills.csv'
# Run the function
read_csv_and_merge_nodes(csv_file_path)

View File

@ -10,10 +10,10 @@ load_dotenv()
def create_neo4j_session(uri, user, password): def create_neo4j_session(uri, user, password):
return GraphDatabase.driver(uri, auth=(user, password)) return GraphDatabase.driver(uri, auth=(user, password))
def create_node(tx, node_type, properties): def create_or_merge_node(tx, node_type, properties):
# Create a Cypher query to create a node with given type and properties # Create a Cypher query to merge a node with given type and properties
properties_string = ', '.join([f"{key}: '{value}'" for key, value in properties.items()]) properties_string = ', '.join([f"{key}: '{value}'" for key, value in properties.items()])
query = f"CREATE (n:{node_type} {{{properties_string}}})" query = f"MERGE (n:{node_type} {{{properties_string}}})"
tx.run(query) tx.run(query)
def main(csv_file_path): def main(csv_file_path):
@ -42,15 +42,13 @@ def main(csv_file_path):
'date_enacted': pd.to_datetime(row['date_enacted']).strftime('%Y-%m-%d') if pd.notnull(row['date_enacted']) else None, 'date_enacted': pd.to_datetime(row['date_enacted']).strftime('%Y-%m-%d') if pd.notnull(row['date_enacted']) else None,
'date_repealed': pd.to_datetime(row['date_repealed']).strftime('%Y-%m-%d') if pd.notnull(row['date_repealed']) else None 'date_repealed': pd.to_datetime(row['date_repealed']).strftime('%Y-%m-%d') if pd.notnull(row['date_repealed']) else None
} }
# Create or merge the node in Neo4j
# Create the node in Neo4j session.write_transaction(create_or_merge_node, node_type, properties)
session.write_transaction(create_node, node_type, properties)
driver.close() driver.close()
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Read a CSV file and create nodes in Neo4j.") parser = argparse.ArgumentParser(description="Read a CSV file and create nodes in Neo4j.")
parser.add_argument("csv_file_path", type=str, help="Path to the CSV file") parser.add_argument("csv_file_path", type=str, help="Path to the CSV file")
args = parser.parse_args() args = parser.parse_args()
main(args.csv_file_path) main(args.csv_file_path)

80
api.py Normal file
View File

@ -0,0 +1,80 @@
from flask import Flask, request, jsonify
import requests
from neo4j import GraphDatabase
from dotenv import load_dotenv
import os
import logging
from cachetools import TTLCache
# Set up basic configuration for logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# Load environment variables from .env file
load_dotenv()
# Retrieve Neo4j connection details and API key
NEO4J_URI = os.getenv('NEO4J_URI')
NEO4J_USER = os.getenv('NEO4J_USER')
NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
CONGRESS_API_KEY = os.getenv('CONGRESS_API_KEY')
if not NEO4J_URI or not NEO4J_USER or not NEO4J_PASSWORD:
raise ValueError("Neo4j connection details not found in .env file")
if not CONGRESS_API_KEY:
raise ValueError("Congress API key not found in .env file")
# Initialize Neo4j driver
driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
# Initialize caches
neo4j_cache = TTLCache(maxsize=100, ttl=3600) # Cache for 1 hour
congress_api_cache = TTLCache(maxsize=100, ttl=3600) # Cache for 1 hour
app = Flask(__name__)
@app.route('/sponsored_legislation', methods=['GET'])
def get_sponsored_legislation():
bioguide_id = request.args.get('bioguideId')
if not bioguide_id:
return jsonify({"error": "bioguideId is required"}), 400
logger.info(f"Fetching sponsored legislation for member with bioguideId {bioguide_id}")
# Check cache before making API request
cached_legislation = congress_api_cache.get(bioguide_id)
if cached_legislation:
logger.info(f"Using cached sponsored legislation for bioguideId {bioguide_id}")
return jsonify({"message": "Sponsored legislation retrieved from cache", "legislations": cached_legislation})
@app.route('/persons', methods=['GET'])
def list_persons():
logger.info("Listing all person nodes from Neo4j")
# Check cache before querying Neo4j
cached_persons = neo4j_cache.get('all_persons')
if cached_persons:
logger.info("Using cached list of persons")
return jsonify({"message": "Persons retrieved from cache", "persons": cached_persons})
with driver.session() as session:
query = "MATCH (p:Person) RETURN p"
logger.info(f"Executing Neo4j query to list all person nodes: {query}")
result = session.run(query)
persons = [record['p'] for record in result]
person_list = []
for person in persons:
node_properties = {key: value for key, value in person.items()}
person_list.append(node_properties)
# Cache the response
neo4j_cache['all_persons'] = person_list
logger.info("Cached list of all persons")
return jsonify({"message": "Persons listed successfully", "persons": person_list})
if __name__ == '__main__':
app.run(debug=True, host='0.0.0.0', port=5000)

104
cosponsored_add.py Normal file
View File

@ -0,0 +1,104 @@
import requests
from neo4j import GraphDatabase
from dotenv import load_dotenv
import os
# Load environment variables from .env file
load_dotenv()
# Retrieve API key and Neo4j connection details from environment variables
CONGRESS_API_KEY = os.getenv('CONGRESS_API_KEY')
if not CONGRESS_API_KEY:
raise ValueError("API key not found in .env file")
NEO4J_URI = os.getenv('NEO4J_URI')
NEO4J_USER = os.getenv('NEO4J_USER')
NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
if not NEO4J_URI or not NEO4J_USER or not NEO4J_PASSWORD:
raise ValueError("Neo4j connection details not found in .env file")
BASE_URL_MEMBERSHIP = 'https://api.congress.gov/v3/member'
BASE_URL_COSPONSORED_LEGISLATION = 'https://api.congress.gov/v3/member/{}/cosponsored-legislation'
# Function to get all person nodes with bioguideId from Neo4j
def get_person_nodes(driver):
with driver.session() as session:
result = session.run("MATCH (p:Person) RETURN p.bioguideId")
return [record['p.bioguideId'] for record in result]
# Function to fetch cosponsored legislation for a given bioguideId
def fetch_cosponsored_legislation(bioguideId, api_key):
url = BASE_URL_COSPONSORED_LEGISLATION.format(bioguideId)
params = {
'api_key': api_key,
'limit': 250 # Maximum limit allowed is 250
}
response = requests.get(url, params=params)
if response.status_code == 200:
return response.json().get('legislations', [])
else:
print(f"Failed to retrieve cosponsored legislation for {bioguideId}: {response.status_code}")
return []
# Function to create or update a person node in Neo4j using MERGE
def merge_person_node(driver, bioguideId):
with driver.session() as session:
query = """
MERGE (p:Person {bioguideId: $bioguideId})
RETURN p.bioguideId
"""
result = session.run(query, bioguideId=bioguideId)
return result.single()[0]
# Function to create a legislation node in Neo4j and return its label
def create_legislation_node(driver, legislation):
with driver.session() as session:
number = legislation.get('number')
properties = {str(key): value for key, value in legislation.items() if value is not None and value != ""}
query = f"""
MERGE (l:Legislation {{id: $id}})
SET l += $properties
RETURN labels(l)
"""
result = session.run(query, id=legislation.get('id'), properties=properties)
return result.single()[0]
# Function to create a cosponsored relationship between a person and legislation node in Neo4j
def create_cosponsored_relationship(driver, bioguideId, number):
with driver.session() as session:
query = """
MATCH (p:Person {bioguideId: $bioguideId})
MATCH (l)
WHERE id(l) IN [id(n) WHERE n:Legislation AND n.id = $number]
MERGE (p)-[:COSPONSORED]->(l)
"""
session.run(query, bioguideId=bioguideId, number=legislation.get('id'))
# Function to process each person node and fetch cosponsored legislation
def process_person_nodes(driver):
person_nodes = get_person_nodes(driver)
for bioguideId in person_nodes:
print(f"Processing member with bioguideId: {bioguideId}")
# Ensure the person node exists in Neo4j
merge_person_node(driver, bioguideId)
# Fetch cosponsored legislation
legislations = fetch_cosponsored_legislation(bioguideId, CONGRESS_API_KEY)
for legislation in legislations:
print(f"Processing legislation: {legislation.get('number')}")
# Create or update the legislation node
create_legislation_node(driver, legislation)
# Create a cosponsored relationship from the person to the legislation
create_cosponsored_relationship(driver, bioguideId, legislation)
# Main execution block
if __name__ == "__main__":
driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
try:
process_person_nodes(driver)
finally:
driver.close()

View File

@ -8,3 +8,69 @@ order^EO 14216^Expanding Access to In Vitro Fertilization^^https://www.federalre
order^EO 14215^Ensuring Accountability for All Agencies^^https://www.federalregister.gov/d/2025-03063^February 18, 2025^ order^EO 14215^Ensuring Accountability for All Agencies^^https://www.federalregister.gov/d/2025-03063^February 18, 2025^
order^EO 14214^Keeping Education Accessible and Ending COVID-19 Vaccine Mandates in Schools^^https://www.federalregister.gov/d/2025-02931^February 14, 2025^ order^EO 14214^Keeping Education Accessible and Ending COVID-19 Vaccine Mandates in Schools^^https://www.federalregister.gov/d/2025-02931^February 14, 2025^
order^EO 14213^Establishing the National Energy Dominance Council^^https://www.federalregister.gov/d/2025-02928^February 14, 2025^ order^EO 14213^Establishing the National Energy Dominance Council^^https://www.federalregister.gov/d/2025-02928^February 14, 2025^
order^EO 14212^Establishing the Presidents Make America Healthy Again Commission^^https://www.federalregister.gov/d/2025-02871^February 13, 2025^
order^EO 14211^One Voice for Americas Foreign Relations^^https://www.federalregister.gov/d/2025-02841^February 12, 2025^
order^EO 14210^"Implementing the Presidents ""Department of Government Efficiency"" Workforce Optimization Initiative"^^https://www.federalregister.gov/d/2025-02762^February 11, 2025^
order^EO 14209^Pausing Foreign Corrupt Practices Act Enforcement To Further American Economic and National Security^^https://www.federalregister.gov/d/2025-02736^February 10, 2025^
order^EO 14208^Ending Procurement and Forced Use of Paper Straws^^https://www.federalregister.gov/d/2025-02735^February 10, 2025^
order^EO 14207^Eliminating the Federal Executive Institute ^^https://www.federalregister.gov/d/2025-02734^February 10, 2025^
order^EO 14206^Protecting Second Amendment Rights^^https://www.federalregister.gov/d/2025-02636^February 7, 2025^
order^EO 14205^Establishment of the White House Faith Office^^https://www.federalregister.gov/d/2025-02635^February 7, 2025^
order^EO 14204^Addressing Egregious Actions of the Republic of South Africa^^https://www.federalregister.gov/d/2025-02630^February 7, 2025^
order^EO 14203^Imposing Sanctions on the International Criminal Court^^https://www.federalregister.gov/d/2025-02612^February 6, 2025^
order^EO 14202^Eradicating Anti-Christian Bias^^https://www.federalregister.gov/d/2025-02611^February 6, 2025^
order^EO 14201^Keeping Men Out of Womens Sports ^^https://www.federalregister.gov/d/2025-02513^February 5, 2025^
order^EO 14200^Amendment to Duties Addressing the Synthetic Opioid Supply Chain in the Peoples Republic of China^^https://www.federalregister.gov/d/2025-02512^February 5, 2025^
order^EO 14199^Withdrawing the United States From and Ending Funding to Certain United Nations Organizations and Reviewing United States Support to All International Organizations^^https://www.federalregister.gov/d/2025-02504^February 4, 2025^
order^EO 14198^Progress on the Situation at Our Southern Border^^https://www.federalregister.gov/d/2025-02479^February 3, 2025^
order^EO 14197^Progress on the Situation at Our Northern Border^^https://www.federalregister.gov/d/2025-02478^February 3, 2025^
order^EO 14196^A Plan for Establishing a United States Sovereign Wealth Fund^^https://www.federalregister.gov/d/2025-02477^February 3, 2025^
order^EO 14195^Imposing Duties To Address the Synthetic Opioid Supply Chain in the Peoples Republic of China^^https://www.federalregister.gov/d/2025-02408^February 1, 2025^
order^EO 14194^Imposing Duties To Address the Situation at Our Southern Border^^https://www.federalregister.gov/d/2025-02407^February 1, 2025^
order^EO 14193^Imposing Duties To Address the Flow of Illicit Drugs Across Our Northern Border^^https://www.federalregister.gov/d/2025-02406^February 1, 2025^
order^EO 14192^Unleashing Prosperity Through Deregulation ^^https://www.federalregister.gov/d/2025-02345^January 31, 2025^
order^EO 14191^Expanding Educational Freedom and Opportunity for Families^^https://www.federalregister.gov/d/2025-02233^January 29, 2025^
order^EO 14190^Ending Radical Indoctrination in K-12 Schooling ^^https://www.federalregister.gov/d/2025-02232^January 29, 2025^
order^EO 14189^Celebrating Americas 250th Birthday^^https://www.federalregister.gov/d/2025-02231^January 29, 2025^
order^EO 14188^Additional Measures To Combat Anti-Semitism^^https://www.federalregister.gov/d/2025-02230^January 29, 2025^
order^EO 14187^Protecting Children From Chemical and Surgical Mutilation^^https://www.federalregister.gov/d/2025-02194^January 28, 2025^
order^EO 14186^The Iron Dome for America ^^https://www.federalregister.gov/d/2025-02182^January 27, 2025^
order^EO 14185^Restoring Americas Fighting Force^^https://www.federalregister.gov/d/2025-02181^January 27, 2025^
order^EO 14184^Reinstating Service Members Discharged Under the Militarys COVID-19 Vaccination Mandate^^https://www.federalregister.gov/d/2025-02180^January 27, 2025^
order^EO 14183^Prioritizing Military Excellence and Readiness^^https://www.federalregister.gov/d/2025-02178^January 27, 2025^
order^EO 14182^Enforcing the Hyde Amendment^^https://www.federalregister.gov/d/2025-02175^January 24, 2025^
order^EO 14181^Emergency Measures To Provide Water Resources in California and Improve Disaster Response in Certain Areas^^https://www.federalregister.gov/d/2025-02174^January 24, 2025^
order^EO 14180^Council To Assess the Federal Emergency Management Agency^^https://www.federalregister.gov/d/2025-02173^January 24, 2025^
order^EO 14179^Removing Barriers to American Leadership in Artificial Intelligence^^https://www.federalregister.gov/d/2025-02172^January 23, 2025^
order^EO 14178^Strengthening American Leadership in Digital Financial Technology^^https://www.federalregister.gov/d/2025-02123^January 23, 2025^
order^EO 14177^Presidents Council of Advisors on Science and Technology^^https://www.federalregister.gov/d/2025-02121^January 23, 2025^
order^EO 14176^Declassification of Records Concerning the Assassinations of President John F. Kennedy, Senator Robert F. Kennedy, and the Reverend Dr. Martin Luther King, Jr.^^https://www.federalregister.gov/d/2025-02116^January 23, 2025^
order^EO 14175^Designation of Ansar Allah as a Foreign Terrorist Organization^^https://www.federalregister.gov/d/2025-02103^January 22, 2025^
order^EO 14174^Revocation of Certain Executive Orders ^^https://www.federalregister.gov/d/2025-02098^January 21, 2025^
order^EO 14173^Ending Illegal Discrimination and Restoring Merit-Based Opportunity^^https://www.federalregister.gov/d/2025-02097^January 21, 2025^
order^EO 14172^Restoring Names That Honor American Greatness^^https://www.federalregister.gov/d/2025-02096^January 20, 2025^
order^EO 14171^Restoring Accountability to Policy-Influencing Positions Within the Federal Workforce^^https://www.federalregister.gov/d/2025-02095^January 20, 2025^
order^EO 14170^Reforming the Federal Hiring Process and Restoring Merit to Government Service^^https://www.federalregister.gov/d/2025-02094^January 20, 2025^
order^EO 14169^Reevaluating and Realigning United States Foreign Aid^^https://www.federalregister.gov/d/2025-02091^January 20, 2025^
order^EO 14168^Defending Women From Gender Ideology Extremism and Restoring Biological Truth to the Federal Government^^https://www.federalregister.gov/d/2025-02090^January 20, 2025^
order^EO 14167^Clarifying the Militarys Role in Protecting the Territorial Integrity of the United States^^https://www.federalregister.gov/d/2025-02089^January 20, 2025^
order^EO 14166^Application of Protecting Americans From Foreign Adversary Controlled Applications Act to TikTok^^https://www.federalregister.gov/d/2025-02087^January 20, 2025^
order^EO 14165^Securing Our Borders^^https://www.federalregister.gov/d/2025-02015^January 20, 2025^
order^EO 14164^Restoring the Death Penalty and Protecting Public Safety^^https://www.federalregister.gov/d/2025-02012^January 20, 2025^
order^EO 14163^Realigning the United States Refugee Admissions Program^^https://www.federalregister.gov/d/2025-02011^January 20, 2025^
order^EO 14162^Putting America First in International Environmental Agreements^^https://www.federalregister.gov/d/2025-02010^January 20, 2025^
order^EO 14161^Protecting the United States From Foreign Terrorists and Other National Security and Public Safety Threats^^https://www.federalregister.gov/d/2025-02009^January 20, 2025^
order^EO 14160^Protecting the Meaning and Value of American Citizenship^^https://www.federalregister.gov/d/2025-02007^January 20, 2025^
order^EO 14159^Protecting the American People Against Invasion^^https://www.federalregister.gov/d/2025-02006^January 20, 2025^
order^EO 14158^"Establishing and Implementing the Presidents ""Department of Government Efficiency"""^^https://www.federalregister.gov/d/2025-02005^January 20, 2025^
order^EO 14157^Designating Cartels and Other Organizations as Foreign Terrorist Organizations and Specially Designated Global Terrorists^^https://www.federalregister.gov/d/2025-02004^January 20, 2025^
order^EO 14156^Declaring a National Energy Emergency^^https://www.federalregister.gov/d/2025-02003^January 20, 2025^
order^EO 14155^Withdrawing the United States From the World Health Organization ^^https://www.federalregister.gov/d/2025-01957^January 20, 2025^
order^EO 14154^Unleashing American Energy^^https://www.federalregister.gov/d/2025-01956^January 20, 2025^
order^EO 14153^Unleashing Alaskas Extraordinary Resource Potential^^https://www.federalregister.gov/d/2025-01955^January 20, 2025^
order^EO 14152^Holding Former Government Officials Accountable for Election Interference and Improper Disclosure of Sensitive Governmental Information^^https://www.federalregister.gov/d/2025-01954^January 20, 2025^
order^EO 14151^Ending Radical and Wasteful Government DEI Programs and Preferencing^^https://www.federalregister.gov/d/2025-01953^January 20, 2025^
order^EO 14150^America First Policy Directive to the Secretary of State^^https://www.federalregister.gov/d/2025-01952^January 20, 2025^
order^EO 14149^Restoring Freedom of Speech and Ending Federal Censorship^^https://www.federalregister.gov/d/2025-01902^January 20, 2025^
order^EO 14148^Initial Rescissions of Harmful Executive Orders and Actions^^https://www.federalregister.gov/d/2025-01901^January 20, 2025^
order^EO 14147^Ending the Weaponization of the Federal Government^^https://www.federalregister.gov/d/2025-01900^January 20, 2025^

Can't render this file because it contains an unexpected character in line 2 and column 16.

80
get_members.py Normal file
View File

@ -0,0 +1,80 @@
import requests
import csv
from dotenv import load_dotenv
import os
# Load environment variables from .env file
load_dotenv()
# Retrieve API key from environment variables
API_KEY = os.getenv('CONGRESS_API_KEY')
if not API_KEY:
raise ValueError("API key not found in .env file")
BASE_URL = 'https://api.congress.gov/v3/member'
# Parameters for the request
params = {
'offset': 0,
'limit': 250, # Maximum limit allowed is 250
'api_key': API_KEY
}
# Open a CSV file to write the data
with open('members.csv', 'w', newline='', encoding='utf-8') as csvfile:
csvwriter = csv.writer(csvfile, delimiter='^')
# Placeholder for headers
headers = None
row_count = 0
while True:
response = requests.get(BASE_URL, params=params)
# Print the retrieved data from the request
print("Retrieved Data:", response.json())
if response.status_code == 200:
data = response.json()
members = data.get('members', [])
for member in members:
depiction = member.pop('depiction', {})
# Ensure all keys are present by setting missing keys to None
flattened_member = {
'bioguideId': member.get('bioguideId'),
'attribution': depiction.get('attribution'),
'imageUrl': depiction.get('imageUrl'),
'district': member.get('district', ''),
'name': member.get('name'),
'partyName': member.get('partyName'),
'state': member.get('state'),
'chamber': member.get('terms', {}).get('item', [{}])[0].get('chamber'),
'startYear': member.get('terms', {}).get('item', [{}])[0].get('startYear'),
'endYear': member.get('terms', {}).get('item', [{}])[0].get('endYear'),
'updateDate': member.get('updateDate'),
'url': member.get('url')
}
# Extract keys to use as headers if not set yet
if headers is None:
headers = list(flattened_member.keys())
csvwriter.writerow(headers)
# Write each row to the CSV file
csvwriter.writerow(list(flattened_member.values()))
row_count += 1
print(f"Writing Row {row_count}: {flattened_member}")
# Check if there are more members to retrieve
if len(members) < params['limit']:
break
# Increment offset for the next batch of members
params['offset'] += params['limit']
else:
print(f"Failed to retrieve data: {response.status_code}")
break
print(f"Total rows written: {row_count}")

64
members_add_node.py Normal file
View File

@ -0,0 +1,64 @@
import csv
from neo4j import GraphDatabase
from dotenv import load_dotenv
import os
# Load environment variables from .env file
load_dotenv()
# Retrieve Neo4j connection details from environment variables
NEO4J_URI = os.getenv('NEO4J_URI')
NEO4J_USER = os.getenv('NEO4J_USER')
NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
if not NEO4J_URI or not NEO4J_USER or not NEO4J_PASSWORD:
raise ValueError("Neo4j connection details not found in .env file")
TEST_FLAG = False
# Function to create or update a person node in Neo4j using MERGE
def merge_person_node(driver, member):
with driver.session() as session:
query = """
MERGE (p:Person {bioguideId: $bioguideId})
ON CREATE SET p += $properties
ON MATCH SET p += $properties
RETURN p
"""
# Filter out empty or None values and ensure keys are strings
properties = {str(key): value for key, value in member.items() if value is not None and value != ""}
if TEST_FLAG:
print(f"MERGE (p:Person {{bioguideId: '{member['bioguideId']}'}}) ON CREATE SET p += ${properties} ON MATCH SET p += ${properties}")
else:
result = session.run(query, bioguideId=member['bioguideId'], properties=properties)
return result.single()
# Read the CSV file and create/update nodes in Neo4j
def read_csv_and_merge_nodes(file_path):
driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
try:
with open(file_path, mode='r', newline='', encoding='utf-8') as csvfile:
reader = csv.DictReader(csvfile, delimiter='^')
for row in reader:
member = {key: value.strip() if isinstance(value, str) else value for key, value in row.items()}
# Debugging print statement to check keys and values
print(f"Processing row with keys: {member.keys()} and values: {member.values()}")
result = merge_person_node(driver, member)
if not TEST_FLAG:
print(f"Created/Updated node: {result}")
finally:
if not TEST_FLAG:
driver.close()
# Path to the CSV file
csv_file_path = 'members.csv'
# Run the function
read_csv_and_merge_nodes(csv_file_path)

66
retrieve_bills.py Normal file
View File

@ -0,0 +1,66 @@
import requests
import csv
from dotenv import load_dotenv
import os
# Load environment variables from .env file
load_dotenv()
# Retrieve API key from environment variables
API_KEY = os.getenv('CONGRESS_API_KEY')
if not API_KEY:
raise ValueError("API key not found in .env file")
BASE_URL = 'https://api.congress.gov/v3/bill'
# Parameters for the request
params = {
'offset': 0,
'limit': 250, # Maximum limit allowed is 250
'api_key': API_KEY
}
# Open a CSV file to write the data
with open('bills.csv', 'w', newline='', encoding='utf-8') as csvfile:
csvwriter = csv.writer(csvfile, delimiter='^')
# Placeholder for headers
headers = None
row_count = 0
while True:
response = requests.get(BASE_URL, params=params)
# Print the retrieved data from the request
print("Retrieved Data:", response.json())
if response.status_code == 200:
data = response.json()
bills = data.get('bills', [])
for bill in bills:
# Flatten the nested 'latestAction' dictionary
latest_action = bill.pop('latestAction', {})
flattened_bill = {**bill, **latest_action}
# Extract keys to use as headers if not set yet
if headers is None:
headers = list(flattened_bill.keys())
csvwriter.writerow(headers)
# Write each row to the CSV file
csvwriter.writerow(list(flattened_bill.values()))
row_count += 1
print(f"Writing Row {row_count}: {flattened_bill}")
# Check if there are more bills to retrieve
if len(bills) < params['limit']:
break
# Increment offset for the next batch of bills
params['offset'] += params['limit']
else:
print(f"Failed to retrieve data: {response.status_code}")
break
print(f"Total rows written: {row_count}")

108
sponsored_add.py Normal file
View File

@ -0,0 +1,108 @@
import requests
from neo4j import GraphDatabase
from dotenv import load_dotenv
import os
# Load environment variables from .env file
load_dotenv()
# Retrieve Neo4j connection details and API key
NEO4J_URI = os.getenv('NEO4J_URI')
NEO4J_USER = os.getenv('NEO4J_USER')
NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
CONGRESS_API_KEY = os.getenv('CONGRESS_API_KEY')
if not NEO4J_URI or not NEO4J_USER or not NEO4J_PASSWORD:
raise ValueError("Neo4j connection details not found in .env file")
if not CONGRESS_API_KEY:
raise ValueError("Congress API key not found in .env file")
# Initialize Neo4j driver
driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
# Function to get all person nodes and their bioguideIds
def get_person_bioguide_ids(driver):
with driver.session() as session:
result = session.run("MATCH (p:Person) RETURN p.bioguideId")
return [record['p.bioguideId'] for record in result]
# Get list of person bioguideIds
person_bioguide_ids = get_person_bioguide_ids(driver)
print("List of Person Bioguide IDs:", person_bioguide_ids)
# Function to fetch sponsored legislation for a given bioguideId from Congress API
def fetch_sponsored_legislation(bioguide_id, api_key):
url = f"https://api.congress.gov/v3/member/{bioguide_id}/sponsored-legislation"
headers = {
'X-API-Key': api_key,
'Content-Type': 'application/json'
}
params = {
'limit': 250 # Maximum limit allowed is 250
}
response = requests.get(url, headers=headers, params=params)
if response.status_code == 200:
return response.json()
else:
print(f"Failed to retrieve sponsored legislation for bioguideId {bioguide_id}: {response.status_code}")
print("Response Content:", response.content.decode('utf-8'))
return None
# Fetch and process sponsored legislation for each person
legislation_dict = {}
for bioguide_id in person_bioguide_ids:
if bioguide_id is not None: # Check if the bioguideId is not None
print(f"Fetching sponsored legislation for member with bioguideId {bioguide_id}")
json_response = fetch_sponsored_legislation(bioguide_id, CONGRESS_API_KEY)
if json_response and 'legislations' in json_response:
for legislation in json_response['legislations']:
legislation_dict[legislation.get('id')] = legislation
# Create legislation nodes and sponsored relationships
with driver.session() as session:
for legislation_id, legislation in legislation_dict.items():
properties_str = ', '.join([f"'{key}': '{value}'" for key, value in legislation.items() if value is not None and value != ''])
# Check if the legislation node already exists
query_check = f"MATCH (l:Legislation {{id: '{legislation_id}'}}) RETURN l"
result_check = session.run(query_check)
legislation_exists = bool(result_check.data())
# Create or update the legislation node
query_create_update = f"MATCH (p:Person {{bioguideId: '{bioguide_id}'}}) MERGE (l:Legislation {{id: '{legislation_id}'}}) SET l = {{{properties_str}}} RETURN l"
result_create_update = session.run(query_create_update)
# Print the creation or update status
if legislation_exists:
print(f"Updated Legislation Node for member {bioguide_id}: {result_create_update.single()}")
else:
print(f"Created Legislation Node for member {bioguide_id}: {result_create_update.single()}")
# Create the sponsored relationship
query_relationship = f"MATCH (p:Person {{bioguideId: '{bioguide_id}'}}), (l:Legislation {{id: '{legislation_id}'}}) MERGE (p)-[:SPONSORED]->(l)"
session.run(query_relationship)
print(f"Created Sponsored Relationship for Person {bioguide_id} and Legislation {legislation_id}")
# Test that nodes and relationships were created properly
with driver.session() as session:
for bioguide_id, legislation in legislation_dict.items():
legislation_id = legislation['id']
# Check if the legislation node exists
query_check_legislation = f"MATCH (l:Legislation {{id: '{legislation_id}'}}) RETURN l"
result_check_legislation = session.run(query_check_legislation)
if not result_check_legislation.data():
print(f"Error: Legislation Node with ID {legislation_id} does not exist.")
# Check if the sponsored relationship exists
query_check_relationship = f"MATCH (p:Person {{bioguideId: '{bioguide_id}'}})-[:SPONSORED]->(l:Legislation {{id: '{legislation_id}'}}) RETURN p, l"
result_check_relationship = session.run(query_check_relationship)
if not result_check_relationship.data():
print(f"Error: Sponsored Relationship for Person {bioguide_id} and Legislation {legislation_id} does not exist.")
# Print success message
print(f"Success: Node and Relationship for Person {bioguide_id} and Legislation {legislation_id} are created properly.")
# Close the Neo4j driver connection
driver.close()

78
write_json_to_csv.py Normal file
View File

@ -0,0 +1,78 @@
import csv
import requests
import sys
def get_api_endpoint_map():
# Define a mapping of names to API endpoints and keys
return {
"member": {
"url": "https://api.congress.gov/v3/member",
"key": "NYjZ0jWEeOuLI5by1vfa7Z7orA717mvd2oIvalQe"
},
# Add more endpoints as needed
}
def fetch_json_data(endpoint_name):
api_endpoint_map = get_api_endpoint_map()
if endpoint_name not in api_endpoint_map:
print(f"Error: Endpoint '{endpoint_name}' is not defined.")
sys.exit(1)
endpoint_info = api_endpoint_map[endpoint_name]
url = endpoint_info["url"]
key = endpoint_info["key"]
# Make the HTTP GET request to fetch JSON data
headers = {
"X-API-KEY": key
}
response = requests.get(url, headers=headers)
if response.status_code != 200:
print(f"Error: Failed to fetch data. Status code {response.status_code}")
sys.exit(1)
json_data = response.json()
# Print the JSON response for debugging
print("JSON Response:")
print(json_data)
return json_data
def write_json_to_csv(json_data, filename="output.csv"):
# Open the CSV file for writing
with open(filename, mode='w', newline='', encoding='utf-8') as csvfile:
writer = csv.writer(csvfile, delimiter='^')
# Write the header row
if json_data and isinstance(json_data, dict) and 'members' in json_data:
data_list = json_data['members']
if data_list and len(data_list) > 0:
headers = data_list[0].keys()
writer.writerow(headers)
# Write each item to a new row
for item in data_list:
print(f"Writing row: {list(item.values())}") # Print statement added here
writer.writerow(item.values())
else:
print("No data to write.")
def main():
if len(sys.argv) != 2:
print("Usage: python script.py <endpoint_name>")
sys.exit(1)
endpoint_name = sys.argv[1]
try:
json_data = fetch_json_data(endpoint_name)
write_json_to_csv(json_data)
print(f"Data has been written to output.csv.")
except Exception as e:
print(f"An error occurred: {e}")
sys.exit(1)
if __name__ == "__main__":
main()