69 lines
1.9 KiB
Python
69 lines
1.9 KiB
Python
|
import os
|
||
|
|
import requests
|
||
|
|
from dotenv import load_dotenv
|
||
|
|
from neo4j import GraphDatabase
|
||
|
|
|
||
|
|
# Load environment variables from .env file
|
||
|
|
load_dotenv()
|
||
|
|
|
||
|
|
# Neo4j connection details
|
||
|
|
NEO4J_URI = os.getenv('NEO4J_URI')
|
||
|
|
NEO4J_USER = os.getenv('NEO4J_USER')
|
||
|
|
NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
|
||
|
|
|
||
|
|
# Congress API key
|
||
|
|
CONGRESS_API_KEY = os.getenv('CONGRESS_API_KEY')
|
||
|
|
|
||
|
|
# Initialize Neo4j driver
|
||
|
|
driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
|
||
|
|
|
||
|
|
def fetch_bills(offset):
|
||
|
|
url = f"https://api.congress.gov/v3/bill/117/sres?offset={offset}&api_key={CONGRESS_API_KEY}"
|
||
|
|
print(f"Fetching data from {url}")
|
||
|
|
response = requests.get(url)
|
||
|
|
if response.status_code == 200:
|
||
|
|
return response.json()
|
||
|
|
else:
|
||
|
|
raise Exception(f"Failed to fetch data: {response.status_code}")
|
||
|
|
|
||
|
|
def flatten_json(y):
|
||
|
|
out = {}
|
||
|
|
def flatten(x, name=''):
|
||
|
|
if type(x) is dict:
|
||
|
|
for a in x:
|
||
|
|
flatten(x[a], name + a + '_')
|
||
|
|
elif type(x) is list:
|
||
|
|
i = 0
|
||
|
|
for a in x:
|
||
|
|
flatten(a, name + str(i) + '_')
|
||
|
|
i += 1
|
||
|
|
else:
|
||
|
|
out[name[:-1]] = x
|
||
|
|
flatten(y)
|
||
|
|
return out
|
||
|
|
|
||
|
|
def create_bill_node(tx, bill_data):
|
||
|
|
flat_bill_data = flatten_json(bill_data)
|
||
|
|
print(f"Creating Bill node with properties: {flat_bill_data}")
|
||
|
|
query = "CREATE (b:Bill $properties)"
|
||
|
|
tx.run(query, properties=flat_bill_data)
|
||
|
|
|
||
|
|
def main():
|
||
|
|
offset = 0
|
||
|
|
while True:
|
||
|
|
try:
|
||
|
|
bills_data = fetch_bills(offset)
|
||
|
|
if not bills_data or len(bills_data['bills']) == 0:
|
||
|
|
break
|
||
|
|
with driver.session() as session:
|
||
|
|
for bill in bills_data['bills']:
|
||
|
|
session.write_transaction(create_bill_node, bill)
|
||
|
|
offset += 250
|
||
|
|
except Exception as e:
|
||
|
|
print(f"Error: {e}")
|
||
|
|
break
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|
||
|
|
driver.close()
|