67 lines
2.0 KiB
Python
67 lines
2.0 KiB
Python
|
import requests
|
||
|
|
import csv
|
||
|
|
from dotenv import load_dotenv
|
||
|
|
import os
|
||
|
|
|
||
|
|
# Load environment variables from .env file
|
||
|
|
load_dotenv()
|
||
|
|
|
||
|
|
# Retrieve API key from environment variables
|
||
|
|
API_KEY = os.getenv('CONGRESS_API_KEY')
|
||
|
|
if not API_KEY:
|
||
|
|
raise ValueError("API key not found in .env file")
|
||
|
|
|
||
|
|
BASE_URL = 'https://api.congress.gov/v3/bill'
|
||
|
|
|
||
|
|
# Parameters for the request
|
||
|
|
params = {
|
||
|
|
'offset': 0,
|
||
|
|
'limit': 250, # Maximum limit allowed is 250
|
||
|
|
'api_key': API_KEY
|
||
|
|
}
|
||
|
|
|
||
|
|
# Open a CSV file to write the data
|
||
|
|
with open('bills.csv', 'w', newline='', encoding='utf-8') as csvfile:
|
||
|
|
csvwriter = csv.writer(csvfile, delimiter='^')
|
||
|
|
|
||
|
|
# Placeholder for headers
|
||
|
|
headers = None
|
||
|
|
row_count = 0
|
||
|
|
|
||
|
|
while True:
|
||
|
|
response = requests.get(BASE_URL, params=params)
|
||
|
|
|
||
|
|
# Print the retrieved data from the request
|
||
|
|
print("Retrieved Data:", response.json())
|
||
|
|
|
||
|
|
if response.status_code == 200:
|
||
|
|
data = response.json()
|
||
|
|
bills = data.get('bills', [])
|
||
|
|
|
||
|
|
for bill in bills:
|
||
|
|
# Flatten the nested 'latestAction' dictionary
|
||
|
|
latest_action = bill.pop('latestAction', {})
|
||
|
|
flattened_bill = {**bill, **latest_action}
|
||
|
|
|
||
|
|
# Extract keys to use as headers if not set yet
|
||
|
|
if headers is None:
|
||
|
|
headers = list(flattened_bill.keys())
|
||
|
|
csvwriter.writerow(headers)
|
||
|
|
|
||
|
|
# Write each row to the CSV file
|
||
|
|
csvwriter.writerow(list(flattened_bill.values()))
|
||
|
|
row_count += 1
|
||
|
|
print(f"Writing Row {row_count}: {flattened_bill}")
|
||
|
|
|
||
|
|
# Check if there are more bills to retrieve
|
||
|
|
if len(bills) < params['limit']:
|
||
|
|
break
|
||
|
|
|
||
|
|
# Increment offset for the next batch of bills
|
||
|
|
params['offset'] += params['limit']
|
||
|
|
else:
|
||
|
|
print(f"Failed to retrieve data: {response.status_code}")
|
||
|
|
break
|
||
|
|
|
||
|
|
print(f"Total rows written: {row_count}")
|