81 lines
2.8 KiB
Python
81 lines
2.8 KiB
Python
|
import requests
|
||
|
|
import csv
|
||
|
|
from dotenv import load_dotenv
|
||
|
|
import os
|
||
|
|
|
||
|
|
# Load environment variables from .env file
|
||
|
|
load_dotenv()
|
||
|
|
|
||
|
|
# Retrieve API key from environment variables
|
||
|
|
API_KEY = os.getenv('CONGRESS_API_KEY')
|
||
|
|
if not API_KEY:
|
||
|
|
raise ValueError("API key not found in .env file")
|
||
|
|
|
||
|
|
BASE_URL = 'https://api.congress.gov/v3/member'
|
||
|
|
|
||
|
|
# Parameters for the request
|
||
|
|
params = {
|
||
|
|
'offset': 0,
|
||
|
|
'limit': 250, # Maximum limit allowed is 250
|
||
|
|
'api_key': API_KEY
|
||
|
|
}
|
||
|
|
|
||
|
|
# Open a CSV file to write the data
|
||
|
|
with open('members.csv', 'w', newline='', encoding='utf-8') as csvfile:
|
||
|
|
csvwriter = csv.writer(csvfile, delimiter='^')
|
||
|
|
|
||
|
|
# Placeholder for headers
|
||
|
|
headers = None
|
||
|
|
row_count = 0
|
||
|
|
|
||
|
|
while True:
|
||
|
|
response = requests.get(BASE_URL, params=params)
|
||
|
|
|
||
|
|
# Print the retrieved data from the request
|
||
|
|
print("Retrieved Data:", response.json())
|
||
|
|
|
||
|
|
if response.status_code == 200:
|
||
|
|
data = response.json()
|
||
|
|
members = data.get('members', [])
|
||
|
|
|
||
|
|
for member in members:
|
||
|
|
depiction = member.pop('depiction', {})
|
||
|
|
|
||
|
|
# Ensure all keys are present by setting missing keys to None
|
||
|
|
flattened_member = {
|
||
|
|
'bioguideId': member.get('bioguideId'),
|
||
|
|
'attribution': depiction.get('attribution'),
|
||
|
|
'imageUrl': depiction.get('imageUrl'),
|
||
|
|
'district': member.get('district', ''),
|
||
|
|
'name': member.get('name'),
|
||
|
|
'partyName': member.get('partyName'),
|
||
|
|
'state': member.get('state'),
|
||
|
|
'chamber': member.get('terms', {}).get('item', [{}])[0].get('chamber'),
|
||
|
|
'startYear': member.get('terms', {}).get('item', [{}])[0].get('startYear'),
|
||
|
|
'endYear': member.get('terms', {}).get('item', [{}])[0].get('endYear'),
|
||
|
|
'updateDate': member.get('updateDate'),
|
||
|
|
'url': member.get('url')
|
||
|
|
}
|
||
|
|
|
||
|
|
# Extract keys to use as headers if not set yet
|
||
|
|
if headers is None:
|
||
|
|
headers = list(flattened_member.keys())
|
||
|
|
csvwriter.writerow(headers)
|
||
|
|
|
||
|
|
# Write each row to the CSV file
|
||
|
|
csvwriter.writerow(list(flattened_member.values()))
|
||
|
|
row_count += 1
|
||
|
|
print(f"Writing Row {row_count}: {flattened_member}")
|
||
|
|
|
||
|
|
# Check if there are more members to retrieve
|
||
|
|
if len(members) < params['limit']:
|
||
|
|
break
|
||
|
|
|
||
|
|
# Increment offset for the next batch of members
|
||
|
|
params['offset'] += params['limit']
|
||
|
|
else:
|
||
|
|
print(f"Failed to retrieve data: {response.status_code}")
|
||
|
|
break
|
||
|
|
|
||
|
|
print(f"Total rows written: {row_count}")
|