import requests
import csv
from dotenv import load_dotenv
import os

# Load environment variables from .env file
load_dotenv()

# Retrieve API key from environment variables
API_KEY = os.getenv('CONGRESS_API_KEY')
if not API_KEY:
    raise ValueError("API key not found in .env file")

BASE_URL = 'https://api.congress.gov/v3/member'

# Parameters for the request
params = {
    'offset': 0,
    'limit': 250,  # Maximum limit allowed is 250
    'api_key': API_KEY
}

# Open a CSV file to write the data
with open('members.csv', 'w', newline='', encoding='utf-8') as csvfile:
    csvwriter = csv.writer(csvfile, delimiter='^')

    # Placeholder for headers
    headers = None
    row_count = 0

    while True:
        response = requests.get(BASE_URL, params=params)

        # Print the retrieved data from the request
        print("Retrieved Data:", response.json())

        if response.status_code == 200:
            data = response.json()
            members = data.get('members', [])

            for member in members:
                depiction = member.pop('depiction', {})

                # Ensure all keys are present by setting missing keys to None
                flattened_member = {
                    'bioguideId': member.get('bioguideId'),
                    'attribution': depiction.get('attribution'),
                    'imageUrl': depiction.get('imageUrl'),
                    'district': member.get('district', ''),
                    'name': member.get('name'),
                    'partyName': member.get('partyName'),
                    'state': member.get('state'),
                    'chamber': member.get('terms', {}).get('item', [{}])[0].get('chamber'),
                    'startYear': member.get('terms', {}).get('item', [{}])[0].get('startYear'),
                    'endYear': member.get('terms', {}).get('item', [{}])[0].get('endYear'),
                    'updateDate': member.get('updateDate'),
                    'url': member.get('url')
                }

                # Extract keys to use as headers if not set yet
                if headers is None:
                    headers = list(flattened_member.keys())
                    csvwriter.writerow(headers)

                # Write each row to the CSV file
                csvwriter.writerow(list(flattened_member.values()))
                row_count += 1
                print(f"Writing Row {row_count}: {flattened_member}")

            # Check if there are more members to retrieve
            if len(members) < params['limit']:
                break

            # Increment offset for the next batch of members
            params['offset'] += params['limit']
        else:
            print(f"Failed to retrieve data: {response.status_code}")
            break

print(f"Total rows written: {row_count}")