add bills and extended properties

add_bill_type.py

@@ -0,0 +1,68 @@
+import os
+import requests
+from dotenv import load_dotenv
+from neo4j import GraphDatabase
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Neo4j connection details
+NEO4J_URI = os.getenv('NEO4J_URI')
+NEO4J_USER = os.getenv('NEO4J_USER')
+NEO4J_PASSWORD = os.getenv('NEO4J_PASSWORD')
+
+# Congress API key
+CONGRESS_API_KEY = os.getenv('CONGRESS_API_KEY')
+
+# Initialize Neo4j driver
+driver = GraphDatabase.driver(NEO4J_URI, auth=(NEO4J_USER, NEO4J_PASSWORD))
+
+def fetch_bills(offset):
+    url = f"https://api.congress.gov/v3/bill/117/sres?offset={offset}&api_key={CONGRESS_API_KEY}"
+    print(f"Fetching data from {url}")
+    response = requests.get(url)
+    if response.status_code == 200:
+        return response.json()
+    else:
+        raise Exception(f"Failed to fetch data: {response.status_code}")
+
+def flatten_json(y):
+    out = {}
+    def flatten(x, name=''):
+        if type(x) is dict:
+            for a in x:
+                flatten(x[a], name + a + '_')
+        elif type(x) is list:
+            i = 0
+            for a in x:
+                flatten(a, name + str(i) + '_')
+                i += 1
+        else:
+            out[name[:-1]] = x
+    flatten(y)
+    return out
+
+def create_bill_node(tx, bill_data):
+    flat_bill_data = flatten_json(bill_data)
+    print(f"Creating Bill node with properties: {flat_bill_data}")
+    query = "CREATE (b:Bill $properties)"
+    tx.run(query, properties=flat_bill_data)
+
+def main():
+    offset = 0
+    while True:
+        try:
+            bills_data = fetch_bills(offset)
+            if not bills_data or len(bills_data['bills']) == 0:
+                break
+            with driver.session() as session:
+                for bill in bills_data['bills']:
+                    session.write_transaction(create_bill_node, bill)
+            offset += 250
+        except Exception as e:
+            print(f"Error: {e}")
+            break
+
+if __name__ == "__main__":
+    main()
+    driver.close()

api/app.py

@@ -9,7 +9,7 @@ import os
 import importlib.util
 
 load_dotenv()
-app = Flask(__name__)
+app = Flask(__name__, static_folder='static', template_folder='templates')
 app.wsgi_app = ProxyFix(app.wsgi_app)
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -58,5 +58,9 @@ def sitemap():
 def render_member():
     return render_template('render_member.html')
 
+@app.route('/render_network')
+def render_network():
+    return render_template('render_network.html')
+
 if __name__ == '__main__':
     app.run(debug=True)

api/endpoints/nodes.py

@@ -1,4 +1,3 @@
-# endpoints/nodes.py
 from flask import Blueprint, request, jsonify
 from app import get_driver, neo4j_logger
 
@@ -13,20 +12,17 @@ def nodes():
 
 def get_nodes():
     node_type = request.args.get('type')
-
     if not node_type:
         return jsonify({"error": "Node type is required"}), 400
 
-    # Get the filter parameter
     filter_property = request.args.get('filter')
-
     driver = get_driver()
+
+    try:
         with driver.session() as session:
             query = f"MATCH (n:{node_type}) RETURN n"
             neo4j_logger.info(f"Executing query: {query}")
             nodes = session.run(query)
-
-        # Convert the nodes to a list of dictionaries
             nodes_list = [
                 {
                     'id': record['n'].id,
@@ -37,12 +33,18 @@ def get_nodes():
             ]
 
         if filter_property:
-        # Filter the results to only include the specified property
-        filtered_nodes_list = [{filter_property: node.get(filter_property)} for node in nodes_list]
+            filtered_nodes_list = []
+            for node in nodes_list:
+                if filter_property in node:
+                    filtered_nodes_list.append({filter_property: node[filter_property]})
             return jsonify({"nodes": filtered_nodes_list})
 
         return jsonify({"nodes": nodes_list})
 
+    except Exception as e:
+        neo4j_logger.error(f"Error interacting with Neo4j: {e}")
+        return jsonify({"error": "An error occurred while interacting with the database"}), 500
+
 def create_node():
     data = request.get_json()
     if not data:
@@ -56,11 +58,11 @@ def create_node():
     query = f"CREATE (n:{node_type} {{{properties}}}) RETURN n"
 
     driver = get_driver()
+    try:
         with driver.session() as session:
             neo4j_logger.info(f"Executing query: {query} with data: {data}")
             result = session.run(query, **data)
 
-        # Convert the created node to a dictionary
             new_node = {
                 'id': result.single()['n'].id,
                 'labels': list(result.single()['n'].labels),
@@ -68,3 +70,7 @@ def create_node():
             }
 
         return jsonify(new_node), 201
+
+    except Exception as e:
+        neo4j_logger.error(f"Error interacting with Neo4j: {e}")
+        return jsonify({"error": "An error occurred while interacting with the database"}), 500

api/endpoints/person_network.py

@@ -0,0 +1,64 @@
+# endpoints/person_network.py
+
+from flask import Blueprint, jsonify
+import neo4j
+from neo4j import GraphDatabase
+from app import get_driver, neo4j_logger
+
+bp = Blueprint('network', __name__)
+
+@bp.route('/person_network', methods=['GET'])
+def person_network():
+    driver = get_driver()
+
+    try:
+        with driver.session() as session:
+            query = """
+            MATCH (p:Person)-[r]->(n)
+            RETURN p, r, n
+            """
+            result = session.run(query)
+
+            nodes = {}
+            links = []
+
+            for record in result:
+                person_node = record['p']
+                relationship = record['r']
+                connected_node = record['n']
+
+                # Add Person node if not already added
+                person_id = person_node.id
+                if person_id not in nodes:
+                    properties = {key: value for key, value in person_node.items()}
+                    properties['id'] = person_id
+                    properties['labels'] = list(person_node.labels)  # Include labels separately
+                    nodes[person_id] = {
+                        **properties,
+                        "group": 1  # Group for Person
+                    }
+
+                # Add connected node if not already added
+                connected_id = connected_node.id
+                if connected_id not in nodes:
+                    properties = {key: value for key, value in connected_node.items()}
+                    properties['id'] = connected_id
+                    properties['labels'] = list(connected_node.labels)  # Include labels separately
+                    nodes[connected_id] = {
+                        **properties,
+                        "group": 2  # Group for other nodes (e.g., Organization, Position)
+                    }
+
+                # Add link
+                links.append({
+                    "source": person_id,
+                    "target": connected_id,
+                    "type": relationship.type,
+                    "rel_properties": {key: value for key, value in relationship.items()}
+                })
+
+        return jsonify({"nodes": list(nodes.values()), "links": links})
+
+    except Exception as e:
+        print(f"Error interacting with Neo4j: {e}")
+        return jsonify({"error": f"An error occurred while interacting with the database: {str(e)}"}), 500

api/templates/render_member.html

@@ -1,128 +1,68 @@
-<!-- templates/render_member.html -->
 <!DOCTYPE html>
 <html lang="en">
 <head>
     <meta charset="UTF-8">
-    <title>Member Network</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Render Members</title>
+    <!-- Include D3.js -->
     <script src="https://d3js.org/d3.v7.min.js"></script>
     <style>
-        .node {
-            stroke: #fff;
-            stroke-width: 1.5px;
+        /* Optional: Add some basic styling */
+        #chart {
+            border: 1px solid black;
+            margin: 20px auto;
+            display: block;
         }
-        .link {
-            stroke: #999;
-            stroke-opacity: 0.6;
+        text {
+            dominant-baseline: middle;
+            text-anchor: middle;
         }
     </style>
 </head>
 <body>
-    <h1>Member Network</h1>
-    <select id="memberSelect">
-        <!-- Options will be populated dynamically -->
-    </select>
-    <div id="graph"></div>
+    <div id="chart"></div>
 
-    <script>
-        // Fetch member names from Neo4j
-        fetch('/get_member_names')
-            .then(response => response.json())
-            .then(members => {
-                const select = document.getElementById('memberSelect');
-                members.forEach(member => {
-                    const option = document.createElement('option');
-                    option.value = member;
-                    option.text = member;
-                    select.appendChild(option);
-                });
-            });
+    <script type="text/javascript">
+        // Fetch data from the Flask endpoint for Person nodes
+        d3.json('/nodes?type=Person').then(function(data) {
+            console.log("Fetched Data:", data);
 
-        // Handle member selection
-        document.getElementById('memberSelect').addEventListener('change', function() {
-            console.log("Member selected:", this.value); // Log the selected member name
-
-            const selectedMember = this.value;
-            fetch(`/list_members?member_name=${encodeURIComponent(selectedMember)}`, {
-                method: 'GET',
-            })
-            .then(response => response.json())
-            .then(data => {
-                console.log("Graph data received:", data); // Log the graph data
-                renderGraph(data);
-            })
-            .catch(error => {
-                console.error("Error fetching graph data:", error); // Log any errors
-            });
-        });
-
-        // Render D3 network graph
-        function renderGraph(data) {
-            const width = 800;
-            const height = 600;
-
-            const svg = d3.select("#graph")
-                .selectAll("svg") // Remove existing SVG elements first
-                .remove()
+            var svg = d3.select("#chart")
                 .append("svg")
-                .attr("width", width)
-                .attr("height", height);
+                .attr("width", 1000)
+                .attr("height", 1000);
 
-            const simulation = d3.forceSimulation(data.nodes)
-                .force("link", d3.forceLink(data.links).id(d => d.id))
-                .force("charge", d3.forceManyBody().strength(-500))
-                .force("center", d3.forceCenter(width / 2, height / 2));
+            // Function to generate random position within the SVG canvas
+            function getRandomPosition() {
+                return Math.random() * 950 + 25; // Random x or y between 25 and 975 to keep nodes inside the border
+            }
 
-            const link = svg.append("g")
-                .attr("class", "links")
-                .selectAll("line")
-                .data(data.links)
-                .enter().append("line")
-                .attr("stroke-width", 2);
-
-            const node = svg.append("g")
-                .attr("class", "nodes")
-                .selectAll("circle")
+            // Render circles for each node with green color and random positions
+            svg.selectAll("circle")
                 .data(data.nodes)
-                .enter().append("circle")
-                .attr("r", 5)
-                .attr("fill", d => d.label === 'Person' ? 'steelblue' : 'orange')
-                .call(d3.drag()
-                    .on("start", dragstarted)
-                    .on("drag", dragged)
-                    .on("end", dragended));
+                .enter()
+                .append("circle")
+                .attr("cx", getRandomPosition) // Random x position
+                .attr("cy", getRandomPosition) // Random y position
+                .attr("r", 10)
+                .style("fill", "green"); // Set node color to green
 
-            node.append("title")
-                .text(d => `${d.name} (${d.label})`);
-
-            simulation.on("tick", () => {
-                link
-                    .attr("x1", d => d.source.x)
-                    .attr("y1", d => d.source.y)
-                    .attr("x2", d => d.target.x)
-                    .attr("y2", d => d.target.y);
-
-                node
-                    .attr("cx", d => d.x)
-                    .attr("cy", d => d.y);
+            // Add labels for each node using the 'name' and 'bioguideId' properties
+            svg.selectAll("text.label")
+                .data(data.nodes)
+                .enter()
+                .append("text")
+                .attr("class", "label")
+                .attr("x", function(d) { return d3.select(this.previousSibling).attr("cx"); }) // Position slightly to the right of the circle
+                .attr("y", function(d) { return d3.select(this.previousSibling).attr("cy") + 15; }) // Position below the circle
+                .text(function(d) {
+                    return `${d.name || 'Person'}\nID: ${d.bioguideId || 'N/A'}`; // Single line with both name and ID
+                })
+                .style("fill", "black")
+                .style("font-size", "12px");
+        }).catch(function(error) {
+            console.error("Error fetching data:", error);
         });
-
-            function dragstarted(event, d) {
-                if (!event.active) simulation.alphaTarget(0.3).restart();
-                d.fx = d.x;
-                d.fy = d.y;
-            }
-
-            function dragged(event, d) {
-                d.fx = event.x;
-                d.fy = event.y;
-            }
-
-            function dragended(event, d) {
-                if (!event.active) simulation.alphaTarget(0);
-                d.fx = null;
-                d.fy = null;
-            }
-        }
     </script>
 </body>
 </html>

api/templates/render_network.html

@@ -0,0 +1,195 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Render Network</title>
+    <!-- Include D3.js -->
+    <script src="https://d3js.org/d3.v7.min.js"></script>
+    <style>
+        body {
+            margin: 0;
+            overflow: hidden;
+        }
+        #chart {
+            width: 100vw;
+            height: 100vh;
+            display: flex;
+            justify-content: center;
+            align-items: center;
+        }
+        .node {
+            stroke: #fff;
+            stroke-width: 1.5px;
+            cursor: pointer; /* Change cursor to indicate interactivity */
+        }
+        .link {
+            fill: none;
+            stroke: #999;
+            stroke-opacity: 0.6;
+        }
+        text {
+            font-size: 12px;
+        }
+        /* Tooltip styles */
+        .tooltip {
+            position: absolute;
+            padding: 10px;
+            background-color: rgba(255, 255, 255, 0.9);
+            border: 1px solid #ccc;
+            pointer-events: none; /* Make sure the tooltip doesn't interfere with mouse events */
+        }
+    </style>
+</head>
+<body>
+    <div id="chart"></div>
+
+    <!-- Tooltip container -->
+    <div class="tooltip" style="display: none;"></div>
+
+    <script type="text/javascript">
+        // Get container dimensions
+        const chartContainer = d3.select("#chart");
+        let width = parseInt(chartContainer.style("width"));
+        let height = parseInt(chartContainer.style("height"));
+
+        // Create SVG container
+        const svg = chartContainer.append("svg")
+            .attr("width", width)
+            .attr("height", height);
+
+        // Define simulation
+        const simulation = d3.forceSimulation()
+            .force("link", d3.forceLink().id(d => d.id).distance(100))
+            .force("charge", d3.forceManyBody().strength(-500)) // Adjust charge strength for better distribution
+            .force("center", d3.forceCenter(width / 2, height / 2));
+
+        // Fetch network data from Flask endpoint
+        d3.json('/person_network').then(function(data) {
+            if (data.error) {
+                console.error(data.error);
+                return;
+            }
+
+            const nodesData = data.nodes;
+            const linksData = data.links;
+
+            console.log("Nodes:", nodesData);
+            console.log("Links:", linksData);
+
+            // Append links to the SVG container
+            const link = svg.append("g")
+                .attr("class", "links")
+                .selectAll("line")
+                .data(linksData)
+                .enter()
+                .append("line")
+                .attr("class", "link");
+
+            // Append nodes to the SVG container
+            const node = svg.append("g")
+                .attr("class", "nodes")
+                .selectAll("circle")
+                .data(nodesData)
+                .enter()
+                .append("circle")
+                .attr("class", "node")
+                .attr("r", d => d.group === 1 ? 8 : 6) // Larger circles for Person nodes
+                .style("fill", d => d.group === 1 ? "green" : "lightblue") // Green for Person, light blue for others
+                .call(d3.drag()
+                    .on("start", dragstarted)
+                    .on("drag", dragged)
+                    .on("end", dragended))
+                .on("click", showTooltip);
+
+            // Append labels to the SVG container
+            const label = svg.append("g")
+                .attr("class", "labels")
+                .selectAll("text")
+                .data(nodesData)
+                .enter()
+                .append("text")
+                .text(d => {
+                    if (d.labels.includes('Person')) {
+                        return d.name || 'Person';
+                    } else if (['Legislation', 'Bill', 'Law'].some(label => d.labels.includes(label))) {
+                        return d.title || 'Title';
+                    }
+                    // Fallback for other types
+                    return Object.keys(d).filter(key => key !== 'labels').map(key => d[key]).join(', ') || 'Node';
+                })
+                .attr("dx", 10) // Offset from node
+                .attr("dy", 4); // Offset from node
+
+            // Initialize simulation with nodes and links
+            simulation.nodes(nodesData)
+                       .on("tick", ticked);
+            simulation.force("link")
+                       .links(linksData);
+
+            function ticked() {
+                link.attr("x1", d => d.source.x)
+                    .attr("y1", d => d.source.y)
+                    .attr("x2", d => d.target.x)
+                    .attr("y2", d => d.target.y);
+
+                node.attr("cx", d => Math.max(10, Math.min(width - 10, d.x)))
+                    .attr("cy", d => Math.max(10, Math.min(height - 10, d.y)));
+
+                label.attr("x", d => d.x)
+                     .attr("y", d => d.y);
+            }
+
+            function dragstarted(event, d) {
+                if (!event.active) simulation.alphaTarget(0.3).restart();
+                d.fx = d.x;
+                d.fy = d.y;
+            }
+
+            function dragged(event, d) {
+                d.fx = event.x;
+                d.fy = event.y;
+            }
+
+            function dragended(event, d) {
+                if (!event.active) simulation.alphaTarget(0);
+                d.fx = null;
+                d.fy = null;
+            }
+
+            // Show tooltip on node click
+            function showTooltip(event, d) {
+                const tooltip = d3.select(".tooltip");
+                tooltip.style("display", "block")
+                        .html(Object.entries(d).map(([key, value]) => `<strong>${key}:</strong> ${value}`).join("<br />"))
+                        .style("left", `${event.pageX}px`)
+                        .style("top", `${event.pageY}px`);
+            }
+
+            // Hide tooltip on document click
+            d3.select(document).on("click.tooltip", function() {
+                const tooltip = d3.select(".tooltip");
+                if (!d3.event.target.classList.contains("node")) {
+                    tooltip.style("display", "none");
+                }
+            });
+
+        }).catch(function(error) {
+            console.error("Error fetching data:", error);
+        });
+
+        // Handle window resize
+        window.addEventListener('resize', () => {
+            width = parseInt(chartContainer.style("width"));
+            height = parseInt(chartContainer.style("height"));
+
+            svg.attr("width", width)
+               .attr("height", height);
+
+            simulation.force("center")
+                       .x(width / 2)
+                       .y(height / 2);
+        });
+    </script>
+</body>
+</html>

update_bills.py

@@ -0,0 +1,99 @@
+import sys
+from neo4j import GraphDatabase
+from dotenv import load_dotenv
+import os
+import requests
+import json
+from flatten_json import flatten
+
+# Global variable to store the list of bill numbers
+bill_numbers = []
+
+def search_bills(bill_type):
+    # Load environment variables from .env file
+    load_dotenv()
+
+    # Get connection information from environment variables
+    uri = os.getenv('NEO4J_URI')
+    user = os.getenv('NEO4J_USER')
+    password = os.getenv('NEO4J_PASSWORD')
+
+    # Connect to Neo4j database
+    driver = GraphDatabase.driver(uri, auth=(user, password))
+
+    try:
+        with driver.session() as session:
+            # Query to find nodes with label 'Bill' and property 'type' matching the provided value
+            query = "MATCH (b:Bill) WHERE b.type = $bill_type RETURN b.number"
+
+            # Execute the query
+            result = session.run(query, bill_type=bill_type)
+
+            # Collect the list of bill numbers
+            global bill_numbers
+            bill_numbers = [record["b.number"] for record in result]
+    finally:
+        # Close the driver connection
+        driver.close()
+
+def get_bill_details(congress, bill_type, bill_number):
+    url = f"https://api.congress.gov/v3/bill/{congress}/{bill_type.lower()}/{bill_number}?format=json&api_key={os.getenv('CONGRESS_API_KEY')}"
+    response = requests.get(url)
+    if response.status_code == 200:
+        return response.json()
+    else:
+        print(f"Failed to fetch bill details for {bill_number}: {response.status_code}")
+        print(f"Response Text: {response.text}")
+        return None
+
+def flatten_json(y):
+    out = {}
+
+    def flatten(x, name=''):
+        if type(x) is dict:
+            for a in x:
+                flatten(x[a], name + a + '.')
+        elif type(x) is list:
+            i = 0
+            for a in x:
+                flatten(a, name + str(i) + '.')
+                i += 1
+        else:
+            out[name[:-1]] = x
+
+    flatten(y)
+    return {k.replace('bill.', ''): v for k, v in out.items()}
+
+def update_bill_node(driver, bill_number, properties):
+    with driver.session() as session:
+        # Remove existing properties
+        query_remove_properties = f"MATCH (b:Bill {{number: $bill_number}}) SET b += {{}}"
+        session.run(query_remove_properties, bill_number=bill_number)
+
+        # Add new properties
+        query_add_properties = f"MATCH (b:Bill {{number: $bill_number}}) SET b += $properties RETURN b"
+        session.run(query_add_properties, bill_number=bill_number, properties=properties)
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: python search_bills.py <congress> <bill_type>")
+        sys.exit(1)
+
+    congress = sys.argv[1]
+    bill_type = sys.argv[2]
+
+    search_bills(bill_type)
+
+    # Connect to Neo4j database
+    driver = GraphDatabase.driver(os.getenv('NEO4J_URI'), auth=(os.getenv('NEO4J_USER'), os.getenv('NEO4J_PASSWORD')))
+
+    for bill_number in bill_numbers:
+        print(f"Fetching details for bill number {bill_number}...")
+        bill_details = get_bill_details(congress, bill_type, bill_number)
+        if bill_details:
+            flattened_properties = flatten_json(bill_details)
+            update_bill_node(driver, bill_number, flattened_properties)
+            print(f"Updated bill node with properties from JSON response for bill number {bill_number}")
+
+    # Close the driver connection
+    driver.close()