Merge pull request #223 from kevinschaul/pictorial-api

unitedstates · Jan 3, 2025 · 5e7a9c1 · 5e7a9c1
2 parents ff693f6 + c570029
commit 5e7a9c1
Show file tree

Hide file tree

Showing 116 changed files with 147 additions and 53 deletions.
diff --git a/.gitignore b/.gitignore
@@ -23,6 +23,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+.python-version
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -73,3 +74,5 @@ congress/metadata/A000000.yaml
 # Thumbnail files
 .DS_Store
 thumbs.db
+
+errors.json
diff --git a/congress/225x275/A000380.jpg b/congress/225x275/A000380.jpg
diff --git a/congress/225x275/D000230.jpg b/congress/225x275/D000230.jpg
diff --git a/congress/225x275/D000530.jpg b/congress/225x275/D000530.jpg
diff --git a/congress/225x275/D000594.jpg b/congress/225x275/D000594.jpg
diff --git a/congress/225x275/D000633.jpg b/congress/225x275/D000633.jpg
diff --git a/congress/225x275/F000477.jpg b/congress/225x275/F000477.jpg
diff --git a/congress/225x275/F000479.jpg b/congress/225x275/F000479.jpg
diff --git a/congress/225x275/G000599.jpg b/congress/225x275/G000599.jpg
diff --git a/congress/225x275/G000600.jpg b/congress/225x275/G000600.jpg
diff --git a/congress/225x275/H001094.jpg b/congress/225x275/H001094.jpg
diff --git a/congress/225x275/H001096.jpg b/congress/225x275/H001096.jpg
diff --git a/congress/225x275/K000398.jpg b/congress/225x275/K000398.jpg
diff --git a/congress/225x275/K000399.jpg b/congress/225x275/K000399.jpg
diff --git a/congress/225x275/L000597.jpg b/congress/225x275/L000597.jpg
diff --git a/congress/225x275/L000600.jpg b/congress/225x275/L000600.jpg
diff --git a/congress/225x275/M001219.jpg b/congress/225x275/M001219.jpg
diff --git a/congress/225x275/M001221.jpg b/congress/225x275/M001221.jpg
diff --git a/congress/225x275/M001227.jpg b/congress/225x275/M001227.jpg
diff --git a/congress/225x275/M001228.jpg b/congress/225x275/M001228.jpg
diff --git a/congress/225x275/P000619.jpg b/congress/225x275/P000619.jpg
diff --git a/congress/225x275/R000617.jpg b/congress/225x275/R000617.jpg
diff --git a/congress/225x275/S001220.jpg b/congress/225x275/S001220.jpg
diff --git a/congress/225x275/S001221.jpg b/congress/225x275/S001221.jpg
diff --git a/congress/225x275/S001223.jpg b/congress/225x275/S001223.jpg
diff --git a/congress/225x275/T000487.jpg b/congress/225x275/T000487.jpg
diff --git a/congress/225x275/V000135.jpg b/congress/225x275/V000135.jpg
diff --git a/congress/450x550/A000380.jpg b/congress/450x550/A000380.jpg
diff --git a/congress/450x550/D000230.jpg b/congress/450x550/D000230.jpg
diff --git a/congress/450x550/D000530.jpg b/congress/450x550/D000530.jpg
diff --git a/congress/450x550/D000594.jpg b/congress/450x550/D000594.jpg
diff --git a/congress/450x550/D000633.jpg b/congress/450x550/D000633.jpg
diff --git a/congress/450x550/F000477.jpg b/congress/450x550/F000477.jpg
diff --git a/congress/450x550/F000479.jpg b/congress/450x550/F000479.jpg
diff --git a/congress/450x550/G000599.jpg b/congress/450x550/G000599.jpg
diff --git a/congress/450x550/G000600.jpg b/congress/450x550/G000600.jpg
diff --git a/congress/450x550/H001094.jpg b/congress/450x550/H001094.jpg
diff --git a/congress/450x550/H001096.jpg b/congress/450x550/H001096.jpg
diff --git a/congress/450x550/K000398.jpg b/congress/450x550/K000398.jpg
diff --git a/congress/450x550/K000399.jpg b/congress/450x550/K000399.jpg
diff --git a/congress/450x550/L000597.jpg b/congress/450x550/L000597.jpg
diff --git a/congress/450x550/L000600.jpg b/congress/450x550/L000600.jpg
diff --git a/congress/450x550/M001219.jpg b/congress/450x550/M001219.jpg
diff --git a/congress/450x550/M001221.jpg b/congress/450x550/M001221.jpg
diff --git a/congress/450x550/M001227.jpg b/congress/450x550/M001227.jpg
diff --git a/congress/450x550/M001228.jpg b/congress/450x550/M001228.jpg
diff --git a/congress/450x550/P000619.jpg b/congress/450x550/P000619.jpg
diff --git a/congress/450x550/R000617.jpg b/congress/450x550/R000617.jpg
diff --git a/congress/450x550/S001220.jpg b/congress/450x550/S001220.jpg
diff --git a/congress/450x550/S001221.jpg b/congress/450x550/S001221.jpg
diff --git a/congress/450x550/S001223.jpg b/congress/450x550/S001223.jpg
diff --git a/congress/450x550/T000487.jpg b/congress/450x550/T000487.jpg
diff --git a/congress/450x550/V000135.jpg b/congress/450x550/V000135.jpg
diff --git a/congress/metadata/A000380.yaml b/congress/metadata/A000380.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/B001319.yaml b/congress/metadata/B001319.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/B001320.yaml b/congress/metadata/B001320.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/D000230.yaml b/congress/metadata/D000230.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/D000530.yaml b/congress/metadata/D000530.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/D000594.yaml b/congress/metadata/D000594.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/D000633.yaml b/congress/metadata/D000633.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/F000477.yaml b/congress/metadata/F000477.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/F000479.yaml b/congress/metadata/F000479.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/G000599.yaml b/congress/metadata/G000599.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/G000600.yaml b/congress/metadata/G000600.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/H001094.yaml b/congress/metadata/H001094.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/H001096.yaml b/congress/metadata/H001096.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/J000307.yaml b/congress/metadata/J000307.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/J000309.yaml b/congress/metadata/J000309.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/K000398.yaml b/congress/metadata/K000398.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/K000399.yaml b/congress/metadata/K000399.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/L000597.yaml b/congress/metadata/L000597.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/L000599.yaml b/congress/metadata/L000599.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/L000600.yaml b/congress/metadata/L000600.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/L000601.yaml b/congress/metadata/L000601.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/L000602.yaml b/congress/metadata/L000602.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/M001217.yaml b/congress/metadata/M001217.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/M001219.yaml b/congress/metadata/M001219.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/M001221.yaml b/congress/metadata/M001221.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/M001222.yaml b/congress/metadata/M001222.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/M001227.yaml b/congress/metadata/M001227.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/M001228.yaml b/congress/metadata/M001228.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/P000619.yaml b/congress/metadata/P000619.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/R000617.yaml b/congress/metadata/R000617.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/S001220.yaml b/congress/metadata/S001220.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/S001221.yaml b/congress/metadata/S001221.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/S001223.yaml b/congress/metadata/S001223.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/T000487.yaml b/congress/metadata/T000487.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/V000135.yaml b/congress/metadata/V000135.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/metadata/V000137.yaml b/congress/metadata/V000137.yaml
@@ -0,0 +1,2 @@
+name: GPO Member Guide
+link: https://pictorial.gpo.gov
diff --git a/congress/original/A000380.jpg b/congress/original/A000380.jpg
diff --git a/congress/original/D000230.jpg b/congress/original/D000230.jpg
diff --git a/congress/original/D000530.jpg b/congress/original/D000530.jpg
diff --git a/congress/original/D000594.jpg b/congress/original/D000594.jpg
diff --git a/congress/original/D000633.jpg b/congress/original/D000633.jpg
diff --git a/congress/original/F000477.jpg b/congress/original/F000477.jpg
diff --git a/congress/original/F000479.jpg b/congress/original/F000479.jpg
diff --git a/congress/original/G000599.jpg b/congress/original/G000599.jpg
diff --git a/congress/original/G000600.jpg b/congress/original/G000600.jpg
diff --git a/congress/original/H001094.jpg b/congress/original/H001094.jpg
diff --git a/congress/original/H001096.jpg b/congress/original/H001096.jpg
diff --git a/congress/original/K000398.jpg b/congress/original/K000398.jpg
diff --git a/congress/original/K000399.jpg b/congress/original/K000399.jpg
diff --git a/congress/original/L000597.jpg b/congress/original/L000597.jpg
diff --git a/congress/original/L000600.jpg b/congress/original/L000600.jpg
diff --git a/congress/original/M001219.jpg b/congress/original/M001219.jpg
diff --git a/congress/original/M001221.jpg b/congress/original/M001221.jpg
diff --git a/congress/original/M001227.jpg b/congress/original/M001227.jpg
diff --git a/congress/original/M001228.jpg b/congress/original/M001228.jpg
diff --git a/congress/original/P000619.jpg b/congress/original/P000619.jpg
diff --git a/congress/original/R000617.jpg b/congress/original/R000617.jpg
diff --git a/congress/original/S001220.jpg b/congress/original/S001220.jpg
diff --git a/congress/original/S001221.jpg b/congress/original/S001221.jpg
diff --git a/congress/original/S001223.jpg b/congress/original/S001223.jpg
diff --git a/congress/original/T000487.jpg b/congress/original/T000487.jpg
diff --git a/congress/original/V000135.jpg b/congress/original/V000135.jpg
diff --git a/scripts/gpo_member_photos.py b/scripts/gpo_member_photos.py
@@ -8,15 +8,15 @@
 import json
 import os
 import re
-import sys
 import time
 from urllib.error import HTTPError
-from urllib.parse import urlencode
 from urllib.request import urlretrieve
 
 # pip install -r requirements.txt
 import mechanicalsoup
 
+CURRENT_CONGRESS = 118
+
 USER_AGENT = (
     "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 "
     "(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
@@ -44,50 +44,38 @@ def pause(last, delay):
     return datetime.datetime.now()
 
 
-def get_photo_list(br, congress_number, delay):
-    last_request_time = None
-
-    page = 1
-    while True:
-        # Fetch a page of results from Congress.gov.
-        print(f"Page {page} of Congress.gov Member listing...")
-        response = br.get(
-            "https://www.congress.gov/search?"
-            + urlencode(
-                {
-                    "q": json.dumps(
-                        {"source": "members", "congress": str(congress_number)}
-                    ),
-                    "pageSize": 250,
-                    "page": page,
-                }
-            )
-        ).text
-
-        if len(response) == 0:
-            sys.exit("Page is blank. Try again later, you may have hit a limit.")
-
-        # Scan for links to Member pages and img tags. The link to the
-        # Congress.gov page uses the Member's Bioguide ID as the key, and the
-        # filename for the photo is the same file name found at
-        # memberguide.gpo.gov for the high-resolution file.
-        for bioguide_id, photo_file in regex1.findall(response):
-            # this part is added by Congress.gov:
-            photo_file = photo_file.replace("_200.jpg", ".jpg")
-            if photo_file == bioguide_id.lower() + ".jpg":
-                continue  # not a file sourced from GPO
-            yield (bioguide_id, photo_file)
-
-        m = regex2.search(response)
-        if m:
-            # fetch next page of results
-            page += 1
-            continue
-        else:
-            # this was the last page (no Next link)
-            break
-
-        last_request_time = pause(last_request_time, delay)
+def get_members_pictorial(br, congress_number):
+    """
+    Get members for the given congress_number
+    API documentation: https://pictorialapi.gpo.gov/index.html
+    """
+    response = br.get(
+        f"https://pictorialapi.gpo.gov/api/GuideMember/GetMembers/{congress_number}"
+    ).json()
+    return [
+        member
+        for member in response["memberCollection"]
+        if member["memberType"] in ("Senator", "Representative")
+        and member["name"] != "Vacant, Vacant"
+    ]
+
+
+def get_legislators_current(br, include_historical=False):
+    """
+    Download legislators from sister project unitedstates/congress-legislators
+    Optionally also include historical legislators (which significantly
+    increases the download size)
+    """
+    legislators = br.get(
+        "https://theunitedstates.io/congress-legislators/legislators-current.json"
+    ).json()
+    if include_historical:
+        historical = br.get(
+            "https://theunitedstates.io/congress-legislators/"
+            "legislators-historical.json"
+        ).json()
+        legislators += historical
+    return legislators
 
 
 def save_metadata(bioguide_id):
@@ -97,7 +85,7 @@ def save_metadata(bioguide_id):
     outfile = os.path.join(outdir, bioguide_id + ".yaml")
     with open(outfile, "w") as f:
         f.write("name: GPO Member Guide\n")
-        f.write("link: https://memberguide.gpo.gov\n")
+        f.write("link: https://pictorial.gpo.gov\n")
 
 
 def download_file(url, outfile):
@@ -120,8 +108,7 @@ def download_photos(br, photo_list, outdir, delay):
 
     ok = 0
 
-    for bioguide_id, photo_filename in photo_list:
-        photo_url = "https://memberguide.gpo.gov/PictorialImages/" + photo_filename
+    for bioguide_id, photo_url in photo_list:
         print(bioguide_id, photo_url)
 
         filename = os.path.join(outdir, bioguide_id + ".jpg")
@@ -148,14 +135,15 @@ def resize_photos():
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        description="Scrape https://memberguide.gpo.gov and save "
-        "members' photos named after their Bioguide IDs",
+        description="Save members' photos from pictorialapi.gpo.gov, named "
+        "after their Bioguide IDs",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
     parser.add_argument(
         "-n",
         "--congress",
-        default="114",
+        default=CURRENT_CONGRESS,
+        type=int,
         help="Congress session number, for example: 110, 111, 112, 113",
     )
     parser.add_argument(
@@ -183,11 +171,42 @@ def resize_photos():
     br = mechanicalsoup.Browser()
     br.set_user_agent(USER_AGENT)
 
-    photo_list = get_photo_list(br, args.congress, args.delay)
+    legislators_current = get_legislators_current(br, args.congress != CURRENT_CONGRESS)
+    members_pictorial = get_members_pictorial(br, args.congress)
+
+    photo_list = []
+    errors = []
+    for m in legislators_current:
+        image_found = False
+        if "pictorial" in m["id"]:
+            try:
+                pictorial_data = next(
+                    p
+                    for p in members_pictorial
+                    if p["memberId"] == m["id"]["pictorial"]
+                )
+
+                if "nophotoimage.jpg" in pictorial_data["imageUrl"]:
+                    pass
+                else:
+                    image_found = True
+                    photo_list.append((m["id"]["bioguide"], pictorial_data["imageUrl"]))
+            except StopIteration:
+                # No matching result from pictorial API
+                pass
+
+        if not image_found:
+            print(f"No photo available for {m['id']['bioguide']}")
+            errors.append(["No photo available", m["id"]["bioguide"], m["name"]])
 
     number = download_photos(br, photo_list, args.outdir, args.delay)
 
     if number:
         resize_photos()
 
+    if len(errors):
+        print(f"{len(errors)} legislators had errors. Details wrote to errors.json")
+        with open("errors.json", "w") as f:
+            json.dump(errors, f, indent=2)
+
 # End of file
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		name: GPO Member Guide
		link: https://pictorial.gpo.gov