How do I scrape Gemini answers and citations in Python?

export QUERYING_API_KEY='<your-key>'
python3 querying_gemini.py 'best wireless earbuds for commuting'

#!/usr/bin/env python3
"""Scrape one consumer Gemini answer and print its text and citations."""

import argparse
import json
import os
import sys
import time
from urllib.error import HTTPError, URLError
from urllib.parse import quote
from urllib.request import Request, urlopen

BASE_URL = "https://api.querying.ai"


def request_json(method, path, api_key, payload=None, timeout=30):
    data = None if payload is None else json.dumps(payload).encode("utf-8")
    request = Request(
        BASE_URL + path,
        data=data,
        headers={
            "Authorization": "Bearer " + api_key,
            "Content-Type": "application/json",
        },
        method=method,
    )
    try:
        with urlopen(request, timeout=timeout) as response:
            result = json.load(response)
    except HTTPError as error:
        raise RuntimeError(f"API returned HTTP {error.code}") from error
    except (URLError, TimeoutError) as error:
        raise RuntimeError("Network request failed; check task status before resubmitting") from error
    if not isinstance(result, dict) or result.get("success") is False:
        raise RuntimeError("API did not return a successful response envelope")
    return result


def collect_answer(api_key, prompt, country="US", timeout=180, poll_interval=2):
    deadline = time.monotonic() + timeout
    accepted = request_json(
        "POST", "/v1/async/task", api_key,
        {"taskType": "GEMINI", "payload": {"prompt": prompt, "country": country}},
        timeout=min(30, timeout),
    )
    task_id = accepted.get("task", {}).get("id")
    if not isinstance(task_id, str) or not task_id:
        raise RuntimeError("Accepted response is missing task.id")
    path = "/v1/async/task/" + quote(task_id, safe="")
    while True:
        remaining = deadline - time.monotonic()
        if remaining <= 0:
            raise RuntimeError(f"Polling timed out; resume GET for task {task_id}")
        result = request_json("GET", path, api_key, timeout=min(30, remaining))
        status = result.get("task", {}).get("status")
        if status == "COMPLETED":
            response = result.get("response")
            if not isinstance(response, dict):
                raise RuntimeError("Completed task is missing response")
            if "text" not in response or "sources" not in response:
                raise RuntimeError("Completed Gemini response is missing text or sources")
            return {"task_id": task_id, "text": response["text"], "sources": response["sources"]}
        if status == "FAILED":
            raise RuntimeError(f"Task {task_id} failed; inspect its result in the dashboard")
        if status not in {"QUEUED", "RUNNING"}:
            raise RuntimeError(f"Task {task_id} returned an unexpected status: {status}")
        time.sleep(min(poll_interval, max(0, deadline - time.monotonic())))


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("prompt", nargs="?", default="best wireless earbuds for commuting")
    parser.add_argument("--country", default="US")
    parser.add_argument("--timeout", type=int, default=180)
    parser.add_argument("--poll-interval", type=int, default=2)
    args = parser.parse_args()
    if args.timeout <= 0 or args.poll_interval <= 0:
        parser.error("--timeout and --poll-interval must be positive")
    api_key = os.environ.get("QUERYING_API_KEY")
    if not api_key:
        parser.error("Set the QUERYING_API_KEY environment variable first")
    try:
        answer = collect_answer(api_key, args.prompt, args.country, args.timeout, args.poll_interval)
    except (RuntimeError, ValueError) as error:
        print(str(error), file=sys.stderr)
        return 1
    print(json.dumps(answer, ensure_ascii=False, indent=2))
    return 0


if __name__ == "__main__":
    sys.exit(main())