feat: set user agent with --user-agent flag (#12)

* feat: set user agent with --user-agent flag
* docs: move request options after export options

Co-authored-by: Simon Willison <swillison@gmail.com>

Author: eliblock

README.md

@@ -55,6 +55,14 @@ The SQLite database will have a table created for each table you export. Those t
 
 If you run this command against an existing SQLite database records with matching primary keys will be over-written by new records from the export.
 
+## Request options
+
+By default the tool uses [python-httpx](https://www.python-httpx.org)'s default configurations.
+
+You can override the `user-agent` using the `--user-agent` option:
+
+    airtable-export export base_id table1 table2 --key=key --user-agent "Airtable Export Robot"
+
 ## Running this using GitHub Actions
 
 [GitHub Actions](https://github.com/features/actions) is GitHub's workflow automation product. You can use it to run `airtable-export` in order to back up your Airtable data to a GitHub repository. Doing this gives you a visible commit history of changes you make to your Airtable data - like [this one](https://github.com/natbat/rockybeaches/commits/main/airtable).

airtable_export/cli.py

@@ -23,6 +23,7 @@
 )
 @click.argument("tables", type=str, nargs=-1)
 @click.option("--key", envvar="AIRTABLE_KEY", help="Airtable API key", required=True)
+@click.option("--user-agent", help="User agent to use for requests")
 @click.option("-v", "--verbose", is_flag=True, help="Verbose output")
 @click.option("--json", is_flag=True, help="JSON format")
 @click.option("--ndjson", is_flag=True, help="Newline delimited JSON format")
@@ -32,7 +33,9 @@
     type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
     help="Export to this SQLite database",
 )
-def cli(output_path, base_id, tables, key, verbose, json, ndjson, yaml, sqlite):
+def cli(
+    output_path, base_id, tables, key, user_agent, verbose, json, ndjson, yaml, sqlite
+):
     "Export Airtable data to YAML file on disk"
     output = pathlib.Path(output_path)
     output.mkdir(parents=True, exist_ok=True)
@@ -48,7 +51,7 @@ def cli(output_path, base_id, tables, key, verbose, json, ndjson, yaml, sqlite):
         records = []
         try:
             db_batch = []
-            for record in all_records(base_id, table, key):
+            for record in all_records(base_id, table, key, user_agent=user_agent):
                 r = {
                     **{"airtable_id": record["id"]},
                     **record["fields"],
@@ -89,17 +92,19 @@ def cli(output_path, base_id, tables, key, verbose, json, ndjson, yaml, sqlite):
             )
 
 
-def all_records(base_id, table, api_key, sleep=0.2):
+def all_records(base_id, table, api_key, sleep=0.2, user_agent=None):
+    headers = {"Authorization": "Bearer {}".format(api_key)}
+    if user_agent is not None:
+        headers["user-agent"] = user_agent
+
     first = True
     offset = None
     while first or offset:
         first = False
         url = "https://api.airtable.com/v0/{}/{}".format(base_id, quote(table))
         if offset:
             url += "?" + urlencode({"offset": offset})
-        response = httpx.get(
-            url, headers={"Authorization": "Bearer {}".format(api_key)}
-        )
+        response = httpx.get(url, headers=headers)
         response.raise_for_status()
         data = response.json()
         offset = data.get("offset")