From 6bc9fe964479d1f43c63f6e51e3e8a7e23aca332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Aristar=C3=A1n?= Date: Sun, 14 Feb 2021 16:31:11 -0300 Subject: [PATCH 1/3] Support specifying a view name (table:view) when querying --- airtable_export/cli.py | 28 +++++++++++++++++++++------- tests/test_airtable_export.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/airtable_export/cli.py b/airtable_export/cli.py index fc3f36c..51a73f7 100644 --- a/airtable_export/cli.py +++ b/airtable_export/cli.py @@ -32,10 +32,16 @@ def cli(output_path, base_id, tables, key, verbose, json, ndjson, yaml): output.mkdir(parents=True, exist_ok=True) if not json and not ndjson and not yaml: yaml = True - for table in tables: + for table_and_view in tables: + parts = table_and_view.split(":") + view = None + if len(parts) == 2: + table, view = parts + else: + table = parts[0] records = [] try: - for record in all_records(base_id, table, key): + for record in all_records(base_id, table, key, view=view): r = { **{"airtable_id": record["id"]}, **record["fields"], @@ -46,17 +52,17 @@ def cli(output_path, base_id, tables, key, verbose, json, ndjson, yaml): raise click.ClickException(exc) filenames = [] if json: - filename = "{}.json".format(table) + filename = "{}{}.json".format(table, "_" + view if view else "") dumped = json_.dumps(records, sort_keys=True, indent=4) (output / filename).write_text(dumped, "utf-8") filenames.append(output / filename) if ndjson: - filename = "{}.ndjson".format(table) + filename = "{}{}.ndjson".format(table, "_" + view if view else "") dumped = "\n".join(json_.dumps(r, sort_keys=True) for r in records) (output / filename).write_text(dumped, "utf-8") filenames.append(output / filename) if yaml: - filename = "{}.yml".format(table) + filename = "{}{}.yml".format(table, "_" + view if view else "") dumped = yaml_.dump(records, sort_keys=True) (output / filename).write_text(dumped, "utf-8") filenames.append(output / filename) @@ -71,14 +77,22 @@ def cli(output_path, base_id, tables, key, verbose, json, ndjson, yaml): ) -def all_records(base_id, table, api_key, sleep=0.2): +def all_records(base_id, table, api_key, view=None, sleep=0.2): first = True offset = None while first or offset: first = False url = "https://api.airtable.com/v0/{}/{}".format(base_id, quote(table)) + + query = {} if offset: - url += "?" + urlencode({"offset": offset}) + query["offset"] = offset + if view: + query["view"] = view + + if query: + url += "?" + urlencode(query) + response = httpx.get( url, headers={"Authorization": "Bearer {}".format(api_key)} ) diff --git a/tests/test_airtable_export.py b/tests/test_airtable_export.py index c82af8d..fccb0d4 100644 --- a/tests/test_airtable_export.py +++ b/tests/test_airtable_export.py @@ -103,6 +103,35 @@ def test_airtable_export(mocked, format, expected): ).read() assert expected.strip() == actual.strip() +@pytest.mark.parametrize("format,expected", FORMATS) +def test_airtable_export_with_view_name(mocked, format, expected): + runner = CliRunner() + with runner.isolated_filesystem(): + args = [ + ".", + "appZOGvNJPXCQ205F", + "tablename:viewname", + "-v", + "--key", + "x", + "--{}".format(format), + ] + result = runner.invoke( + cli.cli, + args, + ) + assert 0 == result.exit_code + assert ( + "Wrote 2 records to tablename_viewname.{}".format( + "yml" if format == "yaml" else format + ) + == result.output.strip() + ) + actual = open( + "tablename_viewname.{}".format("yml" if format == "yaml" else format) + ).read() + assert expected.strip() == actual.strip() + def test_all_three_formats_at_once(mocked): runner = CliRunner() From 032aa63a56a4ebe4cb53700a7af04446bb588d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Aristar=C3=A1n?= Date: Fri, 5 Mar 2021 17:24:22 -0300 Subject: [PATCH 2/3] merge with origin/main --- airtable_export/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtable_export/cli.py b/airtable_export/cli.py index 6a3314d..5735c1e 100644 --- a/airtable_export/cli.py +++ b/airtable_export/cli.py @@ -54,7 +54,7 @@ def cli(output_path, base_id, tables, key, verbose, json, ndjson, yaml, sqlite): records = [] try: db_batch = [] - for record in all_records(base_id, table, key): + for record in all_records(base_id, table, key, view=view): r = { **{"airtable_id": record["id"]}, **record["fields"], From a54b54de9b2818939db00985734ec0077cb12ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Aristar=C3=A1n?= Date: Tue, 30 Mar 2021 21:46:47 -0300 Subject: [PATCH 3/3] maybe airtable doesn't like us? --- airtable_export/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtable_export/cli.py b/airtable_export/cli.py index 5735c1e..d7f1db5 100644 --- a/airtable_export/cli.py +++ b/airtable_export/cli.py @@ -112,7 +112,7 @@ def all_records(base_id, table, api_key, view=None, sleep=0.2): url += "?" + urlencode(query) response = httpx.get( - url, headers={"Authorization": "Bearer {}".format(api_key)} + url, headers={"Authorization": "Bearer {}".format(api_key), "User-Agent": "curl/7.64.1"} ) response.raise_for_status() data = response.json()