From ff648492953bf31a663d52eb8e4a331799d1081d Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Thu, 11 Dec 2025 16:21:33 +0100 Subject: [PATCH 01/16] Extend import help, add currency/commodity synonym, link to beancount option help This fixes a few issues that were hampering me as a newcomer to understand the import process. The import help text is less dense now and presupposes less knowledge about importing in beancount and the detailed differences of the different generations of import mechanisms (beancount.ingest vs beangulp). There are clearer step-by-step instructions and an example that should get a user floating for a basic import-CSV case. Also, the full signature of hook function is now documented, which wasn't done anywhere in the documentations of beancount, beangulp, or fava. Additionally, this adds to the syntax overview the simple information that commoditys = currencies from the perspective of beancount. This would have helped me as a newcomer to find correct BQL functions for my needs. --- src/fava/help/beancount_syntax.md | 10 +- src/fava/help/import.md | 212 +++++++++++++++++++++++++++--- src/fava/help/options.md | 4 +- 3 files changed, 202 insertions(+), 24 deletions(-) diff --git a/src/fava/help/beancount_syntax.md b/src/fava/help/beancount_syntax.md index 670df9734..085e8fa6d 100644 --- a/src/fava/help/beancount_syntax.md +++ b/src/fava/help/beancount_syntax.md @@ -8,14 +8,16 @@ Beancount defines a language in which financial transactions are entered into a text-file, which then can be processed by Beancount. There are a few building blocks that are important to understand Beancount's syntax: -- Commodities, +- Commodities / Currencies, - Accounts, - Directives. -## Commodities +## Commodities / Currencies -All in CAPS: `USD`, `EUR`, `CAD`, `GOOG`, `AAPL`, `RBF1005`, `HOME_MAYST`, -`AIRMILES`, `HOURS`. +Anything that you want to track in some account is called a commodity in +Beancount. We sometimes also use the word "Currency" interchangeably in the +documentation. Write them in ALL-CAPS: `USD`, `EUR`, `CAD`, `GOOG`, `AAPL`, +`RBF1005`, `HOME_MAYST`, `AIRMILES`, `HOURS`. ## Accounts diff --git a/src/fava/help/import.md b/src/fava/help/import.md index 61863f9e4..cd38fcdf4 100644 --- a/src/fava/help/import.md +++ b/src/fava/help/import.md @@ -1,27 +1,201 @@ -# Import +# Importing transactions into Fava -Fava can use Beancount's import system to semi-automatically import entries into -your Beancount ledger. See +Fava integrates with Beancount's import system to automatically generate +transaction entries for your Beancount ledger from account statements. After +setting up your import process as explained below, the *Import* page in Fava +will show an upload button that lets you upload files into your *import folder*. +If you can access the import folder (e.g., if Fava is running on your local +machine), you can also just place your account statement files there directly. + +To import the file contents into your Beancount ledger, you must set up an +*Importer* that can process the file format. Fava lists each file in the import +folder. If any Importer matches the file, a button "Extract" will be visible. +This will import its contents into your ledger. Files without a matching +Importer appear as "Non-importable file". + +You can move any file into your +[documents folder](https://beancount.github.io/docs/beancount_language_syntax.html#documents-from-a-directory) +by clicking the button "Move". If you do not use the document folder feature of +beancount, you can delete the file from the import folder after the import is +complete by clicking the button with the cross at the top right. + +## Define the upload configuration in your Beancount ledger + +Add these lines to your Beancount file: + +
+ +The first line specifies the import folder location. The second line defines the +import configuration - a Python script that handles the format of your account +statements. Fava interprets relative paths relative to your main Beancount +ledger file location. You can also use absolute paths. + +## Write your import configuration + +Your import configuration must be a Python file that defines: + +- `CONFIG`: A list of Importers that process your account statement files. + YCreate subclasses of `beangulp.importers.Importer` with parsing logic for + your specific account statement formats, then add instances of each class to + this list. + +- `HOOKS`: A list of functions to apply to all directives (e.g., transactions) + after generation by any Importer. + +See the example below and the help document [Importing External Data in Beancount](http://furius.ca/beancount/doc/ingest) -for more information on how to write importers. - -You can override the hooks that should be run for your importers by specifying a -variable `HOOKS` with the a list of hooks to apply to the list of -`(filename: str, entries: list[Directive])` tuples. On Beancount version 2 the -duplicates detector hook will be run by default and no hooks will run by default -on Beancount version 3. If you want to use beangulp-style hooks that take list -of -`(filename: str, entries: list[Directive], account: str, importer: Importer)`-tuples, -you can annotate them with the appropriate Python types which Fava will detect -and call with these 4-tuples. - -Set the `import-config` option to point to your import config and set -`import-dirs` to the directories that contain the files that you want to import. -And if you wish to save new entries elsewhere than main file - use -`insert-entry` option. +for more information on how to write importers. Hook functions are explained in +more detail further down on this page. Fava currently only supports entries of type `Transaction`, `Balance`, and `Note`. Set the special metadata key `__source__` to display the corresponding text (CSV-row, XML-fragment, etc.) for the entry in the list of entries to import. Note that this metadata (and all other metadata keys starting with an underscore) will be stripped before saving the entry to file. + +# Example import configuration + +This is an example configuration to read in account statements provided as CSV: + +```python +import csv +import re + +import beangulp +import beangulp.importers.csvbase as csvbase +from beancount.core import data # Transaction, Posting, ... + +class MyCSVImporter(csvbase.Importer): + """Read a CSV file formatted in German style to demonstrate some formatting + options: Column separator = ";", decimals are like this: 2.032,43 (dot as + thousands separator, comma as decimal separator""" + + # The expected column names and formats in the input file are defined + # as member variables, that are instances of csvbase.Column or subclasses + # + # Required columns + date = csvbase.Date('Buchungstag', '%d.%m.%Y') # German date format + narration = csvbase.Column('Verwendungszweck') + # To parse amount, first remove dots, then translate commas to dots, + # to convert 2.032,43 -> 3032.43 + amount = csvbase.Amount('Betrag', subs={r'\.':'', r',': '.'}) + + # Optional further columns: + # flag, payee, account, currency, tag, link, balance. + + # Any additional members of type "Column" can be used by your own + # `finalize()` and `metadata()` functions, access e.g. as row.sepa_iban + sepa_iban = Column('IBAN') + + # The following variables set the CSV format (see csvbase.CSVReader): + # encoding = "utf8" # File encoding. + # header = 0 # Number of header lines to skip. + # footer = 0 # Number of footer lines to ignore. + # names = True # Whether the data file contains a row with column names. + # dialect = None # The CSV dialect used in the input file (str or csv.Delimiter object). + # comments = "#" # Comment character. + # order = None # Order of entries in the CSV file (Default: Infer from file) + # # can be csvbase.Order.ASCENDING or ...DESCENDING + + # Set CSV dialect to use semicolon as separator + dialect = csv.excel + dialect.delimiter = ';' + + def __init__(self): + super().__init__( + account="Assets:MyBank", # default if no account column is defined + currency="EUR", + flag = "*" # optional + ) + + def identify(self, filepath): + """Return True if this importer is suitable for the file of the given + name. This allows to auto-choose the right importer for a file.""" + return "MyBank" in filepath + + def read(self, filepath): + """Override the read method to preprocess the CSV file. + """ + # Add some pre-processing of the input file, then + # call the parent read method with the processed file + for row in super().read(filepath): + yield row + + def metadata(self, filepath, lineno, row): + """ + Called for each row of the input file to set the metadata of the + resulting transaction. + Arguments: + - filepath, lineno: The file path and line number + - row: Has attributes for each class member of type `Column` (or + subtypes) + """ + # Example: Add the values from the additional sepa_iban column as + # metadata + return data.new_metadata(filepath, lineno, {'iban': row.sepa_iban}) + + def finalize(self, txn, row): + """Called for each generated transaction `txn` to make user-defined + changes. The input data is available as attributes of `row`""" + # Example: Add a default second transaction leg to Expenses:Unknown + # Documentation of Transaction object (txt): + # https://beancount.github.io/docs/api_reference/beancount.core.html#beancount.core.data.Transaction + txn.postings.append(data.Posting( + account = "Expenses:Unknown", units = -txn.postings[0].units )) + + return txn + + +# All available importers, one for each file format you need to process +CONFIG = [ + MyCSVImporter() +] + +# Process beancount transaction objects after they have been extracted +HOOKS = [] + +# Allows to call this script as './import extract '. Not needed +# for Fava, but useful for debugging +if __name__ == "__main__": + ingest = beangulp.Ingest(CONFIG, HOOKS) + ingest() +``` + +## Hook functions + +Hook functions are applied to all imported transactions. A hook function +receives the parameters `hook_fn(new_entries_list, existing_entries)` and +returns the modified `new_entries_list`. + +The argument `new_entries_list` is itself a list of tuples. As Fava imports each +file individually (in in contrast to the CLI of Beangulp), this list will always +be of length 1. + +The tuples can can have two (old style, default) or four elements +(beangulp-style). The type signature of the tuples is either + +- `(filename: str, entries: list[Directive])` or +- `(filename: str, entries: list[Directive], account: str, importer: Importer)`. + +You can annotate the hook function with the appropriate Python types and Fava +will detect and call it with these 4-tuples. Types `Directive` and `Importer` +are `beancount.core.data.Directive` and `beangulp.Importer`, respectively. + +So, in summary, the type signature of a hook function is either: + +``` +hook_fn(new_entries_list: List[str, List[Directive]], + existing_entries: Sequence[Directive]) -> List[str, List[Directive]] +``` + +or + +``` +hook_fn(new_entries_list: List[str, list[Directive], str, Importer], + existing_entries: Sequence[Directive]) -> List[str, List[Directive]] +``` diff --git a/src/fava/help/options.md b/src/fava/help/options.md index c76ed78bc..7123422ee 100644 --- a/src/fava/help/options.md +++ b/src/fava/help/options.md @@ -8,7 +8,9 @@ following to your Beancount file. 2016-04-14 custom "fava-option" "auto-reload" "true" 2016-04-14 custom "fava-option" "currency-column" "100" -Below is a list of all possible options for Fava. +Below is a list of all possible options for Fava. In the Beancount documentation +you can find the +[list of Beancount options](https://beancount.github.io/docs/beancount_options_reference.html) ______________________________________________________________________ From 999f94b7b0e2ef1f402cb20ff5330a98054a645d Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Tue, 16 Dec 2025 12:39:54 +0100 Subject: [PATCH 02/16] Help: Add link to unit-tested example import configuration --- src/fava/application.py | 4 + src/fava/help/import.md | 137 ++------------ .../test_json_api-test_api_imports.json | 10 ++ tests/data/import_example_for_docs.py | 168 ++++++++++++++++++ tests/data/import_for_docs.beancount | 9 + tests/test_import_documentation_example.py | 64 +++++++ 6 files changed, 273 insertions(+), 119 deletions(-) create mode 100644 tests/data/import_example_for_docs.py create mode 100644 tests/data/import_for_docs.beancount create mode 100644 tests/test_import_documentation_example.py diff --git a/src/fava/application.py b/src/fava/application.py index bb54e30f4..b219c1d4c 100644 --- a/src/fava/application.py +++ b/src/fava/application.py @@ -14,6 +14,7 @@ import logging import mimetypes +import re from datetime import date from datetime import datetime from datetime import timezone @@ -430,6 +431,9 @@ def help_page(page_slug: str) -> str: html, beancount_version=beancount_version, fava_version=fava_version, + fava_version_nodirty=re.sub( + r"\.d[0-9]+$", "", fava_version + ), ), ), HELP_PAGES=HELP_PAGES, diff --git a/src/fava/help/import.md b/src/fava/help/import.md index cd38fcdf4..93e8f1b0d 100644 --- a/src/fava/help/import.md +++ b/src/fava/help/import.md @@ -1,7 +1,7 @@ -# Importing transactions into Fava +# Importing entries into Fava Fava integrates with Beancount's import system to automatically generate -transaction entries for your Beancount ledger from account statements. After +transaction entries for your Beancount file from account statements. After setting up your import process as explained below, the *Import* page in Fava will show an upload button that lets you upload files into your *import folder*. If you can access the import folder (e.g., if Fava is running on your local @@ -16,18 +16,22 @@ Importer appear as "Non-importable file". You can move any file into your [documents folder](https://beancount.github.io/docs/beancount_language_syntax.html#documents-from-a-directory) by clicking the button "Move". If you do not use the document folder feature of -beancount, you can delete the file from the import folder after the import is +Beancount, you can delete the file from the import folder after the import is complete by clicking the button with the cross at the top right. -## Define the upload configuration in your Beancount ledger +## Define the import configuration in your Beancount ledger Add these lines to your Beancount file:
The first line specifies the import folder location. The second line defines the @@ -40,17 +44,20 @@ ledger file location. You can also use absolute paths. Your import configuration must be a Python file that defines: - `CONFIG`: A list of Importers that process your account statement files. - YCreate subclasses of `beangulp.importers.Importer` with parsing logic for - your specific account statement formats, then add instances of each class to - this list. + Create subclasses of `beangulp.importers.Importer` with parsing logic for your + specific account statement formats, then add instances of each class to this + list. - `HOOKS`: A list of functions to apply to all directives (e.g., transactions) after generation by any Importer. See the example below and the help document [Importing External Data in Beancount](http://furius.ca/beancount/doc/ingest) -for more information on how to write importers. Hook functions are explained in -more detail further down on this page. +for more information on how to write importers. Also, you can find an example of +an import configuration on the \[Fava GitHub +page\](https://github.com/beancount/fava/tree/{{ fava_version_nodirty +}}/tests/data/import_example_for_docs.py). Hook functions are explained in more +detail further down on this page. Fava currently only supports entries of type `Transaction`, `Balance`, and `Note`. Set the special metadata key `__source__` to display the corresponding @@ -58,114 +65,6 @@ text (CSV-row, XML-fragment, etc.) for the entry in the list of entries to import. Note that this metadata (and all other metadata keys starting with an underscore) will be stripped before saving the entry to file. -# Example import configuration - -This is an example configuration to read in account statements provided as CSV: - -```python -import csv -import re - -import beangulp -import beangulp.importers.csvbase as csvbase -from beancount.core import data # Transaction, Posting, ... - -class MyCSVImporter(csvbase.Importer): - """Read a CSV file formatted in German style to demonstrate some formatting - options: Column separator = ";", decimals are like this: 2.032,43 (dot as - thousands separator, comma as decimal separator""" - - # The expected column names and formats in the input file are defined - # as member variables, that are instances of csvbase.Column or subclasses - # - # Required columns - date = csvbase.Date('Buchungstag', '%d.%m.%Y') # German date format - narration = csvbase.Column('Verwendungszweck') - # To parse amount, first remove dots, then translate commas to dots, - # to convert 2.032,43 -> 3032.43 - amount = csvbase.Amount('Betrag', subs={r'\.':'', r',': '.'}) - - # Optional further columns: - # flag, payee, account, currency, tag, link, balance. - - # Any additional members of type "Column" can be used by your own - # `finalize()` and `metadata()` functions, access e.g. as row.sepa_iban - sepa_iban = Column('IBAN') - - # The following variables set the CSV format (see csvbase.CSVReader): - # encoding = "utf8" # File encoding. - # header = 0 # Number of header lines to skip. - # footer = 0 # Number of footer lines to ignore. - # names = True # Whether the data file contains a row with column names. - # dialect = None # The CSV dialect used in the input file (str or csv.Delimiter object). - # comments = "#" # Comment character. - # order = None # Order of entries in the CSV file (Default: Infer from file) - # # can be csvbase.Order.ASCENDING or ...DESCENDING - - # Set CSV dialect to use semicolon as separator - dialect = csv.excel - dialect.delimiter = ';' - - def __init__(self): - super().__init__( - account="Assets:MyBank", # default if no account column is defined - currency="EUR", - flag = "*" # optional - ) - - def identify(self, filepath): - """Return True if this importer is suitable for the file of the given - name. This allows to auto-choose the right importer for a file.""" - return "MyBank" in filepath - - def read(self, filepath): - """Override the read method to preprocess the CSV file. - """ - # Add some pre-processing of the input file, then - # call the parent read method with the processed file - for row in super().read(filepath): - yield row - - def metadata(self, filepath, lineno, row): - """ - Called for each row of the input file to set the metadata of the - resulting transaction. - Arguments: - - filepath, lineno: The file path and line number - - row: Has attributes for each class member of type `Column` (or - subtypes) - """ - # Example: Add the values from the additional sepa_iban column as - # metadata - return data.new_metadata(filepath, lineno, {'iban': row.sepa_iban}) - - def finalize(self, txn, row): - """Called for each generated transaction `txn` to make user-defined - changes. The input data is available as attributes of `row`""" - # Example: Add a default second transaction leg to Expenses:Unknown - # Documentation of Transaction object (txt): - # https://beancount.github.io/docs/api_reference/beancount.core.html#beancount.core.data.Transaction - txn.postings.append(data.Posting( - account = "Expenses:Unknown", units = -txn.postings[0].units )) - - return txn - - -# All available importers, one for each file format you need to process -CONFIG = [ - MyCSVImporter() -] - -# Process beancount transaction objects after they have been extracted -HOOKS = [] - -# Allows to call this script as './import extract '. Not needed -# for Fava, but useful for debugging -if __name__ == "__main__": - ingest = beangulp.Ingest(CONFIG, HOOKS) - ingest() -``` - ## Hook functions Hook functions are applied to all imported transactions. A hook function diff --git a/tests/__snapshots__/test_json_api-test_api_imports.json b/tests/__snapshots__/test_json_api-test_api_imports.json index 376735d4c..1dedcc752 100644 --- a/tests/__snapshots__/test_json_api-test_api_imports.json +++ b/tests/__snapshots__/test_json_api-test_api_imports.json @@ -69,6 +69,16 @@ "importers": [], "name": "TEST_DATA_DIR/import_config.py" }, + { + "basename": "import_example_for_docs.py", + "importers": [], + "name": "TEST_DATA_DIR/import_example_for_docs.py" + }, + { + "basename": "import_for_docs.beancount", + "importers": [], + "name": "TEST_DATA_DIR/import_for_docs.beancount" + }, { "basename": "invalid-unicode.beancount", "importers": [], diff --git a/tests/data/import_example_for_docs.py b/tests/data/import_example_for_docs.py new file mode 100644 index 000000000..9a764e91e --- /dev/null +++ b/tests/data/import_example_for_docs.py @@ -0,0 +1,168 @@ +# ruff: noqa: ERA001, INP001, ARG002 +"""An example import configuration with explanations.""" + +from __future__ import annotations + +import csv +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import TYPE_CHECKING + +import beangulp # Importing tools +from beancount.core import data # Transaction, Posting, ... +from beangulp.importers import csvbase + +if TYPE_CHECKING: + import beancount + + Meta = beancount.core.data.Meta + Transaction = beancount.core.data.Transaction + Row = "Row" # a dynamically defined type based on NamedTuple + + +class MyCSVImporter(csvbase.Importer): + """Read the csv file called "import.csv", in this directory. + + This is a CSV file formatted in German style to demonstrate some formatting + options: Column separator = ";", decimals are like this: 2.032,43 (dot as + thousands separator, comma as decimal separator. + """ + + # The expected column names and formats in the input file are defined + # as member variables, that are instances of csvbase.Column or subclasses + # + # Required columns + date = csvbase.Date("Buchungsdatum", "%Y-%m-%d") + narration = csvbase.Column("Umsatztext") + # To parse amount, first remove dots, then translate commas to dots, + # to convert 2.032,43 -> 3032.43 + amount = csvbase.Amount("Betrag", subs={r"\.": "", r",": "."}) + + # Optional further columns: + # flag, payee, account, currency, tag, link, balance. + # Providing 'balance' will auto-generate a balance assertion after the + # imported entries. + balance = csvbase.Amount("Saldo", subs={r"\.": "", r",": "."}) + + # Any additional members of type "Column" can be used by your own + # `finalize()` and `metadata()` functions, access e.g. as row.sepa_iban + sepa_iban = csvbase.Column("IBAN") + + # The following variables set the CSV format (see csvbase.CSVReader): + # encoding = "utf8" # File encoding. + # skiplines = 0 # NOTE: Will be renamed to "header" in beangulp 0.3.0 + # names = True # Whether the input contains a row with column names. + # dialect = None # The CSV dialect used in the input file + # # (str or csv.Delimiter object). + # comments = "#" # Comment character. + # order = None # Order of entries in the CSV file + # # (Default: Infer from file) + order = csvbase.Order.DESCENDING + + # Set CSV dialect to use semicolon as separator + dialect = csv.excel + dialect.delimiter = ";" + + def __init__(self) -> None: + super().__init__( + account="Assets:MyBank", # default if no account column is defined + currency="EUR", + flag="*", # optional + ) + + def identify(self, filepath: str) -> bool: + """Return True if this importer is suitable for the given filename. + + This allows to auto-choose the right importer for a file. + + Arguments: + filepath: File path to read. + """ + return filepath.endswith("import.csv") + + # ruff: noqa: SIM115 + def read(self, filepath: str) -> Row: + """Override the read method to preprocess the CSV file. + + Arguments: + filepath: File path to read. + + Returns: + Values from one line of the input. Named tuple with attributess + named like class members of type Column. + """ + # Add some pre-processing of the input file, then + # call the parent read method with the processed file + + # Truncate the last line + try: + tmp = NamedTemporaryFile("w", delete=False) + with tmp: + lines = Path(filepath).read_text().splitlines() + lines = lines[:-1] + tmp.write("\n".join(lines)) + + yield from super().read(tmp.name) + + finally: + Path(tmp.name).unlink() + + def metadata(self, filepath: str, lineno: int, row: Row) -> Meta: + """Set the metadata of imported transactions. + + This is called for each row of the input file. + + Arguments: + filepath: Import file name + lineno: Import file line number + row: Values from one line of the input. Named tuple with + attributes named like class members of type Column. + + Returns: + Object as created by beancount.core.data.new_metadata() + """ + # Example: Add the values from the additional sepa_iban column as + # metadata + return data.new_metadata(filepath, lineno, {"iban": row.sepa_iban}) + # To set posting metadata, use the finalize() function. There you can + # access txn.postings[i].meta as a simple dictionary. + + def finalize(self, txn: Transaction, row: Row) -> Transaction: + """Called for each generated transaction to make user-defined changes. + + Arguments: + txn: beancount.data.core.Transaction. + row: Values from one line of the input. Named tuple with + attributes named like class members of type Column. + + Returns: + beancount.core.data.Transaction object. + """ + # Example: Add a default second transaction leg to Expenses:Unknown + # Documentation of Transaction object (txt): + # https://beancount.github.io/docs/api_reference/beancount.core.html#beancount.core.data.Transaction + txn.postings.append( + data.Posting( + "Expenses:Unknown", + -txn.postings[0].units, + None, + None, + None, + None, + ) + ) + + return txn + + +# All available importers, one for each file format you need to process +CONFIG = [MyCSVImporter()] + +# Process beancount transaction objects after they have been extracted +HOOKS = [] + +# Allows to call this script as './import extract '. Not needed +# for Fava, but useful for debugging +if __name__ == "__main__": + ingest = beangulp.Ingest(CONFIG, HOOKS) + ingest() diff --git a/tests/data/import_for_docs.beancount b/tests/data/import_for_docs.beancount new file mode 100644 index 000000000..af9c3ae9d --- /dev/null +++ b/tests/data/import_for_docs.beancount @@ -0,0 +1,9 @@ +option "title" "import" +2017-01-01 custom "fava-option" "import-config" "import_example_for_docs.py" +2017-01-01 custom "fava-option" "import-dirs" "." +option "documents" "./" +2017-01-01 open Assets:Checking +2017-01-01 open Expenses:Widgets + +2017-02-14 * "" " BANKOMAT 00000483 K2 UM 11:56" + Assets:Checking -100.00 EUR diff --git a/tests/test_import_documentation_example.py b/tests/test_import_documentation_example.py new file mode 100644 index 000000000..15f514a12 --- /dev/null +++ b/tests/test_import_documentation_example.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from runpy import run_path +from typing import TYPE_CHECKING + +import beangulp + +import fava +from fava.core import FavaLedger + +if TYPE_CHECKING: + from pathlib import Path + + +def test_example_import_cli(test_data_dir: Path) -> None: + """Test that the importer is successful, when run directly on the data""" + ledger = FavaLedger(test_data_dir / "import_for_docs.beancount") + mod = run_path(str(test_data_dir / "import_example_for_docs.py")) + importer = mod["MyCSVImporter"]() + file = test_data_dir / "import.csv" + extract = beangulp.extract.extract_from_file( + importer, file, ledger.all_entries + ) + + assert len(extract) == 3, ( + "Number lines read from import.csv is not correct. " + "Should be 2 transactions and 1 balance" + ) + assert [type(x).__name__ for x in extract] == [ + "Transaction", + "Transaction", + "Balance", + ] + + +def test_example_import(test_data_dir: Path) -> None: + ledger = FavaLedger(test_data_dir / "import_for_docs.beancount") + + # This tests needs multiple steps as the User clicks multiple buttons until + # the import starts + + ing = fava.core.IngestModule(ledger) + # Read import configuration as defined in the fava option "import-config" + ing.load_file() + # Identify (file, importer) pairs based on the files it sees in the import + # folder + x = ing.import_data() + + # Only one file matches the importer we have defined + x = [e for e in x if len(e.importers) != 0] + assert len(x) == 1 + x = x[0] + + new_entries = ing.extract(x.name, x.importers[0].importer_name) + + assert len(new_entries) == 3, ( + "Number lines read from import.csv is not correct. " + "Should be 2 transactions and 1 balance" + ) + assert [type(x).__name__ for x in new_entries] == [ + "Transaction", + "Transaction", + "Balance", + ] From e20d8f513b0fcc44b7a7916c12e5c3116062bf0b Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Tue, 16 Dec 2025 12:39:54 +0100 Subject: [PATCH 03/16] Help: Add minimal example hook function --- tests/data/import_example_for_docs.py | 34 ++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/tests/data/import_example_for_docs.py b/tests/data/import_example_for_docs.py index 9a764e91e..603eccd68 100644 --- a/tests/data/import_example_for_docs.py +++ b/tests/data/import_example_for_docs.py @@ -15,9 +15,11 @@ if TYPE_CHECKING: import beancount + Importer = beangulp.importer.Importer Meta = beancount.core.data.Meta Transaction = beancount.core.data.Transaction - Row = "Row" # a dynamically defined type based on NamedTuple + Directive = beancount.core.data.Directive + Row = "Row" # dynamically created NamedTuple, see docs of using functions class MyCSVImporter(csvbase.Importer): @@ -158,8 +160,34 @@ def finalize(self, txn: Transaction, row: Row) -> Transaction: # All available importers, one for each file format you need to process CONFIG = [MyCSVImporter()] -# Process beancount transaction objects after they have been extracted -HOOKS = [] + +# Hooks: Process beancount transaction objects after they have been extracted + + +# ruff: noqa: ARG001 +def example_hook( + new_entries: list[tuple[str, list[Directive], str, Importer]], + existing_entries: list[Directive], +) -> list[tuple[str, list[Directive], str, Importer]]: + """Example hook function. + + Arguments: + new_entries: New entries. One tuple per input file. Note that for Fava, + this list will always have only one tuple as the user starts the + import from a single file via the user interface. + existing_entries: List of existing entries, for example to do + deduplication + """ + out = [] + for filename, entries, account, importer in new_entries: + # ... Edit entries (list of Directives), then ... + out.append((filename, entries, account, importer)) + + return out + + +# List all hooks here +HOOKS = [example_hook] # Allows to call this script as './import extract '. Not needed # for Fava, but useful for debugging From 63c1edc6d63eac45b65e6e1ccb644f7016b81e19 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Tue, 16 Dec 2025 14:12:30 +0100 Subject: [PATCH 04/16] Fix linter errors (typing) --- tests/data/import_example_for_docs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/data/import_example_for_docs.py b/tests/data/import_example_for_docs.py index 603eccd68..07b2ac740 100644 --- a/tests/data/import_example_for_docs.py +++ b/tests/data/import_example_for_docs.py @@ -7,6 +7,7 @@ from pathlib import Path from tempfile import NamedTemporaryFile from typing import TYPE_CHECKING +from typing import TypeAlias import beangulp # Importing tools from beancount.core import data # Transaction, Posting, ... @@ -19,7 +20,8 @@ Meta = beancount.core.data.Meta Transaction = beancount.core.data.Transaction Directive = beancount.core.data.Directive - Row = "Row" # dynamically created NamedTuple, see docs of using functions + # dynamically created NamedTuple, see docs of using functions + Row: TypeAlias = "Row" class MyCSVImporter(csvbase.Importer): From febefaa065458a30c5aa43153ac2b47479cd0374 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Tue, 16 Dec 2025 14:24:09 +0100 Subject: [PATCH 05/16] Fix some mypy errors --- tests/data/import_example_for_docs.py | 9 +++++---- tests/test_import_documentation_example.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/data/import_example_for_docs.py b/tests/data/import_example_for_docs.py index 07b2ac740..b61ed8d45 100644 --- a/tests/data/import_example_for_docs.py +++ b/tests/data/import_example_for_docs.py @@ -10,16 +10,17 @@ from typing import TypeAlias import beangulp # Importing tools +import beangulp.importer from beancount.core import data # Transaction, Posting, ... from beangulp.importers import csvbase if TYPE_CHECKING: import beancount - Importer = beangulp.importer.Importer - Meta = beancount.core.data.Meta - Transaction = beancount.core.data.Transaction - Directive = beancount.core.data.Directive + Importer: TypeAlias = beangulp.importer.Importer + Meta: TypeAlias = beancount.core.data.Meta + Transaction: TypeAlias = beancount.core.data.Transaction + Directive: TypeAlias = beancount.core.data.Directive # dynamically created NamedTuple, see docs of using functions Row: TypeAlias = "Row" diff --git a/tests/test_import_documentation_example.py b/tests/test_import_documentation_example.py index 15f514a12..46d39b07a 100644 --- a/tests/test_import_documentation_example.py +++ b/tests/test_import_documentation_example.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING import beangulp +import beangulp.extract import fava from fava.core import FavaLedger @@ -14,7 +15,7 @@ def test_example_import_cli(test_data_dir: Path) -> None: """Test that the importer is successful, when run directly on the data""" - ledger = FavaLedger(test_data_dir / "import_for_docs.beancount") + ledger = FavaLedger(str(test_data_dir / "import_for_docs.beancount")) mod = run_path(str(test_data_dir / "import_example_for_docs.py")) importer = mod["MyCSVImporter"]() file = test_data_dir / "import.csv" @@ -34,7 +35,7 @@ def test_example_import_cli(test_data_dir: Path) -> None: def test_example_import(test_data_dir: Path) -> None: - ledger = FavaLedger(test_data_dir / "import_for_docs.beancount") + ledger = FavaLedger(str(test_data_dir / "import_for_docs.beancount")) # This tests needs multiple steps as the User clicks multiple buttons until # the import starts @@ -49,9 +50,9 @@ def test_example_import(test_data_dir: Path) -> None: # Only one file matches the importer we have defined x = [e for e in x if len(e.importers) != 0] assert len(x) == 1 - x = x[0] + x0 = x[0] - new_entries = ing.extract(x.name, x.importers[0].importer_name) + new_entries = ing.extract(x0.name, x0.importers[0].importer_name) assert len(new_entries) == 3, ( "Number lines read from import.csv is not correct. " From ecb3db74979ee11293b1888a7ae6d2a688f34d23 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Tue, 16 Dec 2025 14:33:30 +0100 Subject: [PATCH 06/16] Prevent linter from line-splitting jinja2 syntax --- .pre-commit-config.yaml | 2 +- pyproject.toml | 5 +++++ src/fava/help/import.md | 7 +++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b0d910fa3..9a54c49e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: rev: 1.0.0 hooks: - id: mdformat - args: ["--wrap", "80"] + args: ["--wrap", "80", "--exclude", "src/fava/help/import.md"] additional_dependencies: - mdformat-gfm - repo: https://github.com/biomejs/pre-commit diff --git a/pyproject.toml b/pyproject.toml index fb1a3e463..8076edbb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -183,6 +183,11 @@ source = [ [tool.ruff] line-length = 79 +[tool.ruff.format] +exclude = [ + "src/fava/help/import.md", +] + [tool.ruff.lint] extend-select = [ "ALL", diff --git a/src/fava/help/import.md b/src/fava/help/import.md index 93e8f1b0d..fd9df7ea7 100644 --- a/src/fava/help/import.md +++ b/src/fava/help/import.md @@ -54,10 +54,9 @@ Your import configuration must be a Python file that defines: See the example below and the help document [Importing External Data in Beancount](http://furius.ca/beancount/doc/ingest) for more information on how to write importers. Also, you can find an example of -an import configuration on the \[Fava GitHub -page\](https://github.com/beancount/fava/tree/{{ fava_version_nodirty -}}/tests/data/import_example_for_docs.py). Hook functions are explained in more -detail further down on this page. +an import configuration on the +[Fava GitHub page](https://github.com/beancount/fava/tree/%7B%7Bfava_version_nodirty%7D%7D/tests/data/import_example_for_docs.py). +Hook functions are explained in more detail further down on this page. Fava currently only supports entries of type `Transaction`, `Balance`, and `Note`. Set the special metadata key `__source__` to display the corresponding From b02f75a333a2d03bcfcb754187444621a8eff2fb Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Tue, 16 Dec 2025 14:39:30 +0100 Subject: [PATCH 07/16] Make the manual and the example more conspicuous in the help text --- src/fava/help/import.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/fava/help/import.md b/src/fava/help/import.md index fd9df7ea7..e62c5d939 100644 --- a/src/fava/help/import.md +++ b/src/fava/help/import.md @@ -51,11 +51,13 @@ Your import configuration must be a Python file that defines: - `HOOKS`: A list of functions to apply to all directives (e.g., transactions) after generation by any Importer. -See the example below and the help document -[Importing External Data in Beancount](http://furius.ca/beancount/doc/ingest) -for more information on how to write importers. Also, you can find an example of -an import configuration on the -[Fava GitHub page](https://github.com/beancount/fava/tree/%7B%7Bfava_version_nodirty%7D%7D/tests/data/import_example_for_docs.py). +For more information on how to write importers, see + +- [Importing External Data in Beancount](http://furius.ca/beancount/doc/ingest) + in the beancount manual. +- The + [Example input configuration file](https://github.com/beancount/fava/tree/{{ fava_version_nodirty }}/tests/data/import_example_for_docs.py). + Hook functions are explained in more detail further down on this page. Fava currently only supports entries of type `Transaction`, `Balance`, and From dbc8251002f13855bf4a2d9c75322fab452b3a79 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Tue, 16 Dec 2025 15:09:34 +0100 Subject: [PATCH 08/16] Make sure setuptools_scm version output is converted to something GitHub accepts as URL --- src/fava/application.py | 15 ++++++++++++--- src/fava/help/import.md | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/fava/application.py b/src/fava/application.py index b219c1d4c..72c1b7767 100644 --- a/src/fava/application.py +++ b/src/fava/application.py @@ -423,6 +423,17 @@ def help_page(page_slug: str) -> str: contents, extras=["fenced-code-blocks", "tables", "header-ids"], ) + # Convert git describe output into something to put in a GitHub URL + # remove dirty suffix + github_version = re.sub(r"\.g[0-9]+$", "", fava_version) + commit_hash = re.search(r"\+g([0-9a-f]+)", fava_version) + tag_name = re.search(r"^v[0-9]+(\.[0-9+]*)", fava_version) + if commit_hash: + github_version = commit_hash.group(1) + elif tag_name: + github_version = tag_name.group(1) + else: + github_version = "main" # fallback to main branch return render_template( "help.html", page_slug=page_slug, @@ -431,9 +442,7 @@ def help_page(page_slug: str) -> str: html, beancount_version=beancount_version, fava_version=fava_version, - fava_version_nodirty=re.sub( - r"\.d[0-9]+$", "", fava_version - ), + github_version=github_version, ), ), HELP_PAGES=HELP_PAGES, diff --git a/src/fava/help/import.md b/src/fava/help/import.md index e62c5d939..ad230562d 100644 --- a/src/fava/help/import.md +++ b/src/fava/help/import.md @@ -56,7 +56,7 @@ For more information on how to write importers, see - [Importing External Data in Beancount](http://furius.ca/beancount/doc/ingest) in the beancount manual. - The - [Example input configuration file](https://github.com/beancount/fava/tree/{{ fava_version_nodirty }}/tests/data/import_example_for_docs.py). + [Example input configuration file](https://github.com/beancount/fava/tree/{{ github_version }}/tests/data/import_example_for_docs.py). Hook functions are explained in more detail further down on this page. From 300c88cdddb8626b957aaff1dba91afe868dbc9c Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Tue, 16 Dec 2025 15:14:02 +0100 Subject: [PATCH 09/16] Remove typo --- src/fava/help/import.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fava/help/import.md b/src/fava/help/import.md index ad230562d..c4549d99a 100644 --- a/src/fava/help/import.md +++ b/src/fava/help/import.md @@ -73,7 +73,7 @@ receives the parameters `hook_fn(new_entries_list, existing_entries)` and returns the modified `new_entries_list`. The argument `new_entries_list` is itself a list of tuples. As Fava imports each -file individually (in in contrast to the CLI of Beangulp), this list will always +file individually (in contrast to the CLI of Beangulp), this list will always be of length 1. The tuples can can have two (old style, default) or four elements From 6ca35f2c2c2c37e50940822562d726100c799244 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Tue, 16 Dec 2025 15:28:49 +0100 Subject: [PATCH 10/16] help: Mention csvbase.Importer --- src/fava/help/import.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fava/help/import.md b/src/fava/help/import.md index c4549d99a..a482d13c5 100644 --- a/src/fava/help/import.md +++ b/src/fava/help/import.md @@ -46,7 +46,8 @@ Your import configuration must be a Python file that defines: - `CONFIG`: A list of Importers that process your account statement files. Create subclasses of `beangulp.importers.Importer` with parsing logic for your specific account statement formats, then add instances of each class to this - list. + list. For CSV files, subclassing `beangulp.importers.csvbase.Importer` is + recommended. - `HOOKS`: A list of functions to apply to all directives (e.g., transactions) after generation by any Importer. From a59c902581a8879cfe76ecb396a5dddee3b7ad5c Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Thu, 18 Dec 2025 11:29:43 +0100 Subject: [PATCH 11/16] Add type stubs for example import configuration --- stubs/beangulp/__init__.pyi | 21 ++++++ stubs/beangulp/importers/__init__.pyi | 1 + stubs/beangulp/importers/csvbase.pyi | 75 ++++++++++++++++++++++ tests/data/import_example_for_docs.py | 12 ++-- tests/importer.pyi | 28 ++++++++ tests/test_import_documentation_example.py | 4 +- 6 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 stubs/beangulp/importers/__init__.pyi create mode 100644 stubs/beangulp/importers/csvbase.pyi create mode 100644 tests/importer.pyi diff --git a/stubs/beangulp/__init__.pyi b/stubs/beangulp/__init__.pyi index 639fc09a8..0b9f2ff87 100644 --- a/stubs/beangulp/__init__.pyi +++ b/stubs/beangulp/__init__.pyi @@ -1,6 +1,7 @@ import datetime from abc import ABC from abc import abstractmethod +from collections.abc import Callable from collections.abc import Sequence from fava.beans.abc import Account @@ -24,3 +25,23 @@ class Importer(ABC): def sort( self, entries: list[Directive], reverse: bool = False ) -> None: ... + +class Ingest: + def __init__( + self, + importer: Sequence[Importer], + hooks: Sequence[ + Callable[ + [ + Sequence[ + tuple[str, Sequence[Directive], str, Importer] + ], # new_entries + Sequence[Directive], # existing_entries + ], + Sequence[ + tuple[str, Sequence[Directive], str, Importer] + ], # (return value) + ] + ], + ) -> None: ... + def __call__(self) -> None: ... diff --git a/stubs/beangulp/importers/__init__.pyi b/stubs/beangulp/importers/__init__.pyi new file mode 100644 index 000000000..a87b6224f --- /dev/null +++ b/stubs/beangulp/importers/__init__.pyi @@ -0,0 +1 @@ +# This file is intentionally left empty to make the directory a package diff --git a/stubs/beangulp/importers/csvbase.pyi b/stubs/beangulp/importers/csvbase.pyi new file mode 100644 index 000000000..58f95e5bc --- /dev/null +++ b/stubs/beangulp/importers/csvbase.pyi @@ -0,0 +1,75 @@ +import csv +import datetime +import enum +from collections.abc import Sequence +from typing import Any +from typing import NamedTuple +from typing import TypeAlias + +import beancount.core.amount +import beangulp +from beancount.core import data + +from fava.beans.abc import Directive + +Row: TypeAlias = NamedTuple + +class Order(enum.Enum): + ASCENDING = ... + DESCENDING = ... + +class Column: + name: str + def __init__(self, name: str) -> None: ... + def parse(self, value: str) -> Any: ... + +class Date(Column): + format: str + def __init__(self, name: str, frmt: str) -> None: ... + def parse(self, value: str) -> datetime.date: ... + +class Amount(Column): + subs: dict[str, str] + def __init__( + self, name: str, subs: dict[str, str] | None = None + ) -> None: ... + def parse(self, value: str) -> beancount.core.amount.Amount: ... + +class Columns(Column): + columns: list[str] + sep: str + def __init__(self, *columns: str, sep: str = " ") -> None: ... + def parse(self, value: str) -> str: ... + +class CreditOrDebit(Column): + credit_name: str + debit_name: str + def __init__(self, credit_name: str, debit_name: str) -> None: ... + def parse(self, value: str) -> beancount.core.amount.Amount: ... + +class CSVMeta(type): ... + +class CSVReader: + encoding: str + skiplines: int + names: bool + dialect: str | csv.Dialect + comments: str + order: Order | None + + def read(self, filepath: str) -> list[Row]: ... + +class Importer(beangulp.Importer, CSVReader): + def __init__( + self, account: str, currency: str, flag: str = "*" + ) -> None: ... + def account(self, filepath: str) -> str: ... + def date(self, filepath: str) -> datetime.date: ... + def extract( + self, filepath: str, existing: Sequence[Directive] + ) -> list[Directive]: ... + def finalize( + self, txn: data.Transaction, row: Row + ) -> data.Transaction | None: ... + def identify(self, filepath: str) -> bool: ... + def metadata(self, filepath: str, lineno: int, row: Row) -> data.Meta: ... diff --git a/tests/data/import_example_for_docs.py b/tests/data/import_example_for_docs.py index b61ed8d45..24b56e43d 100644 --- a/tests/data/import_example_for_docs.py +++ b/tests/data/import_example_for_docs.py @@ -1,4 +1,5 @@ # ruff: noqa: ERA001, INP001, ARG002 +# mypy: disable-error-code="assignment" """An example import configuration with explanations.""" from __future__ import annotations @@ -6,23 +7,23 @@ import csv from pathlib import Path from tempfile import NamedTemporaryFile +from typing import Any from typing import TYPE_CHECKING from typing import TypeAlias import beangulp # Importing tools -import beangulp.importer from beancount.core import data # Transaction, Posting, ... from beangulp.importers import csvbase if TYPE_CHECKING: import beancount - Importer: TypeAlias = beangulp.importer.Importer + Importer: TypeAlias = beangulp.Importer Meta: TypeAlias = beancount.core.data.Meta Transaction: TypeAlias = beancount.core.data.Transaction Directive: TypeAlias = beancount.core.data.Directive # dynamically created NamedTuple, see docs of using functions - Row: TypeAlias = "Row" + Row: TypeAlias = Any class MyCSVImporter(csvbase.Importer): @@ -149,7 +150,8 @@ def finalize(self, txn: Transaction, row: Row) -> Transaction: txn.postings.append( data.Posting( "Expenses:Unknown", - -txn.postings[0].units, + # "and" handles case if .units is None + (txn.postings[0].units and -txn.postings[0].units), None, None, None, @@ -195,5 +197,5 @@ def example_hook( # Allows to call this script as './import extract '. Not needed # for Fava, but useful for debugging if __name__ == "__main__": - ingest = beangulp.Ingest(CONFIG, HOOKS) + ingest = beangulp.Ingest(CONFIG, HOOKS) # type: ignore[arg-type] ingest() diff --git a/tests/importer.pyi b/tests/importer.pyi new file mode 100644 index 000000000..1b52b18c0 --- /dev/null +++ b/tests/importer.pyi @@ -0,0 +1,28 @@ +import abc +import datetime +from collections.abc import Sequence + +from fava.beans.abc import Account +from fava.beans.abc import Directive + +class Importer(abc.ABC): + @property + def name(self) -> str: ... + @abc.abstractmethod + def identify(self, filepath: str) -> bool: ... + @abc.abstractmethod + def account(self, filepath: str) -> Account: ... + def date(self, filepath: str) -> datetime.date | None: ... + def filename(self, filepath: str) -> str | None: ... + def extract( + self, filepath: str, existing: Sequence[Directive] + ) -> list[Directive]: ... + def deduplicate( + self, entries: list[Directive], existing: Sequence[Directive] + ) -> None: ... + def sort( + self, entries: list[Directive], reverse: bool = False + ) -> None: ... + +class ImporterProtocol: ... +class Adapter: ... diff --git a/tests/test_import_documentation_example.py b/tests/test_import_documentation_example.py index 46d39b07a..c02747104 100644 --- a/tests/test_import_documentation_example.py +++ b/tests/test_import_documentation_example.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING import beangulp -import beangulp.extract +import beangulp.extract # type: ignore[import-untyped] import fava from fava.core import FavaLedger @@ -40,7 +40,7 @@ def test_example_import(test_data_dir: Path) -> None: # This tests needs multiple steps as the User clicks multiple buttons until # the import starts - ing = fava.core.IngestModule(ledger) + ing = fava.core.IngestModule(ledger) # type: ignore[attr-defined] # Read import configuration as defined in the fava option "import-config" ing.load_file() # Identify (file, importer) pairs based on the files it sees in the import From e00a59e9092f38966908cfd24f235451db9fbd2a Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Thu, 18 Dec 2025 11:55:51 +0100 Subject: [PATCH 12/16] Work around older mdformat version (no --exclude) --- .pre-commit-config.yaml | 2 +- src/fava/application.py | 3 ++- src/fava/help/import.md | 9 ++++----- tests/test_import_documentation_example.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a54c49e3..b0d910fa3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: rev: 1.0.0 hooks: - id: mdformat - args: ["--wrap", "80", "--exclude", "src/fava/help/import.md"] + args: ["--wrap", "80"] additional_dependencies: - mdformat-gfm - repo: https://github.com/biomejs/pre-commit diff --git a/src/fava/application.py b/src/fava/application.py index 72c1b7767..b8a780989 100644 --- a/src/fava/application.py +++ b/src/fava/application.py @@ -434,6 +434,7 @@ def help_page(page_slug: str) -> str: github_version = tag_name.group(1) else: github_version = "main" # fallback to main branch + github_url = f"https://github.com/beancount/fava/tree/{github_version}/tests/data/import_example_for_docs.py" return render_template( "help.html", page_slug=page_slug, @@ -442,7 +443,7 @@ def help_page(page_slug: str) -> str: html, beancount_version=beancount_version, fava_version=fava_version, - github_version=github_version, + github_url=github_url, ), ), HELP_PAGES=HELP_PAGES, diff --git a/src/fava/help/import.md b/src/fava/help/import.md index a482d13c5..a5a37b09a 100644 --- a/src/fava/help/import.md +++ b/src/fava/help/import.md @@ -46,7 +46,7 @@ Your import configuration must be a Python file that defines: - `CONFIG`: A list of Importers that process your account statement files. Create subclasses of `beangulp.importers.Importer` with parsing logic for your specific account statement formats, then add instances of each class to this - list. For CSV files, subclassing `beangulp.importers.csvbase.Importer` is + list. For CSV files, subclassing `beangulp.importers.csvbase.Importer` is recommended. - `HOOKS`: A list of functions to apply to all directives (e.g., transactions) @@ -56,8 +56,7 @@ For more information on how to write importers, see - [Importing External Data in Beancount](http://furius.ca/beancount/doc/ingest) in the beancount manual. -- The - [Example input configuration file](https://github.com/beancount/fava/tree/{{ github_version }}/tests/data/import_example_for_docs.py). +- The Example input configuration file Hook functions are explained in more detail further down on this page. @@ -74,8 +73,8 @@ receives the parameters `hook_fn(new_entries_list, existing_entries)` and returns the modified `new_entries_list`. The argument `new_entries_list` is itself a list of tuples. As Fava imports each -file individually (in contrast to the CLI of Beangulp), this list will always -be of length 1. +file individually (in contrast to the CLI of Beangulp), this list will always be +of length 1. The tuples can can have two (old style, default) or four elements (beangulp-style). The type signature of the tuples is either diff --git a/tests/test_import_documentation_example.py b/tests/test_import_documentation_example.py index c02747104..320f6ed4d 100644 --- a/tests/test_import_documentation_example.py +++ b/tests/test_import_documentation_example.py @@ -40,7 +40,7 @@ def test_example_import(test_data_dir: Path) -> None: # This tests needs multiple steps as the User clicks multiple buttons until # the import starts - ing = fava.core.IngestModule(ledger) # type: ignore[attr-defined] + ing = fava.core.IngestModule(ledger) # xxtype: ignore[attr-defined] # Read import configuration as defined in the fava option "import-config" ing.load_file() # Identify (file, importer) pairs based on the files it sees in the import From a2aaca50901b2cc249e73fc557db57254ca37e56 Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Thu, 18 Dec 2025 12:02:44 +0100 Subject: [PATCH 13/16] Clarify comment in example hook --- tests/data/import_example_for_docs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/data/import_example_for_docs.py b/tests/data/import_example_for_docs.py index 24b56e43d..9cfddd5c7 100644 --- a/tests/data/import_example_for_docs.py +++ b/tests/data/import_example_for_docs.py @@ -184,8 +184,10 @@ def example_hook( deduplication """ out = [] + # For each imported file: for filename, entries, account, importer in new_entries: - # ... Edit entries (list of Directives), then ... + # ... Edit entries (note that this is itself a list of Directives!), + # then ... out.append((filename, entries, account, importer)) return out From f1d109a24cdbb6e082370a8a45b802bdc0cf40cc Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Thu, 18 Dec 2025 12:06:25 +0100 Subject: [PATCH 14/16] Fix typo --- tests/test_import_documentation_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_import_documentation_example.py b/tests/test_import_documentation_example.py index 320f6ed4d..c02747104 100644 --- a/tests/test_import_documentation_example.py +++ b/tests/test_import_documentation_example.py @@ -40,7 +40,7 @@ def test_example_import(test_data_dir: Path) -> None: # This tests needs multiple steps as the User clicks multiple buttons until # the import starts - ing = fava.core.IngestModule(ledger) # xxtype: ignore[attr-defined] + ing = fava.core.IngestModule(ledger) # type: ignore[attr-defined] # Read import configuration as defined in the fava option "import-config" ing.load_file() # Identify (file, importer) pairs based on the files it sees in the import From 52b724cf168c93758d48bc93129757e04f02690a Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Thu, 18 Dec 2025 12:51:32 +0100 Subject: [PATCH 15/16] Unit-test GitHub URL generation --- src/fava/application.py | 35 ++++++++++++++++++++------- tests/test_application.py | 51 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/src/fava/application.py b/src/fava/application.py index b8a780989..ff6b142d5 100644 --- a/src/fava/application.py +++ b/src/fava/application.py @@ -425,15 +425,7 @@ def help_page(page_slug: str) -> str: ) # Convert git describe output into something to put in a GitHub URL # remove dirty suffix - github_version = re.sub(r"\.g[0-9]+$", "", fava_version) - commit_hash = re.search(r"\+g([0-9a-f]+)", fava_version) - tag_name = re.search(r"^v[0-9]+(\.[0-9+]*)", fava_version) - if commit_hash: - github_version = commit_hash.group(1) - elif tag_name: - github_version = tag_name.group(1) - else: - github_version = "main" # fallback to main branch + github_version = _get_github_version(fava_version) github_url = f"https://github.com/beancount/fava/tree/{github_version}/tests/data/import_example_for_docs.py" return render_template( "help.html", @@ -485,6 +477,31 @@ def _get_locale() -> str | None: Babel(fava_app, locale_selector=_get_locale) # type: ignore[no-untyped-call] +def _get_github_version(version_string: str) -> str: + """Convert a version string into a GitHub-compatible URL segment. + + Args: + version_string: The version string, from setuptools_scm + + Returns: + A string representing the commit hash, tag name, or 'main' as a + fallback. + """ + # Remove the dirty suffix if present + cleaned_version = re.sub(r"\.g[0-9]+$", "", version_string) + cleaned_version = re.sub(r"^v", "", version_string) + + # Extract commit hash or tag name + commit_hash = re.search(r"\+g([0-9a-f]+)", cleaned_version) + tag_name = re.search(r"^[0-9]+(\.[0-9]+)*", cleaned_version) + + if commit_hash: + return commit_hash.group(1) + if tag_name: + return tag_name.group(0) + return "main" # Fallback to main branch + + def create_app( files: Iterable[Path | str], *, diff --git a/tests/test_application.py b/tests/test_application.py index b7d4c18da..c4ec4b02d 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -11,6 +11,7 @@ from beancount import __version__ as beancount_version from fava import __version__ as fava_version +from fava.application import _get_github_version from fava.application import create_app from fava.application import static_url from fava.beans import create @@ -342,3 +343,53 @@ def test_load_extension_endpoint(test_client: FlaskClient) -> None: response = test_client.get(url) assert assert_success(response) assert response.json == ["some data"] + + +def test_get_github_version_with_commit_hash() -> None: + """Test that commit hash is correctly extracted from version string.""" + version_string = "v1.0.0.post3.dev10+g1234567" + assert _get_github_version(version_string) == "1234567" + + version_string = "1.0.0.dev10+g1234567" + assert _get_github_version(version_string) == "1234567" + + +def test_get_github_version_with_tag() -> None: + """Test that tag name is correctly extracted from version string.""" + version_string = "v1.0.0" + assert _get_github_version(version_string) == "1.0.0" + + +def test_get_github_version_with_dirty_suffix() -> None: + """Test that dirty suffix is removed from version string.""" + version_string = "v1.0.0.dev10+g1234567.d20240101" + assert _get_github_version(version_string) == "1234567" + + version_string = "v1.0.0+d20240101" + assert _get_github_version(version_string) == "1.0.0" + + +def test_get_github_version_fallback_to_main() -> None: + """Test that fallback to 'main' works for invalid version strings.""" + version_string = "invalid_version" + assert _get_github_version(version_string) == "main" + + +def test_get_github_version_empty_string() -> None: + """Test that empty string falls back to 'main'.""" + version_string = "" + assert _get_github_version(version_string) == "main" + + +def test_get_github_version_no_prefix() -> None: + """Test version string without 'v' prefix.""" + version_string = "1.0.0.dev10+g1234567" + assert _get_github_version(version_string) == "1234567" + + version_string = "1.0.0.dev10.d20250101" + assert _get_github_version(version_string) == "1.0.0" + + +def test_get_github_version_current_version() -> None: + v = _get_github_version(fava_version) + assert v != "main" From 991959942c6a6870045fc402f0c5d27cf74ce42e Mon Sep 17 00:00:00 2001 From: Moritz Lell Date: Thu, 18 Dec 2025 14:30:23 +0100 Subject: [PATCH 16/16] Skip testing example import configuration for old beangulp version --- tests/test_import_documentation_example.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_import_documentation_example.py b/tests/test_import_documentation_example.py index c02747104..95493ee2d 100644 --- a/tests/test_import_documentation_example.py +++ b/tests/test_import_documentation_example.py @@ -1,10 +1,12 @@ from __future__ import annotations +import importlib.metadata from runpy import run_path from typing import TYPE_CHECKING import beangulp import beangulp.extract # type: ignore[import-untyped] +import pytest import fava from fava.core import FavaLedger @@ -12,7 +14,13 @@ if TYPE_CHECKING: from pathlib import Path +beangulp_version = importlib.metadata.version("beangulp") + +@pytest.mark.skipif( + beangulp_version < "0.2.0", + reason="Documentation example requires beangulp 0.2.0", +) def test_example_import_cli(test_data_dir: Path) -> None: """Test that the importer is successful, when run directly on the data""" ledger = FavaLedger(str(test_data_dir / "import_for_docs.beancount")) @@ -34,6 +42,10 @@ def test_example_import_cli(test_data_dir: Path) -> None: ] +@pytest.mark.skipif( + beangulp_version < "0.2.0", + reason="Documentation example requires beangulp 0.2.0", +) def test_example_import(test_data_dir: Path) -> None: ledger = FavaLedger(str(test_data_dir / "import_for_docs.beancount"))