From a1c2a999e449054d6641bbb633954e45fcd63f90 Mon Sep 17 00:00:00 2001 From: Kristóf Marussy Date: Mon, 25 Jan 2021 01:14:28 +0100 Subject: Add plugins and importers from private config The importers are missing tests, because not having any specifications for the import formats means we must use real, private data as test inputs --- .../importers/transferwise/__init__.py | 0 .../importers/transferwise/__main__.py | 11 + .../importers/transferwise/client.py | 236 +++++++++++++ .../importers/transferwise/transferwise_json.py | 377 +++++++++++++++++++++ 4 files changed, 624 insertions(+) create mode 100644 beancount_extras_kris7t/importers/transferwise/__init__.py create mode 100644 beancount_extras_kris7t/importers/transferwise/__main__.py create mode 100644 beancount_extras_kris7t/importers/transferwise/client.py create mode 100644 beancount_extras_kris7t/importers/transferwise/transferwise_json.py (limited to 'beancount_extras_kris7t/importers/transferwise') diff --git a/beancount_extras_kris7t/importers/transferwise/__init__.py b/beancount_extras_kris7t/importers/transferwise/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/beancount_extras_kris7t/importers/transferwise/__main__.py b/beancount_extras_kris7t/importers/transferwise/__main__.py new file mode 100644 index 0000000..e25580d --- /dev/null +++ b/beancount_extras_kris7t/importers/transferwise/__main__.py @@ -0,0 +1,11 @@ +''' +Importer for Transferwise API transaction history. +''' +__copyright__ = 'Copyright (c) 2020 Kristóf Marussy ' +__license__ = 'GNU GPLv2' + +from importers.transferwise.client import main + + +if __name__ == '__main__': + main() diff --git a/beancount_extras_kris7t/importers/transferwise/client.py b/beancount_extras_kris7t/importers/transferwise/client.py new file mode 100644 index 0000000..a4b6629 --- /dev/null +++ b/beancount_extras_kris7t/importers/transferwise/client.py @@ -0,0 +1,236 @@ +''' +Importer for Transferwise API transaction history from the command line. +''' +__copyright__ = 'Copyright (c) 2020 Kristóf Marussy ' +__license__ = 'GNU GPLv2' + +import datetime as dt +import logging +import os +from typing import Any, Dict, Optional, Tuple, Set + +import beancount +from beancount.core import data + +from beancount_extras_kris7t.importers.transferwise.transferwise_json import Accounts, \ + DATE_FORMAT, Importer + +LOG = logging.getLogger('importers.transferwise.client') + + +def _parse_date_arg(date_str: str) -> dt.date: + return dt.datetime.strptime(date_str, '%Y-%m-%d').date() + + +def _import_config(config_path: str) -> Importer: + import runpy + + config = runpy.run_path(config_path) + importer = config['TRANSFERWISE_CONFIG'] # type: ignore + if isinstance(importer, Importer): + LOG.info('Loaded configuration from %s', config_path) + return importer + else: + raise ValueError(f'Invalid configuration: {config_path}') + + +def _get_reference(transaction: data.Transaction) -> Optional[str]: + for link in transaction.links: + if link.startswith('transferwise_'): + return link[13:] + return None + + +def _get_last_transaction_date(ledger_path: str, skip_references: Set[str]) -> Optional[dt.date]: + from beancount.parser import parser + + LOG.info('Checking %s for already imported transactions', ledger_path) + entries, _, _ = parser.parse_file(ledger_path) + date: Optional[dt.date] = None + skip: Set[str] = set() + for entry in entries: + if isinstance(entry, data.Transaction): + reference = _get_reference(entry) + if not reference: + continue + if date is None or date < entry.date: + date = entry.date + skip.clear() + if date == entry.date: + skip.add(reference) + skip_references.update(skip) + return date + + +def _get_date_range(from_date: Optional[dt.date], + to_date: dt.date, + ledger_path: Optional[str]) -> Tuple[dt.date, dt.date, Set[str]]: + skip_references = set() + if not from_date and ledger_path: + from_date = _get_last_transaction_date(ledger_path, skip_references) + if not from_date: + from_date = to_date - dt.timedelta(days=365) + LOG.info('Fetching transactions from %s to %s', from_date, to_date) + return from_date, to_date, skip_references + + +def _get_secrets(importer: Importer, + api_key: Optional[str], + proxy_uri: Optional[str]) -> Tuple[str, Optional[str]]: + import urllib.parse + + if proxy_uri: + uri_parts = urllib.parse.urlsplit(proxy_uri) + else: + uri_parts = None + if api_key and (not uri_parts or not uri_parts.username or uri_parts.password): + return api_key, proxy_uri + + from contextlib import closing + + import secretstorage + + with closing(secretstorage.dbus_init()) as connection: + collection = secretstorage.get_default_collection(connection) + if not api_key: + items = collection.search_items({ + 'profile_id': str(importer.profile_id), + 'borderless_account_id': str(importer.borderless_account_id), + 'xdg:schema': 'com.marussy.beancount.importer.TransferwiseAPIKey', + }) + item = next(items, None) + if not item: + raise ValueError('No API key found in SecretService') + LOG.info('Found API key secret "%s" from SecretService', item.get_label()) + api_key = item.get_secret().decode('utf-8') + if uri_parts and uri_parts.username and not uri_parts.password: + host = uri_parts.hostname or uri_parts.netloc + items = collection.search_items({ + 'host': host, + 'port': str(uri_parts.port or 1080), + 'user': uri_parts.username, + 'xdg:schema': 'org.freedesktop.Secret.Generic', + }) + item = next(items, None) + if item: + LOG.info('Found proxy password secret "%s" from SecretService', item.get_label()) + password = urllib.parse.quote_from_bytes(item.get_secret()) + uri = f'{uri_parts.scheme}://{uri_parts.username}:{password}@{host}' + if uri_parts.port: + proxy_uri = f'{uri}:{uri_parts.port}' + else: + proxy_uri = uri + else: + LOG.info('No proxy password secret was found in SecretService') + assert api_key # Make pyright happy + return api_key, proxy_uri + + +def _fetch_statements(importer: Importer, + from_date: dt.date, + to_date: dt.date, + api_key: str, + proxy_uri: Optional[str]) -> Dict[str, Any]: + import json + + import requests + + now = dt.datetime.utcnow().time() + from_time_str = dt.datetime.combine(from_date, dt.datetime.min.time()).strftime(DATE_FORMAT) + to_time_str = dt.datetime.combine(to_date, now).strftime(DATE_FORMAT) + uri_prefix = f'https://api.transferwise.com/v3/profiles/{importer.profile_id}/' + \ + f'borderless-accounts/{importer.borderless_account_id}/statement.json' + \ + f'?intervalStart={from_time_str}&intervalEnd={to_time_str}&type=COMPACT¤cy=' + headers = { + 'User-Agent': f'Beancount {beancount.__version__} Transferwise importer {__copyright__}', + 'Authorization': f'Bearer {api_key}', + } + proxy_dict: Dict[str, str] = {} + if proxy_uri: + proxy_dict['https'] = proxy_uri + statements: Dict[str, Any] = {} + for currency in importer.currencies: + result = requests.get(uri_prefix + currency, headers=headers, proxies=proxy_dict) + if result.status_code != 200: + LOG.error( + 'Fetcing %s statement failed with HTTP status code %d: %s', + currency, + result.status_code, + result.text) + else: + try: + statement = json.loads(result.text) + except json.JSONDecodeError as exc: + LOG.error('Failed to decode %s statement', currency, exc_info=exc) + else: + statements[currency] = statement + LOG.info('Fetched %s statement', currency) + return statements + + +def _print_statements(from_date: dt.date, + to_date: dt.date, + entries: data.Entries) -> None: + from beancount.parser import printer + print(f'*** Transferwise from {from_date} to {to_date}', flush=True) + printer.print_entries(entries) + + +def _determine_path(dir: str, currency: str, to_date: dt.date) -> str: + date_str = to_date.strftime('%Y-%m-%d') + simple_path = os.path.join(dir, f'{date_str}.transferwise_{currency}.json') + if not os.path.exists(simple_path): + return simple_path + for i in range(2, 10): + path = os.path.join(dir, f'{date_str}.transferwise_{currency}_{i}.json') + if not os.path.exists(path): + return path + raise ValueError(f'Cannot find unused name for {simple_path}') + + +def _archive_statements(documents_path: str, + to_date: dt.date, + accounts: Accounts, + statements: Dict[str, Any]): + import json + + from beancount.core.account import sep + + for currency, statement in statements.items(): + dir = os.path.join(documents_path, *accounts.get_borderless_account(currency).split(sep)) + os.makedirs(dir, exist_ok=True) + path = _determine_path(dir, currency, to_date) + with open(path, 'w') as file: + json.dump(statement, file, indent=2) + LOG.info('Saved %s statement as %s', currency, path) + + +def main(): + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument('--verbose', '-v', action='store_true') + parser.add_argument('--from-date', '-f', required=False, type=_parse_date_arg) + parser.add_argument('--to-date', '-t', default=dt.date.today(), type=_parse_date_arg) + parser.add_argument('--api-key', '-k', required=False, type=str) + parser.add_argument('--proxy', '-p', required=False, type=str) + parser.add_argument('--archive', '-a', required=False, type=str) + parser.add_argument('config', nargs=1) + parser.add_argument('ledger', nargs='?') + args = parser.parse_args() + if args.verbose: + log_level = logging.INFO + else: + log_level = logging.WARN + logging.basicConfig(level=log_level) + importer = _import_config(args.config[0]) + from_date, to_date, skip = _get_date_range(args.from_date, args.to_date, args.ledger) + if skip: + LOG.info('Skipping %s', skip) + api_key, proxy_uri = _get_secrets(importer, args.api_key, args.proxy) + statements = _fetch_statements(importer, from_date, to_date, api_key, proxy_uri) + if args.archive: + _archive_statements(args.archive, to_date, importer.accounts, statements) + entries = importer.extract_objects(statements.values(), skip) + if entries: + _print_statements(from_date, to_date, entries) diff --git a/beancount_extras_kris7t/importers/transferwise/transferwise_json.py b/beancount_extras_kris7t/importers/transferwise/transferwise_json.py new file mode 100644 index 0000000..c42de90 --- /dev/null +++ b/beancount_extras_kris7t/importers/transferwise/transferwise_json.py @@ -0,0 +1,377 @@ +''' +Importer for Transferwise API transaction history. +''' +__copyright__ = 'Copyright (c) 2020 Kristóf Marussy ' +__license__ = 'GNU GPLv2' + +from collections import defaultdict +import datetime as dt +import json +import logging +import re +from os import path +from typing import Any, Callable, Dict, Iterable, List, NamedTuple, Optional, Set + +from beancount.core import account, data +import beancount.core.amount as am +from beancount.core.amount import Amount +from beancount.core.flags import FLAG_WARNING +from beancount.core.inventory import Inventory +from beancount.core.number import ZERO +from beancount.ingest.cache import _FileMemo as FileMemo +from beancount.ingest.importer import ImporterProtocol + +import beancount_extras_kris7t.importers.utils as utils +from beancount_extras_kris7t.importers.utils import COMMENT_META, InvalidEntry, PAYEE_META + +DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' +DATE_FORMAT_FRACTIONAL = '%Y-%m-%dT%H:%M:%S.%fZ' +CATEGORY_META = 'transferwise-category' +ENTRY_TYPE_META = 'transferwise-entry-type' +TRANSFERWISE_JSON_TAG = 'transferwise-json' +CD_DEBIT = 'DEBIT' +CD_CREDIT = 'CREDIT' +MONEY_ADDED_TYPE = 'MONEY_ADDED' +CARD_TYPE = 'CARD' +CARD_REGEX = re.compile(r'^Card transaction of.*issued by', re.IGNORECASE) + + +def _parse_date(date_str: str) -> dt.date: + # TODO Handle time zones accurately + try: + return utils.parse_date(date_str, DATE_FORMAT) + except InvalidEntry: + return utils.parse_date(date_str, DATE_FORMAT_FRACTIONAL) + + +def _parse_json_amount(data: Any, cd: Optional[str] = None) -> Amount: + # TODO Handle precision better + amount = Amount(round(utils.parse_number(data['value']), 2), data['currency']) + if cd == CD_DEBIT: + return -amount + else: + return amount + + +def _validate_cd(amount: Amount, cd: str) -> None: + if cd == CD_DEBIT: + if amount.number >= ZERO: + raise InvalidEntry(f'Invalid debit amount: {amount}') + elif cd == CD_CREDIT: + if amount.number <= ZERO: + raise InvalidEntry(f'Invalid credit amount: {amount}') + else: + raise InvalidEntry(f'Invalid credit/debit type: {cd}') + + +class Reference(NamedTuple): + type: str + reference_number: str + + +class _ConversionResult(NamedTuple): + converted_fraction: Amount + price: Optional[Amount] + fudge: Optional[Amount] + + +class _Conversion(NamedTuple): + native_amount: Amount + converted_amount: Optional[Amount] + + def get_fraction(self, + total_converted_amount: Amount, + assigned_amount: Amount, + diverted_fees: Inventory) -> _ConversionResult: + fees = diverted_fees.get_currency_units(self.native_amount.currency) + native_amount = am.add(self.native_amount, fees) + fraction = am.div(assigned_amount, total_converted_amount.number).number + if self.converted_amount: + converted_fraction = am.mul(self.converted_amount, fraction) + price = am.div(native_amount, self.converted_amount.number) + assert price.number is not None + price = price._replace(number=round(price.number, 4)) + # TODO Do we have to calculate the fudge here? + return _ConversionResult(converted_fraction, price, None) + else: + return _ConversionResult(am.mul(native_amount, fraction), None, None) + + +class Accounts(NamedTuple): + borderless_root_asset: str + fees_expense: str + + def get_borderless_account(self, currency: str) -> str: + return account.sep.join([self.borderless_root_asset, currency]) + + +class Row(utils.Row): + cd: str + _transacted_amount: Amount + date: dt.date + _conversions: List[_Conversion] + _inputs: Inventory + _fees: Inventory + divert_fees: Optional[str] + + def __init__(self, reference: Reference, transaction_list: List[Any]): + assert len(transaction_list) >= 1 + first_transaction = transaction_list[0] + details = first_transaction['details'] + entry_type = details.get('type', None) + merchant = details.get('merchant', None) + if merchant: + payee = merchant['name'] + else: + payee = None + comment = details['description'] + super().__init__('', 0, entry_type, payee, comment) + if entry_type: + self.meta[ENTRY_TYPE_META] = entry_type + category = details.get('category', None) + if category: + self.meta[CATEGORY_META] = category + if merchant: + if 'category' in merchant and merchant['category'] == category: + del merchant['category'] + not_null_elements = [str(value).strip() for _, value in merchant.items() if value] + self.meta[PAYEE_META] = '; '.join(not_null_elements) + self.meta[COMMENT_META] = comment + self.tags.add(TRANSFERWISE_JSON_TAG) + self.links.add(f'transferwise_{reference.reference_number}') + self.date = _parse_date(first_transaction['date']) + self.cd = reference.type + self.divert_fees = None + self._conversions = [] + self._inputs = Inventory() + self._fees = Inventory() + self._compute_transacted_amount(transaction_list) + for transaction in transaction_list: + self._add_json_transaction(transaction) + + def _compute_transacted_amount(self, transaction_list: List[Any]) -> None: + first_transaction = transaction_list[0] + details = first_transaction['details'] + transacted_json = details.get('amount', None) + if transacted_json: + self._transacted_amount = _parse_json_amount(transacted_json, self.cd) + else: + if len(transaction_list) != 1: + raise InvalidEntry('Cannot determine transaction amount') + if exchange_details := first_transaction.get('exchangeDetails', None): + self._transacted_amount = _parse_json_amount(exchange_details, self.cd) + else: + self._transacted_amount = _parse_json_amount(first_transaction['amount']) + _validate_cd(self._transacted_amount, self.cd) + + def _add_json_transaction(self, transaction: Any) -> None: + if exchange := transaction.get('exchangeDetails', None): + native = _parse_json_amount(exchange['fromAmount'], self.cd) + converted = _parse_json_amount(exchange['toAmount'], self.cd) + _validate_cd(converted, self.cd) + else: + native = _parse_json_amount(transaction['amount']) + converted = None + _validate_cd(native, self.cd) + if total_fees := transaction.get('totalFees', None): + fee = -_parse_json_amount(total_fees) + if fee.number > ZERO: + raise InvalidEntry(f'Invalid transaction fee: {fee}') + if fee.number != ZERO: + self._fees.add_amount(fee) + native_after_fees = am.sub(native, fee) + else: + native_after_fees = native + self._conversions.append(_Conversion(native_after_fees, converted)) + self._inputs.add_amount(native) + + @property + def transacted_amount(self) -> Amount: + return self._transacted_amount + + def _to_transaction(self, accounts: Accounts) -> data.Transaction: + postings = self._get_postings(accounts) + return data.Transaction(self.meta, self.date, self.flag, self.payee, self.comment, + self.tags, self.links, postings) + + def _get_postings(self, accounts: Accounts) -> List[data.Posting]: + postings: List[data.Posting] = [] + for units, cost in self._inputs: + assert cost is None + postings.append(data.Posting( + accounts.get_borderless_account(units.currency), units, None, None, None, None)) + if self.divert_fees: + for units, cost in self._fees: + assert cost is None + postings.append(data.Posting( + self.divert_fees, units, None, None, None, None)) + diverted_fees = self._fees + else: + diverted_fees = Inventory() + # Also add the "fudge" amounts to the fees generated by rounding currency conversions. + all_fees = Inventory() + all_fees.add_inventory(self._fees) + for acc, assigned_units in self.postings: + for conversion in self._conversions: + units, price, fudge = conversion.get_fraction( + self._transacted_amount, assigned_units, diverted_fees) + postings.append(data.Posting(acc, -units, None, price, None, None)) + if fudge: + all_fees.add_amount(-fudge) + for units, cost in all_fees: + assert cost is None + postings.append(data.Posting( + accounts.fees_expense, -units, None, None, None, None)) + return postings + + +Extractor = Callable[[Row], None] + + +def extract_card_transaction(payment_processors: Dict[str, Optional[str]] = {}) -> Extractor: + regexes = [(re.compile(f'^\\s*{key}\\s*\\*', re.IGNORECASE), value) + for key, value in payment_processors.items()] + + def do_extract(row: Row) -> None: + if row.entry_type == CARD_TYPE and CARD_REGEX.search(row.comment): + if row.cd == CD_DEBIT: + row.comment = '' + else: + row.comment = 'Refund' + # Most manually add posting for refunded fees. + row.flag = FLAG_WARNING + if row.payee: + for key, value in regexes: + if match := key.search(row.payee): + if value: + row.tags.add(value) + row.payee = row.payee[match.end():].strip() + return do_extract + + +def extract_add_money(add_money_asset: str, add_money_fees_asset: str) -> Extractor: + def do_extract(row: Row) -> None: + if row.entry_type == MONEY_ADDED_TYPE: + row.payee = 'Transferwise' + row.divert_fees = add_money_fees_asset + row.assign_to_account(add_money_asset) + return do_extract + + +class Importer(ImporterProtocol): + _log: logging.Logger + profile_id: int + borderless_account_id: int + currencies: List[str] + accounts: Accounts + _extractors: List[Extractor] + + def __init__(self, + profile_id: int, + borderless_account_id: int, + currencies: Iterable[str], + accounts: Accounts, + extractors: List[Extractor]): + self._log = logging.getLogger(type(self).__qualname__) + self.profile_id = profile_id + self.borderless_account_id = borderless_account_id + self.currencies = list(currencies) + self.accounts = accounts + self._extractors = extractors + + def _parse_file(self, file: FileMemo) -> Any: + def parse_json(path: str) -> Any: + with open(path, 'r') as json_file: + try: + return json.load(json_file) + except json.JSONDecodeError as exc: + self._log.info('Invalid JSON: %s', path, exc_info=exc) + return None + + return file.convert(parse_json) + + def identify(self, file: FileMemo) -> bool: + _, extension = path.splitext(file.name) + if extension.lower() != '.json': + return False + contents = self._parse_file(file) + try: + query = contents['query'] + return query['accountId'] == self.borderless_account_id and \ + query['currency'] in self.currencies + except (KeyError, TypeError): + return False + + def file_name(self, file: FileMemo) -> str: + return 'statement.json' + + def file_account(self, file: FileMemo) -> str: + contents = self._parse_file(file) + try: + currency = contents['query']['currency'] + except (KeyError, TypeError) as exc: + raise ValueError(f'Invalid account statement: {file.name}') from exc + if not isinstance(currency, str): + raise ValueError(f'Invalid account statement: {file.name}') + return self.accounts.get_borderless_account(currency) + + def file_date(self, file: FileMemo) -> dt.date: + contents = self._parse_file(file) + try: + date_str = contents['query']['intervalEnd'] + except (KeyError, TypeError) as exc: + raise ValueError(f'Invalid account statement: {file.name}') from exc + if not isinstance(date_str, str): + raise ValueError(f'Invalid account statement: {file.name}') + return _parse_date(date_str) + + def extract(self, file: FileMemo) -> data.Entries: + contents = self._parse_file(file) + if contents: + return self.extract_objects([contents]) + else: + return [] + + def extract_objects(self, + statements: Iterable[Any], + skip_references: Set[str] = set()) -> data.Entries: + transactions: Dict[Reference, List[Any]] = defaultdict(lambda: []) + for statement in statements: + for transaction in statement['transactions']: + reference_number = transaction['referenceNumber'] + if reference_number in skip_references: + continue + reference = Reference(transaction['type'], reference_number) + transactions[reference].append(transaction) + entries: data.Entries = [] + for reference, transaction_list in transactions.items(): + if not transaction_list: + continue + try: + row = Row(reference, transaction_list) + except (TypeError, KeyError, InvalidEntry) as exc: + self._log.warn('Invalid entry: %s', reference, exc_info=exc) + continue + try: + utils.run_row_extractors(row, self._extractors) + except InvalidEntry as exc: + self._log.warn('Invalid entry: %s', reference, exc_info=exc) + continue + entries.append(row._to_transaction(self.accounts)) + entries.sort(key=lambda entry: entry.date) + if entries: + self._extract_closing_balances(statements, entries) + return entries + + def _extract_closing_balances(self, + statements: Iterable[Any], + entries: data.Entries) -> None: + for statement in statements: + query = statement['query'] + end_date = _parse_date(query['intervalEnd']) + dt.timedelta(days=1) + currency = query['currency'] + balance = statement['endOfStatementBalance'] + amount = Amount(utils.parse_number(balance['value']), balance['currency']) + meta = data.new_metadata(f'', 0) + account = self.accounts.get_borderless_account(currency) + entries.append(data.Balance(meta, end_date, account, amount, None, None)) -- cgit v1.2.3-54-g00ecf