parser.py

#!/usr/bin/env python3
import argparse
import csv
import hashlib
import re
import subprocess
from dataclasses import dataclass
from decimal import Decimal
from pathlib import Path
from typing import Optional, List, Dict, Tuple

AMOUNT_RE = re.compile(r"\b\d{1,3}(?:\.\d{3})*,\d{2}\b")
DATE_RE = re.compile(r"\b\d{2}\.\d{2}\.\d{4}\b")
DATE_ONLY_RE = re.compile(r"^(?:\d{2}\.\d{2}\.\d{4}\s*)+$")
NUMBER_ONLY_RE = re.compile(r"^\d+(?:[.,]\d+)?\.?$")

def normalize_description(s: str) -> str:
    # Drop trailing page numbers aligned far to the right
    s = s.strip()
    s = re.sub(r"\s{3,}\d{1,2}(?:[.,]\d+)?$", "", s)
    return s.strip()
TX_START_RE = re.compile(r"^\s*(\d+)\.\s+([A-Z]{2}[0-9A-Z]{8,})", re.M)

# Keep purpose-code parsing conservative (avoid false positives like "VISA")
PURPOSE_CODES = ["INTX", "GOVT", "SALA", "OTHR", "CHRG", "TAXS", "PENS"]

def parse_eur(s: str) -> Decimal:
    # Croatian formatting: thousands '.' decimal ','
    s = s.strip().replace(" ", "")
    s = s.replace(".", "").replace(",", ".")
    return Decimal(s)

def make_tx_fingerprint(
    statement_account: Optional[str],
    execution_date: Optional[str],
    amount: Optional[Decimal],
    ref_banke: Optional[str],
    counterparty_account: Optional[str],
    description: Optional[str],
) -> str:
    # Stable, idempotent identifier for dedupe across imports
    parts = [
        statement_account or "",
        execution_date or "",
        str(amount) if amount is not None else "",
        ref_banke or "",
        counterparty_account or "",
        description or "",
    ]
    payload = "|".join(parts)
    return hashlib.sha256(payload.encode("utf-8")).hexdigest()

def extract_first(pattern: str, text: str) -> Optional[str]:
    m = re.search(pattern, text, flags=re.S)
    return m.group(1) if m else None

def normalize_iban(s: Optional[str]) -> Optional[str]:
    if not s:
        return None
    return re.sub(r"\s+", "", s)

def pdftotext_layout(pdf_path: Path) -> str:
    out = subprocess.check_output(["pdftotext", "-layout", str(pdf_path), "-"])
    return out.decode("utf-8", errors="replace")

def amount_column_indices(text: str) -> Tuple[int, int, float]:
    # Find the header row that contains Isplata and Uplata
    for line in text.splitlines():
        if "RBR." in line and "Isplata" in line and "Uplata" in line:
            ispl = line.index("Isplata")
            upl = line.index("Uplata")
            return ispl, upl, (ispl + upl) / 2.0
    raise ValueError("Could not locate Isplata/Uplata header line")

def header_totals(text: str) -> Dict[str, Optional[Decimal]]:
    return {
        "statement_date": extract_first(r"Datum izvatka:\s*(\d{2}\.\d{2}\.\d{4})", text),
        "statement_no": extract_first(r"Izvadak EUR br\.\:\s*(\d+)", text),
        "statement_account": normalize_iban(extract_first(r"Račun broj:\s*(HR[0-9\s]{10,})", text)),
        "prev_balance": (parse_eur(extract_first(r"Stanje prethodnog izvatka:\s*([\d\.]+,\d{2})", text))
                         if extract_first(r"Stanje prethodnog izvatka:\s*([\d\.]+,\d{2})", text) else None),
        "total_debit": (parse_eur(extract_first(r"Ukupni dugovni promet na dan:\s*([\d\.]+,\d{2})", text))
                        if extract_first(r"Ukupni dugovni promet na dan:\s*([\d\.]+,\d{2})", text) else None),
        "total_credit": (parse_eur(extract_first(r"Ukupni potražni promet na dan:\s*([\d\.]+,\d{2})", text))
                         if extract_first(r"Ukupni potražni promet na dan:\s*([\d\.]+,\d{2})", text) else None),
        "new_balance": (parse_eur(extract_first(r"Novo stanje:\s*([\d\.]+,\d{2})", text))
                        if extract_first(r"Novo stanje:\s*([\d\.]+,\d{2})", text) else None),
    }

def parse_statement(pdf_path: Path) -> Tuple[List[Dict], Dict]:
    text = pdftotext_layout(pdf_path)
    hdr = header_totals(text)
    ispl_idx, upl_idx, threshold = amount_column_indices(text)

    blocks = re.split(r"\n\s*_{10,}\s*\n", text)

    rows: List[Dict] = []
    sum_debit = Decimal("0")
    sum_credit = Decimal("0")

    for b in blocks:
        m = TX_START_RE.search(b)
        if not m:
            continue

        tx_no = int(m.group(1))
        counterparty_account = m.group(2)

        lines = b.splitlines()
        # Find the actual transaction “start line” (avoid matching balances like 1.706,08)
        start_i = None
        start_pat = re.compile(rf"^\s*{tx_no}\.\s+[A-Z]{{2}}")
        for i, line in enumerate(lines):
            if start_pat.match(line):
                start_i = i
                break
        if start_i is None:
            continue
        tx_lines = lines[start_i:]
        tx_line = tx_lines[0]
        raw_text_block = "\n".join(tx_lines).strip()

        # Column anchors from this tx line
        acct_start = tx_line.index(counterparty_account)
        after_acct = acct_start + len(counterparty_account)

        ref_banke = None
        ref_start = None
        desc_start = None
        refm = re.search(r"\b\d{8,}\b", tx_line[after_acct:])
        if refm:
            ref_banke = refm.group(0)
            ref_start = after_acct + refm.start()
            after_ref = after_acct + refm.end()
            ws = re.match(r"\s*", tx_line[after_ref:]).group(0)
            desc_start = after_ref + len(ws)
        else:
            ref_start = after_acct + 1
            desc_start = ref_start

        name_parts: List[str] = []
        refs_other: List[str] = []
        desc_parts: List[str] = []
        dates: List[str] = []

        debit: Optional[Decimal] = None
        credit: Optional[Decimal] = None

        for line in tx_lines:
            dates.extend(DATE_RE.findall(line))

            # Find the “final EUR amount” by position near the amount columns
            candidates = []
            for am in AMOUNT_RE.finditer(line):
                if am.start() >= ispl_idx - 20:
                    candidates.append(am)
            if candidates:
                am = max(candidates, key=lambda x: x.start())
                val = parse_eur(am.group(0))
                if am.start() < threshold:
                    debit = val
                else:
                    credit = val

            # Split text by inferred column anchors
            left = line[acct_start:ref_start].strip() if len(line) > acct_start else ""
            mid = line[ref_start:desc_start].strip() if (ref_start is not None and len(line) > ref_start) else ""
            desc = line[desc_start:ispl_idx] if (desc_start is not None and len(line) > desc_start) else ""

            if left and left != counterparty_account:
                name_parts.append(left)
            if mid and mid != (ref_banke or ""):
                refs_other.append(mid)
            if desc:
                normalized_desc = normalize_description(desc)
                if (normalized_desc
                        and not DATE_ONLY_RE.match(normalized_desc)
                        and not NUMBER_ONLY_RE.match(normalized_desc)):
                    desc_parts.append(normalized_desc)

        # Value date / execution date: last two dates in the block (PBZ prints them at the end)
        value_date = dates[-2] if len(dates) >= 2 else (dates[-1] if dates else None)
        execution_date = dates[-1] if dates else None

        # Purpose code (optional; conservative)
        purpose = None
        for code in PURPOSE_CODES:
            if re.search(rf"\b{re.escape(code)}\b", b):
                purpose = code
                break

        direction = "unknown"
        amount = None
        if debit is not None and credit is None:
            direction = "debit"
            amount = -debit
            sum_debit += debit
        elif credit is not None and debit is None:
            direction = "credit"
            amount = credit
            sum_credit += credit
        elif debit is not None and credit is not None:
            direction = "both"

        counterparty_name = " | ".join(name_parts) if name_parts else ""
        description = " | ".join(desc_parts) if desc_parts else ""
        refs_other = " | ".join(refs_other) if refs_other else ""
        tx_fingerprint = make_tx_fingerprint(
            hdr["statement_account"],
            execution_date,
            amount,
            ref_banke,
            counterparty_account,
            description,
        )

        rows.append({
            "statement_date": hdr["statement_date"],
            "statement_no": hdr["statement_no"],
            "statement_account": hdr["statement_account"],
            "tx_no": tx_no,
            "execution_date": execution_date,
            "value_date": value_date,
            "direction": direction,
            "debit": (str(debit) if debit is not None else ""),
            "credit": (str(credit) if credit is not None else ""),
            "amount": (str(amount) if amount is not None else ""),
            "counterparty_account": counterparty_account,
            "counterparty_name": counterparty_name,
            "description": description,
            "purpose_code": purpose or "",
            "ref_banke": ref_banke or "",
            "refs_other": refs_other,
            "source_pdf": pdf_path.name,
            "raw_text_block": raw_text_block,
            "tx_fingerprint": tx_fingerprint,
        })

    # Validation (totals)
    validation = {
        "source_pdf": pdf_path.name,
        "statement_date": hdr["statement_date"],
        "statement_no": hdr["statement_no"],
        "iban": hdr["statement_account"],
        "header_total_debit": str(hdr["total_debit"] or ""),
        "header_total_credit": str(hdr["total_credit"] or ""),
        "parsed_total_debit": str(sum_debit),
        "parsed_total_credit": str(sum_credit),
        "tx_count": str(len(rows)),
        "validation_ok": str((hdr["total_debit"] == sum_debit) and (hdr["total_credit"] == sum_credit)),
    }

    return rows, validation

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("pdfs", nargs="+", type=Path, help="PDF statement files")
    ap.add_argument("--out", type=Path, default=Path("pbz_statements.csv"))
    ap.add_argument("--validation-out", type=Path, default=Path("pbz_validation.csv"))
    ap.add_argument("--no-dedupe", action="store_true", help="Disable tx_fingerprint dedupe")
    args = ap.parse_args()

    all_rows: List[Dict] = []
    validations: List[Dict] = []
    seen_fingerprints: set[str] = set()
    dedupe = not args.no_dedupe

    for pdf in args.pdfs:
        rows, v = parse_statement(pdf)
        if dedupe:
            for row in rows:
                fp = row.get("tx_fingerprint")
                if fp in seen_fingerprints:
                    continue
                seen_fingerprints.add(fp)
                all_rows.append(row)
        else:
            all_rows.extend(rows)
        validations.append(v)

    if not all_rows:
        raise SystemExit("No transactions parsed.")

    fieldnames = list(all_rows[0].keys())
    with args.out.open("w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=fieldnames)
        w.writeheader()
        w.writerows(all_rows)

    v_fields = list(validations[0].keys())
    with args.validation_out.open("w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=v_fields)
        w.writeheader()
        w.writerows(validations)

if __name__ == "__main__":
    main()