initial - ai
This commit is contained in:
Lukas Pierchalla
@@ -0,0 +1,398 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
scalable-sync: Scalable Capital → Wealthfolio MCP Sync Adapter
|
||||
Alle Config über Environment-Variablen.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass, asdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
import requests
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
WF_MCP_URL = os.environ["WF_MCP_URL"]
|
||||
WF_MCP_TOKEN = os.environ["WF_MCP_TOKEN"]
|
||||
WF_ACCOUNT_ID = os.environ.get("WF_ACCOUNT_ID")
|
||||
SYNC_INTERVAL_HOURS = int(os.getenv("SYNC_INTERVAL_HOURS", "6"))
|
||||
SC_OVERLAP_DAYS = int(os.getenv("SC_OVERLAP_DAYS", "14"))
|
||||
SC_SETTLED_ONLY = os.getenv("SC_SETTLED_ONLY", "true").lower() == "true"
|
||||
SC_MAPPING_VERSION = int(os.getenv("SC_MAPPING_VERSION", "1"))
|
||||
ASSET_MAP = json.loads(os.getenv("ASSET_MAP", "{}"))
|
||||
|
||||
# Scalable CLI Config: tmpfs-basiert, automatisch generiert
|
||||
SC_CONFIG_DIR = Path(os.getenv("SC_CONFIG_DIR", tempfile.mkdtemp(prefix="sc-config-")))
|
||||
SC_CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(SC_CONFIG_DIR / "config.toml").write_text(
|
||||
'[auth]\nsession_backend = "file"\nsigning_key_backend = "file"\n'
|
||||
)
|
||||
os.environ["SC_CONFIG_DIR"] = str(SC_CONFIG_DIR)
|
||||
|
||||
DATA_DIR = Path("/app/data")
|
||||
DATA_DIR.mkdir(exist_ok=True)
|
||||
DB_PATH = DATA_DIR / "state.db"
|
||||
AUDIT_PATH = DATA_DIR / "audit.jsonl"
|
||||
DEAD_LETTER_PATH = DATA_DIR / "dead-letter.jsonl"
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
handlers=[logging.StreamHandler(sys.stdout)]
|
||||
)
|
||||
log = logging.getLogger("scalable-sync")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scalable CLI Wrapper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def sc(*args: str, timeout: int = 60) -> dict:
|
||||
cmd = ["sc"] + list(args)
|
||||
env = {**os.environ, "HOME": "/home/sync", "SC_CONFIG_DIR": str(SC_CONFIG_DIR)}
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, env=env)
|
||||
if result.returncode != 0:
|
||||
log.error("sc failed: %s", result.stderr)
|
||||
raise RuntimeError(f"sc exit {result.returncode}: {result.stderr}")
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def sc_transactions(from_time=None, to_time=None, type_filter=None, status=None, isin=None, cursor=None, page_size=100):
|
||||
args = ["broker", "transactions", "--page-size", str(page_size)]
|
||||
if from_time: args += ["--from-time", from_time.isoformat()]
|
||||
if to_time: args += ["--to-time", to_time.isoformat()]
|
||||
if type_filter:
|
||||
for t in type_filter: args += ["--type-filter", t]
|
||||
if status:
|
||||
for s in status: args += ["--status", s]
|
||||
if isin: args += ["--isin", isin]
|
||||
if cursor: args += ["--cursor", cursor]
|
||||
return sc(*args)
|
||||
|
||||
|
||||
def sc_transaction_details(tx_id: str) -> dict:
|
||||
return sc("broker", "transaction", "details", "--transaction-id", tx_id)
|
||||
|
||||
|
||||
def sc_holdings() -> dict:
|
||||
return sc("broker", "holdings")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# State
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def init_db():
|
||||
with sqlite3.connect(DB_PATH) as conn:
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS sync_state (
|
||||
scalable_transaction_id TEXT PRIMARY KEY,
|
||||
source_hash TEXT NOT NULL,
|
||||
source_status TEXT NOT NULL,
|
||||
wealthfolio_activity_ids TEXT,
|
||||
imported_at TEXT,
|
||||
last_seen_at TEXT NOT NULL,
|
||||
mapping_version INTEGER NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_last_seen ON sync_state(last_seen_at)")
|
||||
|
||||
|
||||
def hash_transaction(tx: dict) -> str:
|
||||
relevant = {k: tx.get(k) for k in [
|
||||
"id", "type", "status", "last_event_datetime", "security",
|
||||
"security_trade", "cash", "non_trade_security"
|
||||
]}
|
||||
return json.dumps(relevant, sort_keys=True, default=str)
|
||||
|
||||
|
||||
def is_imported(tx_id: str, current_hash: str) -> bool:
|
||||
with sqlite3.connect(DB_PATH) as conn:
|
||||
row = conn.execute("SELECT source_hash FROM sync_state WHERE scalable_transaction_id = ?", (tx_id,)).fetchone()
|
||||
return row is not None and row[0] == current_hash
|
||||
|
||||
|
||||
def mark_imported(tx_id, tx_hash, status, wf_ids):
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
with sqlite3.connect(DB_PATH) as conn:
|
||||
conn.execute("""
|
||||
INSERT OR REPLACE INTO sync_state VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""", (tx_id, tx_hash, status, json.dumps(wf_ids), now, now, SC_MAPPING_VERSION))
|
||||
|
||||
|
||||
def mark_seen(tx_id, tx_hash, status):
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
with sqlite3.connect(DB_PATH) as conn:
|
||||
conn.execute("""
|
||||
INSERT OR IGNORE INTO sync_state (scalable_transaction_id, source_hash, source_status, last_seen_at, mapping_version)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""", (tx_id, tx_hash, status, now, SC_MAPPING_VERSION))
|
||||
conn.execute("UPDATE sync_state SET last_seen_at = ? WHERE scalable_transaction_id = ?", (now, tx_id))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class WealthfolioActivity:
|
||||
date: str
|
||||
symbol: str
|
||||
activity_type: str
|
||||
quantity: Optional[float] = None
|
||||
unit_price: Optional[float] = None
|
||||
amount: Optional[float] = None
|
||||
fee: Optional[float] = None
|
||||
tax: Optional[float] = None
|
||||
currency: str = "EUR"
|
||||
source: str = "scalable-cli"
|
||||
external_id: Optional[str] = None
|
||||
account_id: Optional[str] = None
|
||||
|
||||
|
||||
def dead_letter(tx, reason):
|
||||
entry = {"timestamp": datetime.now(timezone.utc).isoformat(), "reason": reason, "transaction": tx}
|
||||
with open(DEAD_LETTER_PATH, "a") as f:
|
||||
f.write(json.dumps(entry, default=str) + "\n")
|
||||
log.warning("Dead letter: %s (%s)", tx.get("id"), reason)
|
||||
|
||||
|
||||
def audit(action, payload):
|
||||
entry = {"timestamp": datetime.now(timezone.utc).isoformat(), "action": action, "payload": payload}
|
||||
with open(AUDIT_PATH, "a") as f:
|
||||
f.write(json.dumps(entry, default=str) + "\n")
|
||||
|
||||
|
||||
def map_to_wealthfolio(tx: dict) -> list[WealthfolioActivity]:
|
||||
tx_type = tx.get("type")
|
||||
status = (tx.get("status") or "").upper()
|
||||
if tx.get("is_cancellation") or status in {"CANCELLED", "EXPIRED", "REJECTED", "CANCEL_REQUESTED"}:
|
||||
return []
|
||||
if SC_SETTLED_ONLY and status not in {"SETTLED", "CONFIRMED"}:
|
||||
return []
|
||||
if tx.get("is_pending"):
|
||||
return []
|
||||
|
||||
sec = tx.get("security") or {}
|
||||
isin = sec.get("isin")
|
||||
symbol = ASSET_MAP.get(isin) if isin else None
|
||||
|
||||
st = tx.get("security_trade") or {}
|
||||
cash = tx.get("cash") or {}
|
||||
nts = tx.get("non_trade_security") or {}
|
||||
last_event = (tx.get("last_event_datetime") or "")[:10]
|
||||
tx_id = tx.get("id", "")
|
||||
|
||||
activities = []
|
||||
|
||||
if tx_type in {"BUY", "SELL", "SAVINGS_PLAN", "REINVESTMENT"}:
|
||||
if not symbol:
|
||||
dead_letter(tx, f"unmapped ISIN: {isin}")
|
||||
return []
|
||||
qty = (st.get("number_of_shares") or {}).get("filled") or nts.get("quantity")
|
||||
price = st.get("average_price") or nts.get("average_price")
|
||||
amount = st.get("total_amount") or nts.get("total_amount")
|
||||
fee = st.get("fee") or st.get("transactional_fee")
|
||||
tax = (st.get("aggregated_transaction_taxes") or {}).get("total_tax")
|
||||
activities.append(WealthfolioActivity(
|
||||
date=last_event, symbol=symbol,
|
||||
activity_type="BUY" if tx_type in {"BUY", "SAVINGS_PLAN", "REINVESTMENT"} else "SELL",
|
||||
quantity=qty, unit_price=price, amount=amount, fee=fee, tax=tax,
|
||||
external_id=tx_id, account_id=WF_ACCOUNT_ID
|
||||
))
|
||||
|
||||
elif tx_type == "DEPOSIT":
|
||||
activities.append(WealthfolioActivity(date=last_event, symbol="$CASH", activity_type="DEPOSIT", amount=cash.get("amount"), external_id=tx_id, account_id=WF_ACCOUNT_ID))
|
||||
elif tx_type == "WITHDRAWAL":
|
||||
activities.append(WealthfolioActivity(date=last_event, symbol="$CASH", activity_type="WITHDRAWAL", amount=cash.get("amount"), external_id=tx_id, account_id=WF_ACCOUNT_ID))
|
||||
elif tx_type == "DISTRIBUTION":
|
||||
if not symbol:
|
||||
dead_letter(tx, "distribution without ISIN")
|
||||
return []
|
||||
activities.append(WealthfolioActivity(date=last_event, symbol=symbol, activity_type="DIVIDEND", amount=cash.get("amount"), external_id=tx_id, account_id=WF_ACCOUNT_ID))
|
||||
elif tx_type == "INTEREST":
|
||||
activities.append(WealthfolioActivity(date=last_event, symbol="$CASH", activity_type="INTEREST", amount=cash.get("amount"), external_id=tx_id, account_id=WF_ACCOUNT_ID))
|
||||
elif tx_type == "FEE":
|
||||
activities.append(WealthfolioActivity(date=last_event, symbol="$CASH", activity_type="FEE", amount=cash.get("amount"), external_id=tx_id, account_id=WF_ACCOUNT_ID))
|
||||
elif tx_type == "TAX":
|
||||
activities.append(WealthfolioActivity(date=last_event, symbol="$CASH", activity_type="TAX", amount=cash.get("amount"), external_id=tx_id, account_id=WF_ACCOUNT_ID))
|
||||
elif tx_type == "TAX_RETURN":
|
||||
activities.append(WealthfolioActivity(date=last_event, symbol="$CASH", activity_type="CREDIT", amount=cash.get("amount"), external_id=tx_id, account_id=WF_ACCOUNT_ID))
|
||||
elif tx_type in {"TRANSFER_IN", "CASH_TRANSFER_IN"}:
|
||||
target = "TRANSFER_IN" if symbol else "DEPOSIT"
|
||||
activities.append(WealthfolioActivity(
|
||||
date=last_event, symbol=symbol or "$CASH", activity_type=target,
|
||||
quantity=nts.get("quantity"), amount=cash.get("amount") or nts.get("total_amount"),
|
||||
external_id=tx_id, account_id=WF_ACCOUNT_ID
|
||||
))
|
||||
elif tx_type in {"TRANSFER_OUT", "CASH_TRANSFER_OUT"}:
|
||||
target = "TRANSFER_OUT" if symbol else "WITHDRAWAL"
|
||||
activities.append(WealthfolioActivity(
|
||||
date=last_event, symbol=symbol or "$CASH", activity_type=target,
|
||||
quantity=nts.get("quantity"), amount=cash.get("amount") or nts.get("total_amount"),
|
||||
external_id=tx_id, account_id=WF_ACCOUNT_ID
|
||||
))
|
||||
elif tx_type in {
|
||||
"SWAP_IN", "SWAP_OUT", "CURRENCY_SWITCH_BUY", "CURRENCY_SWITCH_SELL",
|
||||
"POCKET_MONEY", "REINVESTMENT_DISTRIBUTION", "REINVESTMENT_POCKET_MONEY"
|
||||
}:
|
||||
dead_letter(tx, f"unsupported type: {tx_type}")
|
||||
return []
|
||||
else:
|
||||
dead_letter(tx, f"unknown type: {tx_type}")
|
||||
return []
|
||||
|
||||
return activities
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wealthfolio MCP Client
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class WealthfolioMCP:
|
||||
def __init__(self, url, token):
|
||||
self.url = url
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
})
|
||||
|
||||
def _rpc(self, method, params=None):
|
||||
payload = {"jsonrpc": "2.0", "id": int(time.time() * 1000), "method": method}
|
||||
if params: payload["params"] = params
|
||||
resp = self.session.post(self.url, json=payload, timeout=30)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def ping(self):
|
||||
try:
|
||||
return self._rpc("ping").get("result") == "pong"
|
||||
except Exception as e:
|
||||
log.error("MCP ping failed: %s", e)
|
||||
return False
|
||||
|
||||
def record_activities(self, activities):
|
||||
payload = [{
|
||||
"accountId": a.account_id, "date": a.date, "symbol": a.symbol, "activityType": a.activity_type,
|
||||
"quantity": a.quantity, "unitPrice": a.unit_price, "amount": a.amount,
|
||||
"fee": a.fee, "tax": a.tax, "currency": a.currency,
|
||||
"source": a.source, "externalId": a.external_id,
|
||||
} for a in activities]
|
||||
return self._rpc("tools/call", {"name": "record_activities", "arguments": {"activities": payload}})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sync
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def reconcile(mcp):
|
||||
try:
|
||||
sc_h = sc_holdings()
|
||||
log.info("Scalable holdings count: %s", len(sc_h.get("items", [])))
|
||||
audit("reconcile", {"scalable_holdings": sc_h})
|
||||
return True
|
||||
except Exception as e:
|
||||
log.error("Reconcile failed: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def sync_once(dry_run=False):
|
||||
init_db()
|
||||
mcp = WealthfolioMCP(WF_MCP_URL, WF_MCP_TOKEN)
|
||||
if not mcp.ping():
|
||||
raise RuntimeError("Wealthfolio MCP unreachable")
|
||||
|
||||
from_time = datetime.now(timezone.utc) - timedelta(days=SC_OVERLAP_DAYS)
|
||||
to_time = datetime.now(timezone.utc)
|
||||
stats = {"new": 0, "skipped": 0, "dead_letter": 0, "error": 0}
|
||||
cursor = None
|
||||
|
||||
while True:
|
||||
page = sc_transactions(from_time=from_time, to_time=to_time, cursor=cursor, page_size=100)
|
||||
items = page.get("items", [])
|
||||
cursor = page.get("cursor")
|
||||
|
||||
for summary in items:
|
||||
tx_id = summary.get("id")
|
||||
tx_hash = hash_transaction(summary)
|
||||
status = (summary.get("status") or "").upper()
|
||||
|
||||
if is_imported(tx_id, tx_hash):
|
||||
mark_seen(tx_id, tx_hash, status)
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
detail = sc_transaction_details(tx_id)
|
||||
except Exception as e:
|
||||
log.error("Failed details %s: %s", tx_id, e)
|
||||
stats["error"] += 1
|
||||
continue
|
||||
|
||||
detail_hash = hash_transaction(detail)
|
||||
activities = map_to_wealthfolio(detail)
|
||||
if not activities:
|
||||
stats["dead_letter"] += 1
|
||||
continue
|
||||
|
||||
if dry_run:
|
||||
log.info("[DRY RUN] Would import %s: %s", tx_id, [
|
||||
f"{a.activity_type} {a.symbol} qty={a.quantity} amt={a.amount}" for a in activities
|
||||
])
|
||||
audit("dry_run", {"tx_id": tx_id, "activities": [asdict(a) for a in activities]})
|
||||
else:
|
||||
try:
|
||||
result = mcp.record_activities(activities)
|
||||
mark_imported(tx_id, detail_hash, status, [str(i) for i in range(len(activities))])
|
||||
audit("imported", {"tx_id": tx_id, "activities": len(activities)})
|
||||
stats["new"] += len(activities)
|
||||
except Exception as e:
|
||||
log.error("Import failed %s: %s", tx_id, e)
|
||||
stats["error"] += 1
|
||||
|
||||
if not cursor or not items:
|
||||
break
|
||||
|
||||
reconcile(mcp)
|
||||
log.info("Sync complete: %s", stats)
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--once", action="store_true")
|
||||
p.add_argument("--dry-run", action="store_true")
|
||||
p.add_argument("--daemon", action="store_true")
|
||||
args = p.parse_args()
|
||||
|
||||
if args.once or args.dry_run:
|
||||
stats = sync_once(dry_run=args.dry_run)
|
||||
sys.exit(0 if stats["error"] == 0 else 1)
|
||||
|
||||
if args.daemon:
|
||||
log.info("Starting daemon, interval=%dh", SYNC_INTERVAL_HOURS)
|
||||
while True:
|
||||
try:
|
||||
sync_once()
|
||||
except Exception as e:
|
||||
log.exception("Sync failed: %s", e)
|
||||
time.sleep(SYNC_INTERVAL_HOURS * 3600)
|
||||
else:
|
||||
p.print_help()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user