1 {-|
    2 
    3 Convert account data in CSV format (eg downloaded from a bank) to ledger
    4 format, and print it on stdout.
    5 
    6 Usage: hledger convert CSVFILE ACCOUNTNAME RULESFILE
    7 
    8 ACCOUNTNAME is the base account to use for transactions.  RULESFILE
    9 provides some rules to help convert the data. It should contain paragraphs
   10 separated by one blank line.  The first paragraph is a single line of five
   11 comma-separated numbers, which are the csv field positions corresponding
   12 to the ledger transaction's date, status, code, description, and amount.
   13 All other paragraphs specify one or more regular expressions, followed by
   14 the ledger account to use when a transaction's description matches any of
   15 them. A regexp may optionally have a replacement pattern specified after =.
   16 Here's an example rules file:
   17 
   18 > 0,2,3,4,1
   19 >
   20 > ATM DEPOSIT
   21 > assets:bank:checking
   22 >
   23 > (TO|FROM) SAVINGS
   24 > assets:bank:savings
   25 >
   26 > ITUNES
   27 > BLKBSTR=BLOCKBUSTER
   28 > expenses:entertainment
   29 
   30 Roadmap: 
   31 Support for other formats will be added. To update a ledger file, pipe the
   32 output into the import command. The rules will move to a hledger config
   33 file. When no rule matches, accounts will be guessed based on similarity
   34 to descriptions in the current ledger, with interactive prompting and
   35 optional rule saving.
   36 
   37 -}
   38 
   39 module Commands.Convert where
   40 import Data.Maybe (isJust)
   41 import Data.List.Split (splitOn)
   42 import Options -- (Opt,Debug)
   43 import Ledger.Types (Ledger,AccountName)
   44 import Ledger.Utils (strip)
   45 import System (getArgs)
   46 import System.IO (stderr, hPutStrLn)
   47 import Text.CSV (parseCSVFromFile, Record)
   48 import Text.Printf (printf)
   49 import Text.RegexPR (matchRegexPR)
   50 import Data.Maybe
   51 import Ledger.Dates (firstJust, showDate)
   52 import Locale (defaultTimeLocale)
   53 import Data.Time.Format (parseTime)
   54 import Control.Monad (when)
   55 
   56 
   57 convert :: [Opt] -> [String] -> Ledger -> IO ()
   58 convert opts args l = do
   59   when (length args /= 3) (error "please specify a csv file, base account, and import rules file.")
   60   let [csvfile,baseacct,rulesfile] = args
   61   rulesstr <- readFile rulesfile
   62   (fieldpositions,rules) <- parseRules rulesstr
   63   parse <- parseCSVFromFile csvfile
   64   let records = case parse of
   65                   Left e -> error $ show e
   66                   Right rs -> reverse rs
   67   mapM_ (print_ledger_txn (Debug `elem` opts) (baseacct,fieldpositions,rules)) records
   68 
   69 
   70 type Rule = (
   71    [(String, Maybe String)] -- list of patterns and optional replacements
   72   ,AccountName              -- account name to use for a matched transaction
   73   )
   74 
   75 parseRules :: String -> IO ([Int],[Rule])
   76 parseRules s = do
   77   let ls = map strip $ lines s
   78   let paras = splitOn [""] ls
   79   let fieldpositions = map read $ splitOn "," $ head $ head paras
   80   let rules = [(map parsePatRepl $ init ls, last ls) | ls <- tail paras]
   81   return (fieldpositions,rules)
   82 
   83 parsePatRepl :: String -> (String, Maybe String)
   84 parsePatRepl l = case splitOn "=" l of
   85                    (p:r:_) -> (p, Just r)
   86                    (p:_)   -> (p, Nothing)
   87 
   88 print_ledger_txn debug (baseacct,fieldpositions,rules) record@(a:b:c:d:e) = do
   89   let [date,cleared,number,description,amount] = map (record !!) fieldpositions
   90       amount' = strnegate amount where strnegate ('-':s) = s
   91                                        strnegate s = '-':s
   92       unknownacct | (read amount' :: Double) < 0 = "income:unknown"
   93                   | otherwise = "expenses:unknown"
   94       (acct,desc) = choose_acct_desc rules (unknownacct,description)
   95   when (debug) $ hPutStrLn stderr $ printf "using %s for %s" desc description
   96   putStrLn $ printf "%s%s %s" (fixdate date) (if not (null number) then printf " (%s)" number else "") desc
   97   putStrLn $ printf "    %-30s  %15s" acct (printf "$%s" amount' :: String)
   98   putStrLn $ printf "    %s\n" baseacct
   99 print_ledger_txn True _ record = do
  100   hPutStrLn stderr $ printf "ignoring %s" $ show record
  101 print_ledger_txn _ _ _ = return ()
  102 
  103 choose_acct_desc :: [Rule] -> (String,String) -> (String,String)
  104 choose_acct_desc rules (acct,desc) | null matchingrules = (acct,desc)
  105                                    | otherwise = (a,d)
  106     where
  107       matchingrules = filter ismatch rules :: [Rule]
  108           where ismatch = any (isJust . flip matchregex desc . fst) . fst
  109       (prs,a) = head matchingrules
  110       mrs = filter (isJust . fst) $ map (\(p,r) -> (matchregex p desc, r)) prs
  111       (m,repl) = head mrs
  112       matched = fst $ fst $ fromJust m
  113       d = fromMaybe matched repl
  114 
  115 matchregex s = matchRegexPR ("(?i)"++s)
  116 
  117 fixdate :: String -> String
  118 fixdate s = maybe "0000/00/00" showDate $ 
  119               firstJust
  120               [parseTime defaultTimeLocale "%Y/%m/%d" s
  121               ,parseTime defaultTimeLocale "%Y-%m-%d" s
  122               ,parseTime defaultTimeLocale "%m/%d/%Y" s
  123               ,parseTime defaultTimeLocale "%m-%d-%Y" s
  124               ]
  125