From 3c4974797e6983c1248bb5162551276734fe8f3d Mon Sep 17 00:00:00 2001 From: Daniel O'Connell Date: Tue, 8 Oct 2019 21:10:07 +0200 Subject: [PATCH] Worklogs from emails --- README.md | 103 ++++++++++++++++++++++++++---------- project.clj | 2 +- resources/config.edn | 58 +++++++++++--------- src/invoices/core.clj | 48 +++++++++-------- src/invoices/email.clj | 29 +++++----- src/invoices/pdf.clj | 27 ++++++---- src/invoices/settings.clj | 52 ++++++++++++++++-- src/invoices/timesheets.clj | 27 ++++++++-- 8 files changed, 239 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index 016391e..5ebfc44 100644 --- a/README.md +++ b/README.md @@ -21,16 +21,16 @@ The following options are available: ## Config file -The config file should be a EDN file containing a list of invoices. Each invoice can have the -following keys: +The config file should be a EDN file containing a list of invoices, seller info, +optional font info and optional worklogs info, e.g.: - * :seller - the seller's (i.e. the entity to be paid) information. This is required - * :buyer - the buyer's (i.e. the entity that will pay) information. This is required - * :items - a list of items to be paid for - * :credentials - (optional) JIRA and Tempo access credentials. These are needed if the price depends on tracked time - * :font-path - (optional) the path to a font file, e.g. "/usr/share/fonts/truetype/freefont/FreeSans.ttf" - * :callbacks - (optional) a list of commands to be called with the resulting pdf file + {:seller {(...)} + :invoices [(...)] + :font-path "/path/to/font" + :worklogs [(...)]} + +`:font-path` should be the path to a font file, e.g. "/usr/share/fonts/truetype/freefont/FreeSans.ttf" See `resources/config.edn` for an example configuration. ### Seller @@ -42,10 +42,17 @@ See `resources/config.edn` for an example configuration. * :nip - (required) the NIP of the seller, e.g. 1234567890 * :account - (required) the number of the account to which the payment should go, e.g. "12 4321 8765 1000 0000 1222 3212" * :bank - (required) the name of the bank in which the account is held, e.g. "Piggy bank" - * :email - (optional) the email of the seller, e.g. "mr.blobby@bla.com". This is required if a confirmation email is to be sent * :phone - (optional) the phone number of the seller 555333111 * :team - (optional) a team name, to be prepended to the name of the resulting pdf, e.g. "the A team" +### Invoices + +Each invoice can have the following keys: + + * :buyer - the buyer's (i.e. the entity that will pay) information. This is required + * :items - a list of items to be paid for + * :imap - (optional) email credentials. These are needed if a confirmation email is to be sent + * :callbacks - (optional) a list of commands to be called with the resulting pdf file ### Buyer @@ -77,6 +84,8 @@ The price can be provided in one of the following ways: * :function - an S-expression describing how to calculate the net value. Only numbers, basic mathematical operations (+, -, /, *) and timesheet specific variables are supported (:worked, :required). +If the price is to be calculated on the basis of a worklog, add a `:worklog` key +and make sure the `:worklogs` section has an item that can be used to access the worklog. Examples: @@ -84,42 +93,80 @@ Examples: {:vat 8 :netto 600 :title "Shoes" :to "2019-05-30" :notes ["A note at the bottom"]} ; 12% VAT, and an hourly rate of 12, first appearing on 2019-07-01 - {:vat 12 :hourly 12 :title "Something worth 12/h" :from "2019-07-01"} + {:vat 12 :hourly 12 :title "Something worth 12/h" :from "2019-07-01" :worklog "washed_dishes"} ; 23% VAT, working part time with a base salary of 5000 - {:vat 23 :base 5000 :per-day 4 :title "Part time job at 5000"}] + {:vat 23 :base 5000 :per-day 4 :title "Part time job at 5000" :worklog "cleaned_shoes"}] ; 23% VAT, with a custom function - {:vat 23 :function (* :worked (/ 10000 :required)) :title "Custom function"}] + {:vat 23 :function (* :worked (/ 10000 :required)) :title "Custom function"} :worklog :from-jira] -### Credentials +### Worklogs In the case of hourly rates or variable hours, the number of hours worked needs to be fetched -from a time tracker. Which requires appropriate credentials. See -[Jira's](https://developer.atlassian.com/cloud/jira/platform/jira-rest-api-basic-authentication/) +from a time tracker. Which requires appropriate credentials, described in the +following sections. Apart from provider specific values, each credentials map +must contain a `:type` key that describes the provider, and a `:ids` list, which +should contain all worklog ids that can be found in the given worklog. These ids +are used to link worklog values with items via the `:worklog` key of items. + +#### Jira + +See [Jira's](https://developer.atlassian.com/cloud/jira/platform/jira-rest-api-basic-authentication/) and [Tempo's](https://tempo-io.atlassian.net/wiki/spaces/KB/pages/199065601/How+to+use+Tempo+Cloud+REST+APIs) -documentation on how to get the appropriate tokens. Once the tokens are generated, the :credentials -should look like the following: +documentation on how to get the appropriate tokens. Once the tokens are generated, add an appropriate +worklog entry like the following: - :credentials {:tempo-token "5zq7zF9LADefEGAs12eDDas3FDttiM" - :jira-token "qypaAsdFwASasEddDDddASdC" - :jira-user "mr.blobby@boots.rs"} + :worklogs [{:type :jira + :ids [:from-jira] + :tempo-token "5zq7zF9LADefEGAs12eDDas3FDttiM" + :jira-token "qypaAsdFwASasEddDDddASdC" + :jira-user "mr.blobby@boots.rs"}] -If a confirmation email is to be sent, a :smtp key must also be provided, e.g.: +#### Emails +Emails with worklogs should be sent in a psudo csv format, seperated by `;` or +whitespace. Use the `:headers` key to describe what data is contained in each +column. +The emails are looked for in the `:folder` folder of the email account, and all +emails from `:from` (or anyone if `:from` is nil or missing) and with the subject +contining `:subject` formatted with the processed date. - :credentials {:smtp {:host "smtp.gmail.com" - :user "mr.blobby@buty.sa" - :pass "asd;l;kjsdfkljld" - :ssl true}} +Assuming the processed date is 2012-12-12, and the following configuration is provided: + + :worklogs [{:type :imap + :ids [:item1 :item2] + :folder "inbox" + :host "imap.gmail.com" + :user "mr.blobby@boots.rs" + :pass "lksjdfklsjdflkjw" + :from "hr@boots.rs" + :subject "'Hours 'YYYY-MM" + :headers [:id :worked]}] + +if `hr@boots.rs` sends an email to the `inbox` folder of `mr.blobby@boots.rs`'s +email account with the title `Hours 2012-12` and the following contents (notice the underscores): + + washed_dishes; 12 + cleaned_shoes; 43 + +the following work logs will be found: + + [{:id "washed_dished" :worked 12} + {:id "cleaned_shoes" :worked 43}] ## Confirmation emails -Each invoice can also be sent via email to the appropriate seller. For this to work, both the seller -and the buyer must have an :email key set and the credentials must contain a :smtp key with the -:smtp settings for the email server. +Each invoice can also be sent via email to the appropriate seller. For this to work, the buyer must +have an :email key set and a :smtp key with the :smtp settings for the email server should be provided. + :invoices [{:buyer {(...) :email "accounting@boots.rs"} + (...) + :smtp {:host "smtp.gmail.com" + :user "mr.blobby@buty.sa" + :pass "asd;l;kjsdfkljld" + :ssl true}}] ## Callbacks diff --git a/project.clj b/project.clj index 4735784..8936bee 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject invoices "0.1.0-SNAPSHOT" +(defproject invoices "0.1.1" :description "Generate invoices on the basis of a config file" :url "https://github.com/mruwnik/invoices" :license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0" diff --git a/resources/config.edn b/resources/config.edn index 6d6eb35..e369c8a 100644 --- a/resources/config.edn +++ b/resources/config.edn @@ -1,25 +1,33 @@ -[{:seller {:name "Mr. Blobby" - :email "mr.blobby@buty.sa" - :address "ul. podwodna, 12-345, Mierzów" - :nip 1234567890 - :phone 876543216 - :account "65 2345 1233 1233 4322 3211 4567" - :bank "Skok hop"} - :buyer {:name "Buty S.A." - :email "faktury@buty.sa" - :address "ul. Szewska 32, 76-543, Bąków" - :nip 9875645342} - :items [{:vat 8 :netto 123.21 :title "Buty kowbojskie"} - {:vat 21 :hourly 43.12 :title "Usługa szewska"} - {:netto 321.45 :title "Usługa szewska bez VAT"} - {:netto 321.45 :title "Usługa szewska zwolniona z VAT" :notes ["Podstawa zwolnienia z VAT: art. 113 ust. 1 i 9 Ustawa o VAT"]} - {:vat 23 :function (* :worked (+ 1 2 3 (- 23 13))) :title "Pucowania obuwia"} - {:vat 23 :base 4300.00 :per-day 4 :title "Praca za ladą"}] - :font-path "/usr/share/fonts/truetype/freefont/FreeSans.ttf" - :credentials {:tempo-token "5zq7zF9LADefEGAs12eDDas3FDttiM" - :jira-token "qypaAsdFwASasEddDDddASdC" - :jira-user "mr.blobby@boots.rs" - :smtp {:host "smtp.gmail.com" - :user "mr.blobby@buty.sa" - :pass "asd;l;kjsdfkljld" - :ssl true}}}] +{:seller {:name "Mr. Blobby" + :email "mr.blobby@buty.sa" + :address "ul. podwodna, 12-345, Mierzów" + :nip 1234567890 + :phone 876543216 + :account "65 2345 1233 1233 4322 3211 4567" + :bank "Skok hop"} + :font-path "/usr/share/fonts/truetype/freefont/FreeSans.ttf" + :invoices [{:buyer {:name "Buty S.A." + :email "faktury@buty.sa" + :address "ul. Szewska 32, 76-543, Bąków" + :nip 9875645342} + :items [{:vat 8 :netto 123.21 :title "Buty kowbojskie"} + {:vat 21 :hourly 43.12 :title "Usługa szewska" :worklog :from-jira} + {:netto 321.45 :title "Usługa szewska bez VAT"} + {:netto 321.45 :title "Usługa szewska zwolniona z VAT" :notes ["Podstawa zwolnienia z VAT: art. 113 ust. 1 i 9 Ustawa o VAT"]} + {:vat 23 :function (* :worked (+ 1 2 3 (- 23 13))) :title "Pucowania obuwia"} + {:vat 23 :base 4300.00 :per-day 4 :title "Praca za ladą"}] + :smtp {:host "smtp.gmail.com" :user "mr.blobby@buty.sa" :pass "asd;l;kjsdfkljld" :ssl true}}] + :worklogs [{:type :jira + :ids [:from-jira] + :tempo-token "5zq7zF9LADefEGAs12eDDas3FDttiM" + :jira-token "qypaAsdFwASasEddDDddASdC" + :jira-user "mr.blobby@boots.rs"} + {:type :imap + :ids [:item1 :item2] + :folder "inbox" + :host "imap.gmail.com" + :user "mr.blobby@boots.rs" + :pass "lksjdfklsjdflkjw" + :from "hr@boots.rs" + :subject "'Hours 'YYYY-MM" + :headers [:id :worked]}]} diff --git a/src/invoices/core.clj b/src/invoices/core.clj index bacde2b..978134b 100644 --- a/src/invoices/core.clj +++ b/src/invoices/core.clj @@ -1,10 +1,10 @@ (ns invoices.core (:require [invoices.pdf :as pdf] [invoices.settings :refer [invoices]] - [invoices.timesheets :refer [prev-timesheet]] + [invoices.timesheets :refer [timesheets]] [invoices.time :refer [prev-month last-working-day date-applies?]] [invoices.calc :refer [set-price]] - [invoices.email :refer [send-email]] + [invoices.email :as email] [clojure.tools.cli :refer [parse-opts]] [clojure.string :as str] [clojure.java.shell :refer [sh]]) @@ -25,26 +25,28 @@ (defn run-callbacks [invoice callbacks] (doall (map (partial run-callback invoice) callbacks))) - -(defn render-month [when {seller :seller buyer :buyer items :items creds :credentials font-path :font-path} number] - (pdf/render seller buyer - (->> items - (filter (partial date-applies? when)) - (map (partial set-price (prev-timesheet when creds)))) - (last-working-day when) - (invoice-number when number) - font-path)) - -(defn for-month [when {seller :seller buyer :buyer creds :credentials callbacks :callbacks :as invoice} & [number]] - (let [file (-> when (render-month invoice number) (str ".pdf") java.io.File. .getAbsolutePath)] - (send-email (:email buyer) (:email seller) creds file) +(defn for-month [{seller :seller buyer :buyer smtp :smtp callbacks :callbacks :as invoice} when & [number font]] + (let [file (pdf/render invoice (last-working-day when) (invoice-number when number))] + (email/send-invoice file (:email buyer) smtp) (run-callbacks file callbacks))) -(defn get-invoices [nips config] - (if (seq nips) - (filter #(some #{(-> % :buyer :nip str)} nips) (invoices config)) - (invoices config))) +(defn item-price [worklogs item] + (-> item :worklog (worklogs) (set-price item))) +(defn calc-prices [invoice worklogs] + (update invoice :items (partial map (partial item-price worklogs)))) + +(defn prepare-invoice [{seller :seller font :font-path} month worklogs invoice] + (-> invoice + (assoc :seller seller) + (assoc :font-path font) + (calc-prices worklogs))) + +(defn process-invoices [{invoices :invoices :as config} month worklogs] + (let [invoices (map (partial prepare-invoice config month worklogs) invoices)] + (doseq [[i invoice] (map-indexed vector invoices)] + (for-month invoice month (inc i)) + (println)))) (def cli-options [["-n" "--number NUMBER" "Invoice number. In the case of multiple invoices, they will have subsequent numbers" @@ -78,8 +80,10 @@ (:help options) (help summary) errors (exit -1 (str/join "\n" errors)) (not= 1 (count arguments)) (exit -1 "No config file provided")) + (println "Generating invoices") - (doseq [[i invoice] (map-indexed vector (get-invoices (:company options) (first arguments)))] - (for-month (:when options) invoice (+ i (:number options))) - (println))) + (let [month (java.time.LocalDate/now) + config (-> "config.edn" (invoices month)) + worklogs (timesheets month (:worklogs config))] + (process-invoices config month worklogs))) (shutdown-agents)) diff --git a/src/invoices/email.clj b/src/invoices/email.clj index 99f71a9..38747f2 100644 --- a/src/invoices/email.clj +++ b/src/invoices/email.clj @@ -7,18 +7,17 @@ [clojure-mail.folder :as folder] [clojure-mail.message :refer (read-message) :as mess])) -(defn send-email [to from {smtp :smtp} invoice] - (when (not-any? nil? [to from smtp invoice]) - (->> - (send-message smtp {:from from - :to [to] - :subject invoice - :body [{:type :attachment - :content (java.io.File. (str invoice ".pdf")) - :content-type "application/pdf"}]}) - :error (= :SUCCESS) - (println " - email sent: ")))) - +(defn send-invoice [invoice to {from :user :as smtp}] + (when (and (not-any? nil? [to from smtp invoice]) + (->> + (send-message smtp {:from from + :to [to] + :subject (.getName invoice) + :body [{:type :attachment + :content (.getAbsolutePath invoice) + :content-type "application/pdf"}]}) + :error (= :SUCCESS))) + (println " - email sent to " to))) (defn server-find-messages "Find all messages in the given folder, filtering them by subject and sender (use nil to ignore)." @@ -62,11 +61,15 @@ (defn zip-item [headers cell] (into (sorted-map) (map vector headers cell))) +(defn parse-float [s] + (Float. (re-find #"[\d\.]+" s ))) + (defn extract-items [headers message] (->> message mess/get-content split-cells - (map (partial zip-item headers)))) + (map (partial zip-item headers)) + (map #(update % :worked parse-float)))) (defn get-worklogs "Get all worklogs for the given month from the given imap server." diff --git a/src/invoices/pdf.clj b/src/invoices/pdf.clj index ce1e219..25c27e8 100644 --- a/src/invoices/pdf.clj +++ b/src/invoices/pdf.clj @@ -30,14 +30,21 @@ [:table {:border false :padding 0 :spacing 0} [[:phrase {:style :bold} "Uwagi:"]]] (map vector notes))])) +;;; Title helpers +(def month-names + ["" "styczen" "luty" "marzec" "kwiecien" "maj" "czerwiec" "lipiec" "sierpien" "wrzesien" "pazdziernik" "listopad" "grudzien"]) -(defn get-title [team who which] - (let [[nr month year] (-> which (str/split #"/")) - months ["" "styczen" "luty" "marzec" "kwiecien" "maj" "czerwiec" "lipiec" - "sierpien" "wrzesien" "pazdziernik" "listopad" "grudzien"]] - (-> (str/join "_" (remove nil? [team who (nth months (Integer/parseInt month)) which])) - (str/replace #"[ -/]" "_")))) +(defn month-name [month] + (->> (str/split month #"/") second (Integer/parseInt) (nth month-names))) +(defn title-base [{{team :team who :name} :seller title :title}] + (if title title (->> [team who] (remove nil?) (str/join "_")))) + +(defn get-title [item which] + (->> [(title-base item) (month-name which) which] + (map #(str/replace % #"[ -/]" "_")) + (str/join "_"))) +;;; (defn pdf-body "Generate the actual pdf body" @@ -90,11 +97,11 @@ (->> items (map :notes) (remove nil?) flatten distinct format-notes (conj body))) -(defn render [seller buyer items when number & [font-path]] - (let [title (get-title (:team seller) (:name seller) number)] - (println " -" title) +(defn render [{seller :seller buyer :buyer items :items font-path :font-path :as invoice} when number] + (let [title (get-title invoice number)] + (println " - rendering" title) (-> title (pdf-body seller buyer items when number (clojure.core/when font-path {:encoding :unicode :ttf-name font-path})) (add-notes items) (pdf (str title ".pdf"))) - title)) + (-> title (str ".pdf") java.io.File.))) diff --git a/src/invoices/settings.clj b/src/invoices/settings.clj index 55f82cb..268dbf1 100644 --- a/src/invoices/settings.clj +++ b/src/invoices/settings.clj @@ -1,9 +1,55 @@ (ns invoices.settings - (:require [clojure.edn :as edn])) + (:require [clojure.edn :as edn] + [clojure.set :as set] + [invoices.time :refer [date-applies?]])) (defn load-config "Given a filename, load & return a config file" [filename] - (edn/read-string (slurp filename))) + (-> filename slurp edn/read-string)) -(defn invoices [config] (load-config config)) +(defn filter-invoice-nips + "Remove all invoices that don't apply to the given list of seller NIPs." + [{invoices :invoices :as config} nips] + (if (seq nips) + (->> invoices + (filter #(some #{(-> % :buyer :nip str)} nips)) + (assoc config :invoices)) + config)) + +(defn filter-unused-items + "Remove all items from the given invoice that don't match the date." + [{items :items :as invoice} date] + (->> items + (filter (partial date-applies? date)) + (assoc invoice :items))) + +(defn filter-invoice-dates + "Remove all items that don't apply to the given date, and also any with no items." + [{invoices :invoices :as config} date] + (->> invoices + (map #(filter-unused-items % date)) + (filter (comp seq :items)) + (assoc config :invoices))) + +(defn used-worklogs + "Return all worklog ids used by the given invoices." + [invoices] + (->> invoices + (map #(->> % :items (map :worklog))) + flatten distinct set)) + +(defn filter-worklogs + "Remove any worklogs that aren't used by the given invoices." + [{invoices :invoices worklogs :worklogs :as config}] + (->> worklogs + (filter (comp seq #(set/intersection % (used-worklogs invoices)) set :ids)) + (assoc config :worklogs))) + +(defn invoices + "Get all invoices that apply to the given month and (optional NIPs)." + [config month & [nips]] + (-> config load-config + (filter-invoice-nips nips) + (filter-invoice-dates month) + filter-worklogs)) diff --git a/src/invoices/timesheets.clj b/src/invoices/timesheets.clj index 5cf9618..a42ec22 100644 --- a/src/invoices/timesheets.clj +++ b/src/invoices/timesheets.clj @@ -1,5 +1,6 @@ (ns invoices.timesheets (:require [clj-http.client :as client] + [invoices.email :as email] [invoices.time :refer [last-day prev-month]])) (defn tempo [{tempo-token :tempo-token} endpoint params] @@ -17,13 +18,29 @@ (defn timesheet [{spent :timeSpentSeconds required :requiredSeconds period :period}] (merge {:worked (/ spent 3600) :required (/ required 3600)} period)) -(defn get-timesheet [who when credentials] - (->> {"from" (-> when (.withDayOfMonth 1) .toString) "to" (-> when last-day .toString)} - (tempo credentials (str "/timesheet-approvals/user/" who)) - (timesheet))) +(defn jira-timesheet [who when credentials] + (let [log (->> {"from" (-> when (.withDayOfMonth 1) .toString) "to" (-> when last-day .toString)} + (tempo credentials (str "/timesheet-approvals/user/" who)) + (timesheet))] + (map (partial assoc log :id) (:ids credentials)))) (defn prev-timesheet "Get the timesheet for the previous month" [when credentials] (clojure.core/when (:jira-user credentials) - (get-timesheet (me credentials) (prev-month when) credentials))) + (jira-timesheet (me credentials) (prev-month when) credentials))) + + +(defn get-timesheet [month {type :type :as creds}] + (condp = type + :jira (jira-timesheet (me creds) month creds) + :imap (email/get-worklogs month creds))) + +(defn timesheets + "Return timesheets for the given month from the given worklogs." + [month worklogs] + (->> worklogs + (map (partial get-timesheet month)) + flatten + (map (fn [{id :id :as log}] [id log])) + (into {})))