mirror of
https://github.com/taoensso/telemere.git
synced 2025-12-17 01:51:10 +00:00
[new] Add archiving file handler
This commit is contained in:
Peter Taoussanis
parent
15577ab106
commit
21a02f286b
6 changed files with 657 additions and 33 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -10,6 +10,7 @@ pom.xml*
|
|||
/target/
|
||||
/checkouts/
|
||||
/logs/
|
||||
/test/logs/
|
||||
/.clj-kondo/.cache
|
||||
.idea/
|
||||
*.iml
|
||||
|
|
|
|||
|
|
@ -34,10 +34,6 @@
|
|||
(enc/assert-min-encore-version [3 98 0])
|
||||
|
||||
;;;; TODO
|
||||
;; - File handler (with rotating, rolling, etc.)
|
||||
;; - Postal handler (or example)?
|
||||
;; - Template / example handler?
|
||||
;;
|
||||
;; - Review, TODOs, missing docstrings
|
||||
;; - Reading plan, wiki docs, explainer/demo video
|
||||
;;
|
||||
|
|
@ -373,8 +369,10 @@
|
|||
|
||||
;;;; Handlers
|
||||
|
||||
(enc/defaliases handlers/console-handler
|
||||
#?(:cljs handlers/raw-console-handler))
|
||||
(enc/defaliases
|
||||
#?(:default handlers/console-handler)
|
||||
#?(:cljs handlers/raw-console-handler)
|
||||
#?(:clj handlers/file-handler))
|
||||
|
||||
(defonce ^:no-doc __add-default-handlers
|
||||
(do
|
||||
|
|
@ -426,7 +424,8 @@
|
|||
(ex-info "Ex2" {:b :B}
|
||||
(ex-info "Ex1" {:a :A}))}))]
|
||||
|
||||
#?(:cljs (let [hf (handlers/raw-console-handler)] (hf sig) (hf)))
|
||||
(do (let [hf (handlers/console-handler)] (hf sig) (hf)))))
|
||||
(do (let [hf (handlers/file-handler)] (hf sig) (hf)))
|
||||
(do (let [hf (handlers/console-handler)] (hf sig) (hf)))
|
||||
#?(:cljs (let [hf (handlers/raw-console-handler)] (hf sig) (hf)))))
|
||||
|
||||
;;;;
|
||||
|
|
|
|||
|
|
@ -1,15 +1,26 @@
|
|||
(ns taoensso.telemere.handlers
|
||||
"Built-in Telemere handlers."
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[taoensso.encore :as enc :refer [have have?]]
|
||||
[taoensso.telemere.utils :as utils]))
|
||||
[taoensso.telemere.utils :as utils]
|
||||
#?(:clj [taoensso.telemere.handlers.file-handler :as file-handler])))
|
||||
|
||||
(comment
|
||||
(require '[taoensso.telemere :as tel])
|
||||
(remove-ns 'taoensso.telemere.handlers)
|
||||
(:api (enc/interns-overview)))
|
||||
|
||||
;;;; Console handlers
|
||||
|
||||
(enc/def* help:signal-formatters
|
||||
"Common signal formatters include:
|
||||
(utils/format-signal-str->fn) {<opts>}) ; For human-readable string output (default)
|
||||
(utils/format-signal->edn-fn) {<opts>}) ; For edn output
|
||||
(utils/format-signal->json-fn {<opts>}) ; For JSON output
|
||||
|
||||
See relevant docstrings for details."
|
||||
"See docstring")
|
||||
|
||||
#?(:clj
|
||||
(defn console-handler
|
||||
"Experimental, subject to change.
|
||||
|
|
@ -18,16 +29,11 @@
|
|||
- Takes a Telemere signal.
|
||||
- Writes a formatted signal string to stream.
|
||||
|
||||
Stream (`java.io.Writer`):
|
||||
Defaults to `*err*` if `utils/error-signal?` is true, and `*out*` otherwise.
|
||||
Options:
|
||||
`:format-signal-fn` - (fn [signal]) => output, see `help:signal-formatters`
|
||||
|
||||
Common formatting alternatives:
|
||||
(utils/format-signal-str->fn) {<opts>}) ; For human-readable string output (default)
|
||||
(utils/format-signal->edn-fn) {<opts>}) ; For edn output
|
||||
(utils/format-signal->json-fn {<opts>}) ; For JSON output
|
||||
etc.
|
||||
|
||||
See each format builder for options, etc."
|
||||
`:stream` - `java.io.writer`
|
||||
Defaults to `*err*` if `utils/error-signal?` is true, and `*out*` otherwise."
|
||||
|
||||
([] (console-handler nil))
|
||||
([{:keys [format-signal-fn stream]
|
||||
|
|
@ -54,13 +60,8 @@
|
|||
- Takes a Telemere signal.
|
||||
- Writes a formatted signal string to JavaScript console.
|
||||
|
||||
Common formatting alternatives:
|
||||
(utils/format-signal-str->fn) {<opts>}) ; For human-readable string output (default)
|
||||
(utils/format-signal->edn-fn) {<opts>}) ; For edn output
|
||||
(utils/format-signal->json-fn {<opts>}) ; For JSON output
|
||||
etc.
|
||||
|
||||
See each format builder for options, etc."
|
||||
Options:
|
||||
`:format-signal-fn` - (fn [signal]) => output, see `help:signal-formatters`"
|
||||
|
||||
([] (console-handler nil))
|
||||
([{:keys [format-signal-fn]
|
||||
|
|
@ -126,3 +127,7 @@
|
|||
(.call logger logger stack))
|
||||
|
||||
(.groupEnd js/console)))))))))
|
||||
|
||||
;;;; File handler
|
||||
|
||||
#?(:clj (enc/defalias file-handler/file-handler))
|
||||
|
|
|
|||
401
src/taoensso/telemere/handlers/file_handler.clj
Normal file
401
src/taoensso/telemere/handlers/file_handler.clj
Normal file
|
|
@ -0,0 +1,401 @@
|
|||
(ns ^:no-doc taoensso.telemere.handlers.file-handler
|
||||
"Private ns, implementation detail."
|
||||
(:require
|
||||
[taoensso.encore :as enc :refer [have have?]]
|
||||
[taoensso.telemere.utils :as utils]))
|
||||
|
||||
(comment
|
||||
(remove-ns 'taoensso.telemere.handlers.file-handler)
|
||||
(:api (enc/interns-overview)))
|
||||
|
||||
;;;; Implementation
|
||||
|
||||
(defn gzip-file
|
||||
"Compresses contents of `file-in` to `file-out` using gzip."
|
||||
[file-in file-out]
|
||||
(let [file-in (utils/as-file file-in)
|
||||
file-out (utils/as-file file-out)]
|
||||
|
||||
(with-open
|
||||
[stream-in (java.io.FileInputStream. file-in)
|
||||
stream-out (java.io.FileOutputStream. file-out)
|
||||
gz-out (java.util.zip.GZIPOutputStream. stream-out 2048 false)]
|
||||
|
||||
(let [read-buffer (byte-array 4096)]
|
||||
(loop []
|
||||
(let [bytes-read (.read stream-in read-buffer)]
|
||||
(when-not (== -1 bytes-read)
|
||||
(.write gz-out read-buffer 0 bytes-read))))))
|
||||
|
||||
true))
|
||||
|
||||
(comment (gzip-file "foo.txt" "foo.txt.gz"))
|
||||
|
||||
(defn get-file-name
|
||||
"(main-path)(-YYYY-MM-DD(d/w/m))(.part)?(.gz)?"
|
||||
^String [main-path ?timestamp ?part gz?]
|
||||
(str main-path
|
||||
(when-let [ts ?timestamp] (str "-" ts))
|
||||
(when-let [p ?part] (str "." p (when gz? ".gz")))))
|
||||
|
||||
(comment (get-file-name "test/logs/app.log" nil nil true))
|
||||
|
||||
;; Timestamp handling, edy (long epoch day) as base type
|
||||
(let [utc java.time.ZoneOffset/UTC
|
||||
^java.time.format.DateTimeFormatter dtf
|
||||
(.withZone java.time.format.DateTimeFormatter/ISO_LOCAL_DATE
|
||||
utc)]
|
||||
|
||||
(let [cf (* 24 60 60 1000)]
|
||||
(defn udt->edy ^long [^long udt] (quot udt cf))
|
||||
(defn edy->udt ^long [^long edy] (* edy cf)))
|
||||
|
||||
(let [ta (java.time.temporal.TemporalAdjusters/previousOrSame java.time.DayOfWeek/MONDAY)]
|
||||
(defn edy-week ^long [^long edy] (.toEpochDay (.with (java.time.LocalDate/ofEpochDay edy) ta))))
|
||||
|
||||
(let [ta (java.time.temporal.TemporalAdjusters/firstDayOfMonth)]
|
||||
(defn edy-month ^long [^long edy] (.toEpochDay (.with (java.time.LocalDate/ofEpochDay edy) ta))))
|
||||
|
||||
(defn file-timestamp->edy ^long [^String timestamp]
|
||||
(let [timestamp (subs timestamp 0 (dec (count timestamp)))]
|
||||
(.toEpochDay (java.time.LocalDate/parse timestamp dtf))))
|
||||
|
||||
(defn file-last-modified->edy ^long [^java.io.File file]
|
||||
(.toEpochDay (.toLocalDate (.atZone (java.time.Instant/ofEpochMilli (.lastModified file)) utc))))
|
||||
|
||||
(defn format-file-timestamp
|
||||
^String [interval ^long edy]
|
||||
(case interval
|
||||
:daily (str (.format dtf (java.time.LocalDate/ofEpochDay edy)) "d")
|
||||
:weekly (str (.format dtf (java.time.LocalDate/ofEpochDay (edy-week edy))) "w")
|
||||
:monthly (str (.format dtf (java.time.LocalDate/ofEpochDay (edy-month edy))) "m")
|
||||
(enc/unexpected-arg! interval
|
||||
{:context `file-timestamp
|
||||
:param 'interval
|
||||
:expected #{:daily :weekly :monthly}}))))
|
||||
|
||||
(comment (file-timestamp->edy (format-file-timestamp :weekly (udt->edy (enc/now-udt*)))))
|
||||
|
||||
(defn manage-test-files!
|
||||
"Describes/creates/deletes files used for tests/debugging, etc."
|
||||
[action]
|
||||
(have? [:el #{:return :println :create :delete}] action)
|
||||
(let [fnames_ (volatile! [])
|
||||
action!
|
||||
(fn [app timestamp part gz? timestamp main?]
|
||||
(let [path (str "test/logs/app" app ".log")
|
||||
fname (get-file-name path (when-not main? timestamp) part gz?)
|
||||
file (utils/as-file fname)]
|
||||
|
||||
(case action
|
||||
:return nil
|
||||
:println (println fname)
|
||||
:delete (.delete file)
|
||||
:create
|
||||
(do
|
||||
(utils/writeable-file! file)
|
||||
(spit file fname)
|
||||
(when timestamp
|
||||
(.setLastModified file
|
||||
(edy->udt (file-timestamp->edy timestamp))))))
|
||||
|
||||
(vswap! fnames_ conj fname)))]
|
||||
|
||||
(doseq [{:keys [app gz? timestamps parts]}
|
||||
[{:app 1}
|
||||
{:app 2, :gz? true, :parts [1 2 3 4 5]}
|
||||
{:app 3, :gz? false, :parts [1 2 3 4 5]}
|
||||
|
||||
{:app 4, :gz? true, :parts [1 2 3 4 5]}
|
||||
{:app 4, :gz? false, :parts [1 2 3 4 5]}
|
||||
|
||||
{:app 5, :gz? true, :timestamps
|
||||
["2020-01-01d" "2020-01-02d" "2020-02-01d" "2020-02-02d" "2021-01-01d"
|
||||
"2020-01-01w" "2020-02-01m"]}
|
||||
|
||||
{:app 6, :gz? true, :parts [1 2 3 4 5],
|
||||
:timestamps
|
||||
["2020-01-01d" "2020-01-02d" "2020-02-01d" "2020-02-02d" "2021-01-01d"
|
||||
"2020-01-01w" "2020-02-01m"]}]]
|
||||
|
||||
(action! app nil nil false (peek timestamps) :main)
|
||||
|
||||
(doseq [timestamp (or timestamps [nil])
|
||||
part (or parts [nil])]
|
||||
|
||||
(action! app timestamp part gz? timestamp (not :main))))
|
||||
|
||||
@fnames_))
|
||||
|
||||
(comment (manage-test-files! :create))
|
||||
|
||||
(defn scan-files
|
||||
"Returns ?[{:keys [file edy part ...]}] for files in same dir as `main-path` that:
|
||||
- Have the same `interval` type ∈ #{:daily :weekly :monthly nil} (=> ?timestamped).
|
||||
- Have the given timestamp (e.g. \"2020-01-01d\", or nil for NO timestamp)."
|
||||
[main-path interval timestamp sort?]
|
||||
(have? [:el #{:daily :weekly :monthly nil}] interval)
|
||||
(let [main-file (utils/as-file main-path) ; `logs/app.log`
|
||||
main-dir (.getParentFile (.getAbsoluteFile main-file)) ; `.../logs`
|
||||
|
||||
file-pattern ; Matches ?[_ timestamp part gz]
|
||||
(let [main (str "\\Q" (.getName main-file) "\\E")
|
||||
end "(\\.\\d+)?(\\.gz)?"]
|
||||
|
||||
(if interval
|
||||
(let [ts-suffix (case interval :daily "d" :weekly "w" :monthly "m")]
|
||||
(re-pattern (str main "-(\\d{4}-\\d{2}-\\d{2}" ts-suffix ")" end)))
|
||||
(re-pattern (str main "(__no-timestamp__)?" end))))
|
||||
|
||||
ref-timestamp timestamp
|
||||
any-timestamp? (and interval (nil? ref-timestamp))]
|
||||
|
||||
(when-let [file-maps
|
||||
(not-empty
|
||||
(reduce
|
||||
(fn [acc ^java.io.File file-in]
|
||||
(or
|
||||
(when-let [[_ timestamp part gz] (re-matches file-pattern (.getName file-in))]
|
||||
(when (or any-timestamp? (= timestamp ref-timestamp))
|
||||
(let [edy (when timestamp (file-timestamp->edy timestamp))
|
||||
part (when part (enc/as-pos-int (subs part 1)))
|
||||
gz? (boolean gz)
|
||||
file-name (get-file-name main-path timestamp part gz?)]
|
||||
|
||||
;; Verify that scanned file name matches our template
|
||||
(let [actual (.getAbsolutePath file-in)
|
||||
expected file-name]
|
||||
(when-not (.endsWith actual expected)
|
||||
(throw
|
||||
(ex-info "Unexpected file name"
|
||||
{:actual actual, :expected expected}))))
|
||||
|
||||
(conj acc
|
||||
{:file file-in
|
||||
:file-name file-name
|
||||
:timestamp timestamp
|
||||
:edy edy
|
||||
:part part
|
||||
:gz? gz?}))))
|
||||
acc))
|
||||
[] (.listFiles main-dir)))]
|
||||
|
||||
(if sort? ; For unit tests, etc.
|
||||
(sort-by (fn [{:keys [edy part]}] [edy part]) file-maps)
|
||||
(do file-maps)))))
|
||||
|
||||
(comment (group-by :edy (scan-files "logs/app.log" nil nil false)))
|
||||
(comment
|
||||
(mapv #(select-keys % [:full-name :edy :part :gz?])
|
||||
(scan-files "test/logs/app6.log" :daily nil :sort)))
|
||||
|
||||
;; Debugger used to test/debug file ops
|
||||
(defn debugger [] (let [log_ (volatile! [])] (fn ([ ] @log_) ([x] (vswap! log_ conj x)))))
|
||||
|
||||
(defn archive-main-file!
|
||||
"Renames main -> <timestamp>.1.gz archive. Makes room by first rotating
|
||||
pre-existing parts (n->n+1) and maintaining `max-num-parts` limit.
|
||||
Expensive. Must manually reset any main file streams after!"
|
||||
[main-path interval timestamp max-num-parts gz? ?debugger]
|
||||
|
||||
;; Rename n->n+1, deleting when n+1>max
|
||||
(when-let [file-maps (scan-files main-path interval timestamp false)] ; [<file-map> ...]
|
||||
(let [file-maps-by-edy (group-by :edy file-maps)] ; {<edy> [<file-map> ...]}
|
||||
(enc/run-kv!
|
||||
(fn [edy file-maps]
|
||||
(doseq [{:keys [^java.io.File file file-name timestamp part gz?]}
|
||||
(sort-by :part enc/rcompare file-maps)]
|
||||
|
||||
(when part
|
||||
(let [part (long part)
|
||||
part+ (inc part)]
|
||||
|
||||
(if-let [drop? (and max-num-parts (> part+ (long max-num-parts)))]
|
||||
(if-let [df ?debugger]
|
||||
(df [:delete file-name])
|
||||
(.delete file))
|
||||
|
||||
(let [file-name+ (get-file-name main-path timestamp part+ gz?)]
|
||||
(if-let [df ?debugger]
|
||||
(df [:rename file-name file-name+])
|
||||
(.renameTo file (utils/as-file file-name+)))))))))
|
||||
file-maps-by-edy)))
|
||||
|
||||
;; Rename main -> <timestamp>.1.gz archive
|
||||
(let [arch-file-name-gz (get-file-name main-path timestamp 1 false)
|
||||
arch-file-name+gz (get-file-name main-path timestamp 1 gz?)]
|
||||
|
||||
(if-let [df ?debugger]
|
||||
(df [:rename main-path arch-file-name+gz])
|
||||
(let [main-file (utils/as-file main-path) ; `logs/app.log`
|
||||
arch-file-gz (utils/as-file arch-file-name-gz) ; `logs/app.log.1` or `logs/app.log-2020-01-01d.1`
|
||||
arch-file+gz (utils/as-file arch-file-name+gz) ; `logs/app.log.1.gz` or `logs/app.log-2020-01-01d.1.gz`
|
||||
]
|
||||
|
||||
(have? false? (.exists arch-file+gz)) ; No pre-existing `.1.gz`
|
||||
(.renameTo main-file arch-file-gz)
|
||||
(.createNewFile main-file)
|
||||
|
||||
(when gz?
|
||||
(gzip-file arch-file-gz arch-file+gz)
|
||||
(.delete arch-file-gz))))))
|
||||
|
||||
(defn prune-archive-files!
|
||||
"Scans files in same dir as `main-path`, and maintains `max-num-intervals` limit
|
||||
by deleting ALL parts for oldest intervals. Expensive."
|
||||
[main-path interval max-num-intervals ?debugger]
|
||||
(when (and interval max-num-intervals)
|
||||
(when-let [file-maps (scan-files main-path interval nil false)] ; [<file-map> ...]
|
||||
(let [file-maps-by-edy (group-by :edy file-maps) ; {<edy> [<file-map> ...]}
|
||||
n-prune (- (count file-maps-by-edy) (long max-num-intervals))]
|
||||
|
||||
(when (pos? n-prune) ; Prune some (oldest) intervals
|
||||
(doseq [old-edy (take n-prune (sort (keys file-maps-by-edy)))]
|
||||
|
||||
;; Delete every part of this interval
|
||||
(doseq [{:keys [^java.io.File file file-name]}
|
||||
(sort-by :part enc/rcompare
|
||||
(get file-maps-by-edy old-edy))]
|
||||
|
||||
(if-let [df ?debugger]
|
||||
(df [:delete file-name])
|
||||
(.delete file)))))))))
|
||||
|
||||
;;;; Handler
|
||||
|
||||
(defn ^:public file-handler
|
||||
"Experimental, subject to change.
|
||||
|
||||
Returns a (fn handler [signal]) that:
|
||||
- Takes a Telemere signal.
|
||||
- Writes a formatted signal string to file.
|
||||
|
||||
Signals will be appended to file specified by `path`.
|
||||
Depending on options, archives may be maintained:
|
||||
- `logs/app.log.n.gz` (for nil `:interval`, non-nil `:max-file-size`)
|
||||
- `logs/app.log-YYYY-MM-DDd.n.gz` (for non-nil `:interval`) ; d=daily/w=weekly/m=monthly
|
||||
|
||||
Example files with default options:
|
||||
`/logs/telemere.log` ; Current file
|
||||
`/logs/telemere.log-2020-01-01m.1.gz` ; Archive for Jan 2020, part 1 (newest entries)
|
||||
...
|
||||
`/logs/telemere.log-2020-01-01m.8.gz` ; Archive for Jan 2020, part 8 (oldest entries)
|
||||
|
||||
Options:
|
||||
`:format-signal-fn`- (fn [signal]) => output, see `help:signal-formatters`.
|
||||
`:path` - Path string of the target output file (default `logs/telemere.log`).
|
||||
`:interval` - ∈ #{nil :daily :weekly :monthly} (default `:monthly`).
|
||||
When non-nil, causes interval-based archives to be maintained.
|
||||
|
||||
`:max-file-size` ∈ #{nil <pos-int>} (default 4MB)
|
||||
When `path` file size > ~this many bytes, rotates old content to numbered archives.
|
||||
|
||||
`:max-num-parts` ∈ #{nil <pos-int>} (default 8)
|
||||
Maximum number of numbered archives to retain for any particular interval.
|
||||
|
||||
`:max-num-intervals` ∈ #{nil <pos-int>} (default 6)
|
||||
Maximum number of intervals (days/weeks/months) to retain."
|
||||
|
||||
([] (file-handler nil))
|
||||
([{:keys
|
||||
[format-signal-fn
|
||||
path interval
|
||||
max-file-size
|
||||
max-num-parts
|
||||
max-num-intervals
|
||||
gzip-archives?]
|
||||
|
||||
:or
|
||||
{format-signal-fn (utils/format-signal->str-fn)
|
||||
path "logs/telemere.log" ; Main path, we'll ALWAYS write to this exact file
|
||||
interval :monthly
|
||||
max-file-size (* 1024 1024 4) ; 4MB
|
||||
max-num-parts 8
|
||||
max-num-intervals 6
|
||||
gzip-archives? true}}]
|
||||
|
||||
(let [main-path path
|
||||
main-file (utils/as-file main-path)
|
||||
fw (utils/file-writer main-file true)
|
||||
|
||||
>max-file-size?
|
||||
(when max-file-size
|
||||
(let [max-file-size (long max-file-size)
|
||||
rl (enc/rate-limiter-once-per 2500)]
|
||||
(fn [] (and (not (rl)) (> (.length main-file) max-file-size)))))
|
||||
|
||||
prev-timestamp_ (enc/latom nil) ; Initially nil
|
||||
curr-timestamp_ (enc/latom nil) ; Will be bootstrapped based on main file
|
||||
|
||||
;; Called on every write attempt,
|
||||
;; maintains `timestamp_`s and returns true iff timestamp changed.
|
||||
new-interval!?
|
||||
(when interval
|
||||
(let [init-edy (let [n (file-last-modified->edy main-file)] (when (pos? n) n))
|
||||
curr-edy_ (enc/latom init-edy)
|
||||
updated!? ; Returns ?[old new] on change
|
||||
(fn [latom_ new]
|
||||
(let [old (latom_)]
|
||||
(when
|
||||
(and
|
||||
(not= old new)
|
||||
(compare-and-set! latom_ old new))
|
||||
[old new])))]
|
||||
|
||||
(when init-edy ; Don't bootstrap "1970-01-01d", etc.
|
||||
(reset! curr-timestamp_
|
||||
(format-file-timestamp interval init-edy)))
|
||||
|
||||
(fn new-interval!? []
|
||||
(let [curr-edy (udt->edy (System/currentTimeMillis))]
|
||||
(when (updated!? curr-edy_ curr-edy) ; Day changed
|
||||
(let [curr-timestamp (format-file-timestamp interval curr-edy)]
|
||||
(when-let [[prev-timestamp _] (updated!? curr-timestamp_ curr-timestamp)]
|
||||
;; Timestamp changed (recall: interval may not be daily)
|
||||
(reset! prev-timestamp_ prev-timestamp)
|
||||
true)))))))
|
||||
|
||||
lock (Object.)]
|
||||
|
||||
(fn a-file-handler
|
||||
([] (locking lock (fw))) ; Close writer
|
||||
([signal]
|
||||
(when-let [output (format-signal-fn signal)]
|
||||
(let [output-str (str output utils/newline)
|
||||
new-interval? (when interval (new-interval!?))
|
||||
>max-file-size? (when max-file-size (>max-file-size?))
|
||||
reset-stream? (or new-interval? >max-file-size?)]
|
||||
|
||||
(locking lock
|
||||
|
||||
(if new-interval?
|
||||
(do
|
||||
;; Rename main -> <prev-timestamp>.1.gz, etc.
|
||||
(when-let [prev-timestamp (prev-timestamp_)]
|
||||
(archive-main-file! main-path interval prev-timestamp
|
||||
max-num-parts gzip-archives? nil))
|
||||
|
||||
(when max-num-intervals
|
||||
(prune-archive-files! main-path interval
|
||||
max-num-intervals nil)))
|
||||
|
||||
(when >max-file-size?
|
||||
;; Rename main -> <curr-timestamp>.1.gz, etc.
|
||||
(archive-main-file! main-path interval (curr-timestamp_)
|
||||
max-num-parts gzip-archives? nil)))
|
||||
|
||||
(when reset-stream? (fw :writer/reset!))
|
||||
(do (fw output-str))))))))))
|
||||
|
||||
(comment
|
||||
(manage-test-files! :create)
|
||||
(.setLastModified (utils/as-file "test/logs/app6.log")
|
||||
(enc/as-udt "1999-01-01T01:00:00.00Z"))
|
||||
|
||||
(let [hf
|
||||
(file-handler
|
||||
{:path "test/logs/app6.log"
|
||||
:max-num-intervals 2
|
||||
:max-num-parts 2})]
|
||||
|
||||
(hf {:info :level :msg_ "hello"}) (hf)))
|
||||
|
|
@ -2,8 +2,9 @@
|
|||
"Misc utils useful for Telemere handlers, middleware, etc."
|
||||
(:refer-clojure :exclude [newline])
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[taoensso.encore :as enc :refer [have have?]]))
|
||||
[clojure.string :as str]
|
||||
#?(:clj [clojure.java.io :as jio])
|
||||
[taoensso.encore :as enc :refer [have have?]]))
|
||||
|
||||
(comment
|
||||
(require '[taoensso.telemere :as tel])
|
||||
|
|
@ -145,6 +146,100 @@
|
|||
(enc/qb 1e6 ; 683
|
||||
(minify-signal s))))
|
||||
|
||||
;;;; Files
|
||||
|
||||
#?(:clj (defn ^:no-doc as-file ^java.io.File [file] (jio/as-file file)))
|
||||
#?(:clj
|
||||
(defn ^:no-doc writeable-file!
|
||||
"Private, don't use.
|
||||
Returns writable `java.io.File`, or throws."
|
||||
^java.io.File [file]
|
||||
(let [file (as-file file)]
|
||||
(when-not (.exists file)
|
||||
(when-let [parent (.getParentFile file)] (.mkdirs parent))
|
||||
(.createNewFile file))
|
||||
|
||||
(if (.canWrite file)
|
||||
file
|
||||
(throw
|
||||
(ex-info "Unable to prepare writable `java.io.File`"
|
||||
{:path (.getAbsolutePath file)}))))))
|
||||
|
||||
#?(:clj
|
||||
(defn ^:no-doc file-stream
|
||||
"Private, don't use.
|
||||
Returns a new `java.io.FileOutputStream` for given `java.io.File`, etc."
|
||||
^java.io.FileOutputStream [file append?]
|
||||
(java.io.FileOutputStream. (as-file file) (boolean append?))))
|
||||
|
||||
#?(:clj
|
||||
(defn file-writer
|
||||
"Experimental, subject to change!!
|
||||
|
||||
Opens the specified file and returns a stateful fn of 2 arities:
|
||||
[content] => Writes given content to file, or no-ops if closed.
|
||||
[] => Closes the writer.
|
||||
|
||||
Thread safe. Automatically creates file and parent dirs as necessary.
|
||||
Writers MUST ALWAYS be manually closed after use!
|
||||
|
||||
Useful for handlers that write to files, etc."
|
||||
[file append?]
|
||||
(let [file (writeable-file! file)
|
||||
stream_ (volatile! (file-stream file append?))
|
||||
open?_ (enc/latom true)
|
||||
|
||||
close!
|
||||
(fn []
|
||||
(when (compare-and-set! open?_ true false)
|
||||
(when-let [^java.io.FileOutputStream stream (.deref stream_)]
|
||||
(.close stream)
|
||||
(vreset! stream_ nil)
|
||||
true)))
|
||||
|
||||
reset!
|
||||
(fn []
|
||||
(close!)
|
||||
(vreset! stream_ (file-stream file append?))
|
||||
(reset! open?_ true)
|
||||
true)
|
||||
|
||||
write-ba!
|
||||
(fn [^bytes ba-content retrying?]
|
||||
(when-let [^java.io.FileOutputStream stream (.deref stream_)]
|
||||
(.write stream ba-content)
|
||||
(.flush stream)
|
||||
true))
|
||||
|
||||
file-exists!
|
||||
(let [rl (enc/rate-limiter-once-per 250)]
|
||||
(fn []
|
||||
(or (rl) (.exists file)
|
||||
(throw (java.io.IOException. "File doesn't exist")))))
|
||||
|
||||
lock (Object.)]
|
||||
|
||||
(fn file-writer
|
||||
([] (when (open?_) (locking lock (close!))))
|
||||
([content-or-action]
|
||||
(case content-or-action ; Undocumented
|
||||
:writer/open? (open?_)
|
||||
:writer/file file
|
||||
:writer/stream (.deref stream_)
|
||||
:writer/reset! (locking lock (reset!))
|
||||
(when (open?_)
|
||||
(let [content content-or-action
|
||||
ba (.getBytes (str content) java.nio.charset.StandardCharsets/UTF_8)]
|
||||
(locking lock
|
||||
(try
|
||||
(file-exists!)
|
||||
(write-ba! ba false)
|
||||
(catch java.io.IOException _
|
||||
(reset!)
|
||||
(write-ba! ba true))))))))))))
|
||||
|
||||
(comment (def fw1 (file-writer "test.txt" true)) (fw1 "x") (fw1))
|
||||
|
||||
;;;; Formatters
|
||||
|
||||
(defn format-nsecs-fn
|
||||
|
|
|
|||
|
|
@ -8,8 +8,10 @@
|
|||
:refer [signal! with-signal with-signals]
|
||||
:rename {signal! sig!, with-signal with-sig, with-signals with-sigs}]
|
||||
|
||||
[taoensso.telemere.utils :as utils]
|
||||
[taoensso.telemere.timbre-shim :as timbre]
|
||||
[taoensso.telemere.utils :as utils]
|
||||
[taoensso.telemere.timbre-shim :as timbre]
|
||||
[taoensso.telemere.handlers :as handlers]
|
||||
#?(:clj [taoensso.telemere.handlers.file-handler :as fh])
|
||||
#?(:clj [taoensso.telemere.slf4j :as slf4j])
|
||||
#?(:clj [clojure.tools.logging :as ctl])
|
||||
#?(:clj [jsonista.core :as jsonista])))
|
||||
|
|
@ -621,6 +623,26 @@
|
|||
(is (= (utils/error-signal? {:level :fatal}) true))
|
||||
(is (= (utils/error-signal? {:error? true}) true))])
|
||||
|
||||
#?(:clj
|
||||
(testing "File writer"
|
||||
(let [f (java.io.File/createTempFile "file-writer-test" ".txt")
|
||||
fw (utils/file-writer f false)]
|
||||
|
||||
[(is (true? (fw "1")))
|
||||
(is (true? (.delete f)))
|
||||
(do (Thread/sleep 500) :sleep) ; Wait for `exists` cache to clear
|
||||
(is (true? (fw "2")))
|
||||
(is (= (slurp f) "2"))
|
||||
|
||||
(is (true? (.delete f)))
|
||||
(is (true? (.createNewFile f))) ; Can break stream without triggering auto reset
|
||||
|
||||
(is (fw :writer/reset!))
|
||||
(is (true? (fw "3")))
|
||||
(is (= (slurp f) "3"))
|
||||
(is (true? (fw "3")))
|
||||
(is (true? (.delete f)))])))
|
||||
|
||||
(testing "Formatters, etc."
|
||||
[(is (= (utils/error-in-signal->maps {:level :info, :error ex2})
|
||||
{:level :info, :error [{:type ex-info-type, :msg "Ex2", :data {:k2 "v2"}}
|
||||
|
|
@ -667,9 +689,110 @@
|
|||
(is (enc/str-starts-with? ((utils/format-signal->str-fn) sig)
|
||||
"2024-06-09T21:15:20.170Z INFO EVENT"))))])])
|
||||
|
||||
;;;; Handlers
|
||||
;;;; File handler
|
||||
|
||||
;; TODO
|
||||
#?(:clj
|
||||
(deftest _file-names
|
||||
[(is (= (fh/get-file-name "/logs/app.log" nil nil false) "/logs/app.log"))
|
||||
(is (= (fh/get-file-name "/logs/app.log" nil nil true) "/logs/app.log"))
|
||||
(is (= (fh/get-file-name "/logs/app.log" "ts" nil true) "/logs/app.log-ts"))
|
||||
(is (= (fh/get-file-name "/logs/app.log" "ts" 1 false) "/logs/app.log-ts.1"))
|
||||
(is (= (fh/get-file-name "/logs/app.log" "ts" 1 true) "/logs/app.log-ts.1.gz"))
|
||||
(is (= (fh/get-file-name "/logs/app.log" nil 1 false) "/logs/app.log.1"))
|
||||
(is (= (fh/get-file-name "/logs/app.log" nil 1 true) "/logs/app.log.1.gz"))]))
|
||||
|
||||
#?(:clj
|
||||
(deftest _file-timestamps
|
||||
[(is (= (fh/format-file-timestamp :daily (fh/udt->edy udt0)) "2024-06-09d"))
|
||||
(is (= (fh/format-file-timestamp :weekly (fh/udt->edy udt0)) "2024-06-03w"))
|
||||
(is (= (fh/format-file-timestamp :monthly (fh/udt->edy udt0)) "2024-06-01m"))]))
|
||||
|
||||
(comment (fh/manage-test-files! :create))
|
||||
|
||||
#?(:clj
|
||||
(deftest _file-handling
|
||||
[(is (boolean (fh/manage-test-files! :create)))
|
||||
|
||||
(testing "`scan-files`"
|
||||
;; Just checking basic counts here, should be sufficient
|
||||
[(is (= (count (fh/scan-files "test/logs/app1.log" nil nil :sort)) 1) "1 main, 0 parts")
|
||||
(is (= (count (fh/scan-files "test/logs/app1.log" :daily nil :sort)) 0) "0 stamped")
|
||||
(is (= (count (fh/scan-files "test/logs/app2.log" nil nil :sort)) 6) "1 main, 5 parts (+gz)")
|
||||
(is (= (count (fh/scan-files "test/logs/app3.log" nil nil :sort)) 6) "1 main, 5 parts (-gz")
|
||||
(is (= (count (fh/scan-files "test/logs/app4.log" nil nil :sort)) 11) "1 main, 5 parts (+gz) + 5 parts (-gz)")
|
||||
(is (= (count (fh/scan-files "test/logs/app5.log" nil nil :sort)) 1) "1 main, 0 unstamped")
|
||||
(is (= (count (fh/scan-files "test/logs/app5.log" :daily nil :sort)) 5) "5 stamped")
|
||||
(is (= (count (fh/scan-files "test/logs/app6.log" nil nil :sort)) 1) "1 main, 0 unstamped")
|
||||
(is (= (count (fh/scan-files "test/logs/app6.log" :daily nil :sort)) 25) "5 stamped * 5 parts")
|
||||
(is (= (count (fh/scan-files "test/logs/app6.log" :weekly nil :sort)) 5) "5 stamped")])
|
||||
|
||||
(testing "`archive-main-file!`"
|
||||
[(is (= (let [df (fh/debugger)] (fh/archive-main-file! "test/logs/app1.log" nil nil 2 :gz df) (df))
|
||||
[[:rename "test/logs/app1.log" "test/logs/app1.log.1.gz"]]))
|
||||
|
||||
(is (= (let [df (fh/debugger)] (fh/archive-main-file! "test/logs/app2.log" nil nil 2 :gz df) (df))
|
||||
[[:delete "test/logs/app2.log.5.gz"]
|
||||
[:delete "test/logs/app2.log.4.gz"]
|
||||
[:delete "test/logs/app2.log.3.gz"]
|
||||
[:delete "test/logs/app2.log.2.gz"]
|
||||
[:rename "test/logs/app2.log.1.gz" "test/logs/app2.log.2.gz"]
|
||||
[:rename "test/logs/app2.log" "test/logs/app2.log.1.gz"]]))
|
||||
|
||||
(is (= (let [df (fh/debugger)] (fh/archive-main-file! "test/logs/app3.log" nil nil 2 :gz df) (df))
|
||||
[[:delete "test/logs/app3.log.5"]
|
||||
[:delete "test/logs/app3.log.4"]
|
||||
[:delete "test/logs/app3.log.3"]
|
||||
[:delete "test/logs/app3.log.2"]
|
||||
[:rename "test/logs/app3.log.1" "test/logs/app3.log.2"]
|
||||
[:rename "test/logs/app3.log" "test/logs/app3.log.1.gz"]]))
|
||||
|
||||
(is (= (let [df (fh/debugger)] (fh/archive-main-file! "test/logs/app6.log" :daily "2021-01-01d" 2 :gz df) (df))
|
||||
[[:delete "test/logs/app6.log-2021-01-01d.5.gz"]
|
||||
[:delete "test/logs/app6.log-2021-01-01d.4.gz"]
|
||||
[:delete "test/logs/app6.log-2021-01-01d.3.gz"]
|
||||
[:delete "test/logs/app6.log-2021-01-01d.2.gz"]
|
||||
[:rename "test/logs/app6.log-2021-01-01d.1.gz" "test/logs/app6.log-2021-01-01d.2.gz"]
|
||||
[:rename "test/logs/app6.log" "test/logs/app6.log-2021-01-01d.1.gz"]]))])
|
||||
|
||||
(testing "`prune-archive-files!`"
|
||||
[(is (= (let [df (fh/debugger)] (fh/prune-archive-files! "test/logs/app1.log" nil 2 df) (df)) []))
|
||||
(is (= (let [df (fh/debugger)] (fh/prune-archive-files! "test/logs/app2.log" nil 2 df) (df)) []))
|
||||
(is (= (let [df (fh/debugger)] (fh/prune-archive-files! "test/logs/app5.log" nil 2 df) (df)) []))
|
||||
(is (= (let [df (fh/debugger)] (fh/prune-archive-files! "test/logs/app5.log" :daily 2 df) (df))
|
||||
[[:delete "test/logs/app5.log-2020-01-01d"]
|
||||
[:delete "test/logs/app5.log-2020-01-02d"]
|
||||
[:delete "test/logs/app5.log-2020-02-01d"]]))
|
||||
|
||||
(is (= (let [df (fh/debugger)] (fh/prune-archive-files! "test/logs/app6.log" :daily 2 df) (df))
|
||||
[[:delete "test/logs/app6.log-2020-01-01d.5.gz"]
|
||||
[:delete "test/logs/app6.log-2020-01-01d.4.gz"]
|
||||
[:delete "test/logs/app6.log-2020-01-01d.3.gz"]
|
||||
[:delete "test/logs/app6.log-2020-01-01d.2.gz"]
|
||||
[:delete "test/logs/app6.log-2020-01-01d.1.gz"]
|
||||
|
||||
[:delete "test/logs/app6.log-2020-01-02d.5.gz"]
|
||||
[:delete "test/logs/app6.log-2020-01-02d.4.gz"]
|
||||
[:delete "test/logs/app6.log-2020-01-02d.3.gz"]
|
||||
[:delete "test/logs/app6.log-2020-01-02d.2.gz"]
|
||||
[:delete "test/logs/app6.log-2020-01-02d.1.gz"]
|
||||
|
||||
[:delete "test/logs/app6.log-2020-02-01d.5.gz"]
|
||||
[:delete "test/logs/app6.log-2020-02-01d.4.gz"]
|
||||
[:delete "test/logs/app6.log-2020-02-01d.3.gz"]
|
||||
[:delete "test/logs/app6.log-2020-02-01d.2.gz"]
|
||||
[:delete "test/logs/app6.log-2020-02-01d.1.gz"]])
|
||||
|
||||
"Prune oldest 3 intervals, with 5 parts each")])
|
||||
|
||||
(is (boolean (fh/manage-test-files! :delete)))]))
|
||||
|
||||
;;;; Other handlers
|
||||
|
||||
(deftest _other-handlers
|
||||
;; For now just testing that basic construction succeeds
|
||||
[#?(:default (is (fn? (handlers/console-handler))))
|
||||
#?(:cljs (is (fn? (handlers/raw-console-handler))))
|
||||
#?(:clj (is (fn? (handlers/file-handler))))])
|
||||
|
||||
;;;;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue