diff --git a/.gitignore b/.gitignore index 3f8f601..8c2ffc1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,10 +10,11 @@ pom.xml* /target/ /checkouts/ /logs/ +/test/logs/ /.clj-kondo/.cache .idea/ *.iml /wiki/.git .shadow-cljs/ public/js/ -out/ \ No newline at end of file +out/ diff --git a/src/taoensso/telemere.cljc b/src/taoensso/telemere.cljc index abca0ed..6e507ec 100644 --- a/src/taoensso/telemere.cljc +++ b/src/taoensso/telemere.cljc @@ -34,10 +34,6 @@ (enc/assert-min-encore-version [3 98 0]) ;;;; TODO -;; - File handler (with rotating, rolling, etc.) -;; - Postal handler (or example)? -;; - Template / example handler? -;; ;; - Review, TODOs, missing docstrings ;; - Reading plan, wiki docs, explainer/demo video ;; @@ -373,8 +369,10 @@ ;;;; Handlers -(enc/defaliases handlers/console-handler - #?(:cljs handlers/raw-console-handler)) +(enc/defaliases + #?(:default handlers/console-handler) + #?(:cljs handlers/raw-console-handler) + #?(:clj handlers/file-handler)) (defonce ^:no-doc __add-default-handlers (do @@ -426,7 +424,8 @@ (ex-info "Ex2" {:b :B} (ex-info "Ex1" {:a :A}))}))] - #?(:cljs (let [hf (handlers/raw-console-handler)] (hf sig) (hf))) - (do (let [hf (handlers/console-handler)] (hf sig) (hf))))) + (do (let [hf (handlers/file-handler)] (hf sig) (hf))) + (do (let [hf (handlers/console-handler)] (hf sig) (hf))) + #?(:cljs (let [hf (handlers/raw-console-handler)] (hf sig) (hf))))) ;;;; diff --git a/src/taoensso/telemere/handlers.cljc b/src/taoensso/telemere/handlers.cljc index 42e3ede..5987b4a 100644 --- a/src/taoensso/telemere/handlers.cljc +++ b/src/taoensso/telemere/handlers.cljc @@ -1,15 +1,26 @@ (ns taoensso.telemere.handlers "Built-in Telemere handlers." (:require - [clojure.string :as str] [taoensso.encore :as enc :refer [have have?]] - [taoensso.telemere.utils :as utils])) + [taoensso.telemere.utils :as utils] + #?(:clj [taoensso.telemere.handlers.file-handler :as file-handler]))) (comment (require '[taoensso.telemere :as tel]) (remove-ns 'taoensso.telemere.handlers) (:api (enc/interns-overview))) +;;;; Console handlers + +(enc/def* help:signal-formatters + "Common signal formatters include: + (utils/format-signal-str->fn) {}) ; For human-readable string output (default) + (utils/format-signal->edn-fn) {}) ; For edn output + (utils/format-signal->json-fn {}) ; For JSON output + + See relevant docstrings for details." + "See docstring") + #?(:clj (defn console-handler "Experimental, subject to change. @@ -18,16 +29,11 @@ - Takes a Telemere signal. - Writes a formatted signal string to stream. - Stream (`java.io.Writer`): - Defaults to `*err*` if `utils/error-signal?` is true, and `*out*` otherwise. + Options: + `:format-signal-fn` - (fn [signal]) => output, see `help:signal-formatters` - Common formatting alternatives: - (utils/format-signal-str->fn) {}) ; For human-readable string output (default) - (utils/format-signal->edn-fn) {}) ; For edn output - (utils/format-signal->json-fn {}) ; For JSON output - etc. - - See each format builder for options, etc." + `:stream` - `java.io.writer` + Defaults to `*err*` if `utils/error-signal?` is true, and `*out*` otherwise." ([] (console-handler nil)) ([{:keys [format-signal-fn stream] @@ -54,13 +60,8 @@ - Takes a Telemere signal. - Writes a formatted signal string to JavaScript console. - Common formatting alternatives: - (utils/format-signal-str->fn) {}) ; For human-readable string output (default) - (utils/format-signal->edn-fn) {}) ; For edn output - (utils/format-signal->json-fn {}) ; For JSON output - etc. - - See each format builder for options, etc." + Options: + `:format-signal-fn` - (fn [signal]) => output, see `help:signal-formatters`" ([] (console-handler nil)) ([{:keys [format-signal-fn] @@ -126,3 +127,7 @@ (.call logger logger stack)) (.groupEnd js/console))))))))) + +;;;; File handler + +#?(:clj (enc/defalias file-handler/file-handler)) diff --git a/src/taoensso/telemere/handlers/file_handler.clj b/src/taoensso/telemere/handlers/file_handler.clj new file mode 100644 index 0000000..dc1946b --- /dev/null +++ b/src/taoensso/telemere/handlers/file_handler.clj @@ -0,0 +1,401 @@ +(ns ^:no-doc taoensso.telemere.handlers.file-handler + "Private ns, implementation detail." + (:require + [taoensso.encore :as enc :refer [have have?]] + [taoensso.telemere.utils :as utils])) + +(comment + (remove-ns 'taoensso.telemere.handlers.file-handler) + (:api (enc/interns-overview))) + +;;;; Implementation + +(defn gzip-file + "Compresses contents of `file-in` to `file-out` using gzip." + [file-in file-out] + (let [file-in (utils/as-file file-in) + file-out (utils/as-file file-out)] + + (with-open + [stream-in (java.io.FileInputStream. file-in) + stream-out (java.io.FileOutputStream. file-out) + gz-out (java.util.zip.GZIPOutputStream. stream-out 2048 false)] + + (let [read-buffer (byte-array 4096)] + (loop [] + (let [bytes-read (.read stream-in read-buffer)] + (when-not (== -1 bytes-read) + (.write gz-out read-buffer 0 bytes-read)))))) + + true)) + +(comment (gzip-file "foo.txt" "foo.txt.gz")) + +(defn get-file-name + "(main-path)(-YYYY-MM-DD(d/w/m))(.part)?(.gz)?" + ^String [main-path ?timestamp ?part gz?] + (str main-path + (when-let [ts ?timestamp] (str "-" ts)) + (when-let [p ?part] (str "." p (when gz? ".gz"))))) + +(comment (get-file-name "test/logs/app.log" nil nil true)) + +;; Timestamp handling, edy (long epoch day) as base type +(let [utc java.time.ZoneOffset/UTC + ^java.time.format.DateTimeFormatter dtf + (.withZone java.time.format.DateTimeFormatter/ISO_LOCAL_DATE + utc)] + + (let [cf (* 24 60 60 1000)] + (defn udt->edy ^long [^long udt] (quot udt cf)) + (defn edy->udt ^long [^long edy] (* edy cf))) + + (let [ta (java.time.temporal.TemporalAdjusters/previousOrSame java.time.DayOfWeek/MONDAY)] + (defn edy-week ^long [^long edy] (.toEpochDay (.with (java.time.LocalDate/ofEpochDay edy) ta)))) + + (let [ta (java.time.temporal.TemporalAdjusters/firstDayOfMonth)] + (defn edy-month ^long [^long edy] (.toEpochDay (.with (java.time.LocalDate/ofEpochDay edy) ta)))) + + (defn file-timestamp->edy ^long [^String timestamp] + (let [timestamp (subs timestamp 0 (dec (count timestamp)))] + (.toEpochDay (java.time.LocalDate/parse timestamp dtf)))) + + (defn file-last-modified->edy ^long [^java.io.File file] + (.toEpochDay (.toLocalDate (.atZone (java.time.Instant/ofEpochMilli (.lastModified file)) utc)))) + + (defn format-file-timestamp + ^String [interval ^long edy] + (case interval + :daily (str (.format dtf (java.time.LocalDate/ofEpochDay edy)) "d") + :weekly (str (.format dtf (java.time.LocalDate/ofEpochDay (edy-week edy))) "w") + :monthly (str (.format dtf (java.time.LocalDate/ofEpochDay (edy-month edy))) "m") + (enc/unexpected-arg! interval + {:context `file-timestamp + :param 'interval + :expected #{:daily :weekly :monthly}})))) + +(comment (file-timestamp->edy (format-file-timestamp :weekly (udt->edy (enc/now-udt*))))) + +(defn manage-test-files! + "Describes/creates/deletes files used for tests/debugging, etc." + [action] + (have? [:el #{:return :println :create :delete}] action) + (let [fnames_ (volatile! []) + action! + (fn [app timestamp part gz? timestamp main?] + (let [path (str "test/logs/app" app ".log") + fname (get-file-name path (when-not main? timestamp) part gz?) + file (utils/as-file fname)] + + (case action + :return nil + :println (println fname) + :delete (.delete file) + :create + (do + (utils/writeable-file! file) + (spit file fname) + (when timestamp + (.setLastModified file + (edy->udt (file-timestamp->edy timestamp)))))) + + (vswap! fnames_ conj fname)))] + + (doseq [{:keys [app gz? timestamps parts]} + [{:app 1} + {:app 2, :gz? true, :parts [1 2 3 4 5]} + {:app 3, :gz? false, :parts [1 2 3 4 5]} + + {:app 4, :gz? true, :parts [1 2 3 4 5]} + {:app 4, :gz? false, :parts [1 2 3 4 5]} + + {:app 5, :gz? true, :timestamps + ["2020-01-01d" "2020-01-02d" "2020-02-01d" "2020-02-02d" "2021-01-01d" + "2020-01-01w" "2020-02-01m"]} + + {:app 6, :gz? true, :parts [1 2 3 4 5], + :timestamps + ["2020-01-01d" "2020-01-02d" "2020-02-01d" "2020-02-02d" "2021-01-01d" + "2020-01-01w" "2020-02-01m"]}]] + + (action! app nil nil false (peek timestamps) :main) + + (doseq [timestamp (or timestamps [nil]) + part (or parts [nil])] + + (action! app timestamp part gz? timestamp (not :main)))) + + @fnames_)) + +(comment (manage-test-files! :create)) + +(defn scan-files + "Returns ?[{:keys [file edy part ...]}] for files in same dir as `main-path` that: + - Have the same `interval` type ∈ #{:daily :weekly :monthly nil} (=> ?timestamped). + - Have the given timestamp (e.g. \"2020-01-01d\", or nil for NO timestamp)." + [main-path interval timestamp sort?] + (have? [:el #{:daily :weekly :monthly nil}] interval) + (let [main-file (utils/as-file main-path) ; `logs/app.log` + main-dir (.getParentFile (.getAbsoluteFile main-file)) ; `.../logs` + + file-pattern ; Matches ?[_ timestamp part gz] + (let [main (str "\\Q" (.getName main-file) "\\E") + end "(\\.\\d+)?(\\.gz)?"] + + (if interval + (let [ts-suffix (case interval :daily "d" :weekly "w" :monthly "m")] + (re-pattern (str main "-(\\d{4}-\\d{2}-\\d{2}" ts-suffix ")" end))) + (re-pattern (str main "(__no-timestamp__)?" end)))) + + ref-timestamp timestamp + any-timestamp? (and interval (nil? ref-timestamp))] + + (when-let [file-maps + (not-empty + (reduce + (fn [acc ^java.io.File file-in] + (or + (when-let [[_ timestamp part gz] (re-matches file-pattern (.getName file-in))] + (when (or any-timestamp? (= timestamp ref-timestamp)) + (let [edy (when timestamp (file-timestamp->edy timestamp)) + part (when part (enc/as-pos-int (subs part 1))) + gz? (boolean gz) + file-name (get-file-name main-path timestamp part gz?)] + + ;; Verify that scanned file name matches our template + (let [actual (.getAbsolutePath file-in) + expected file-name] + (when-not (.endsWith actual expected) + (throw + (ex-info "Unexpected file name" + {:actual actual, :expected expected})))) + + (conj acc + {:file file-in + :file-name file-name + :timestamp timestamp + :edy edy + :part part + :gz? gz?})))) + acc)) + [] (.listFiles main-dir)))] + + (if sort? ; For unit tests, etc. + (sort-by (fn [{:keys [edy part]}] [edy part]) file-maps) + (do file-maps))))) + +(comment (group-by :edy (scan-files "logs/app.log" nil nil false))) +(comment + (mapv #(select-keys % [:full-name :edy :part :gz?]) + (scan-files "test/logs/app6.log" :daily nil :sort))) + +;; Debugger used to test/debug file ops +(defn debugger [] (let [log_ (volatile! [])] (fn ([ ] @log_) ([x] (vswap! log_ conj x))))) + +(defn archive-main-file! + "Renames main -> .1.gz archive. Makes room by first rotating + pre-existing parts (n->n+1) and maintaining `max-num-parts` limit. + Expensive. Must manually reset any main file streams after!" + [main-path interval timestamp max-num-parts gz? ?debugger] + + ;; Rename n->n+1, deleting when n+1>max + (when-let [file-maps (scan-files main-path interval timestamp false)] ; [ ...] + (let [file-maps-by-edy (group-by :edy file-maps)] ; { [ ...]} + (enc/run-kv! + (fn [edy file-maps] + (doseq [{:keys [^java.io.File file file-name timestamp part gz?]} + (sort-by :part enc/rcompare file-maps)] + + (when part + (let [part (long part) + part+ (inc part)] + + (if-let [drop? (and max-num-parts (> part+ (long max-num-parts)))] + (if-let [df ?debugger] + (df [:delete file-name]) + (.delete file)) + + (let [file-name+ (get-file-name main-path timestamp part+ gz?)] + (if-let [df ?debugger] + (df [:rename file-name file-name+]) + (.renameTo file (utils/as-file file-name+))))))))) + file-maps-by-edy))) + + ;; Rename main -> .1.gz archive + (let [arch-file-name-gz (get-file-name main-path timestamp 1 false) + arch-file-name+gz (get-file-name main-path timestamp 1 gz?)] + + (if-let [df ?debugger] + (df [:rename main-path arch-file-name+gz]) + (let [main-file (utils/as-file main-path) ; `logs/app.log` + arch-file-gz (utils/as-file arch-file-name-gz) ; `logs/app.log.1` or `logs/app.log-2020-01-01d.1` + arch-file+gz (utils/as-file arch-file-name+gz) ; `logs/app.log.1.gz` or `logs/app.log-2020-01-01d.1.gz` + ] + + (have? false? (.exists arch-file+gz)) ; No pre-existing `.1.gz` + (.renameTo main-file arch-file-gz) + (.createNewFile main-file) + + (when gz? + (gzip-file arch-file-gz arch-file+gz) + (.delete arch-file-gz)))))) + +(defn prune-archive-files! + "Scans files in same dir as `main-path`, and maintains `max-num-intervals` limit + by deleting ALL parts for oldest intervals. Expensive." + [main-path interval max-num-intervals ?debugger] + (when (and interval max-num-intervals) + (when-let [file-maps (scan-files main-path interval nil false)] ; [ ...] + (let [file-maps-by-edy (group-by :edy file-maps) ; { [ ...]} + n-prune (- (count file-maps-by-edy) (long max-num-intervals))] + + (when (pos? n-prune) ; Prune some (oldest) intervals + (doseq [old-edy (take n-prune (sort (keys file-maps-by-edy)))] + + ;; Delete every part of this interval + (doseq [{:keys [^java.io.File file file-name]} + (sort-by :part enc/rcompare + (get file-maps-by-edy old-edy))] + + (if-let [df ?debugger] + (df [:delete file-name]) + (.delete file))))))))) + +;;;; Handler + +(defn ^:public file-handler + "Experimental, subject to change. + + Returns a (fn handler [signal]) that: + - Takes a Telemere signal. + - Writes a formatted signal string to file. + + Signals will be appended to file specified by `path`. + Depending on options, archives may be maintained: + - `logs/app.log.n.gz` (for nil `:interval`, non-nil `:max-file-size`) + - `logs/app.log-YYYY-MM-DDd.n.gz` (for non-nil `:interval`) ; d=daily/w=weekly/m=monthly + + Example files with default options: + `/logs/telemere.log` ; Current file + `/logs/telemere.log-2020-01-01m.1.gz` ; Archive for Jan 2020, part 1 (newest entries) + ... + `/logs/telemere.log-2020-01-01m.8.gz` ; Archive for Jan 2020, part 8 (oldest entries) + + Options: + `:format-signal-fn`- (fn [signal]) => output, see `help:signal-formatters`. + `:path` - Path string of the target output file (default `logs/telemere.log`). + `:interval` - ∈ #{nil :daily :weekly :monthly} (default `:monthly`). + When non-nil, causes interval-based archives to be maintained. + + `:max-file-size` ∈ #{nil } (default 4MB) + When `path` file size > ~this many bytes, rotates old content to numbered archives. + + `:max-num-parts` ∈ #{nil } (default 8) + Maximum number of numbered archives to retain for any particular interval. + + `:max-num-intervals` ∈ #{nil } (default 6) + Maximum number of intervals (days/weeks/months) to retain." + + ([] (file-handler nil)) + ([{:keys + [format-signal-fn + path interval + max-file-size + max-num-parts + max-num-intervals + gzip-archives?] + + :or + {format-signal-fn (utils/format-signal->str-fn) + path "logs/telemere.log" ; Main path, we'll ALWAYS write to this exact file + interval :monthly + max-file-size (* 1024 1024 4) ; 4MB + max-num-parts 8 + max-num-intervals 6 + gzip-archives? true}}] + + (let [main-path path + main-file (utils/as-file main-path) + fw (utils/file-writer main-file true) + + >max-file-size? + (when max-file-size + (let [max-file-size (long max-file-size) + rl (enc/rate-limiter-once-per 2500)] + (fn [] (and (not (rl)) (> (.length main-file) max-file-size))))) + + prev-timestamp_ (enc/latom nil) ; Initially nil + curr-timestamp_ (enc/latom nil) ; Will be bootstrapped based on main file + + ;; Called on every write attempt, + ;; maintains `timestamp_`s and returns true iff timestamp changed. + new-interval!? + (when interval + (let [init-edy (let [n (file-last-modified->edy main-file)] (when (pos? n) n)) + curr-edy_ (enc/latom init-edy) + updated!? ; Returns ?[old new] on change + (fn [latom_ new] + (let [old (latom_)] + (when + (and + (not= old new) + (compare-and-set! latom_ old new)) + [old new])))] + + (when init-edy ; Don't bootstrap "1970-01-01d", etc. + (reset! curr-timestamp_ + (format-file-timestamp interval init-edy))) + + (fn new-interval!? [] + (let [curr-edy (udt->edy (System/currentTimeMillis))] + (when (updated!? curr-edy_ curr-edy) ; Day changed + (let [curr-timestamp (format-file-timestamp interval curr-edy)] + (when-let [[prev-timestamp _] (updated!? curr-timestamp_ curr-timestamp)] + ;; Timestamp changed (recall: interval may not be daily) + (reset! prev-timestamp_ prev-timestamp) + true))))))) + + lock (Object.)] + + (fn a-file-handler + ([] (locking lock (fw))) ; Close writer + ([signal] + (when-let [output (format-signal-fn signal)] + (let [output-str (str output utils/newline) + new-interval? (when interval (new-interval!?)) + >max-file-size? (when max-file-size (>max-file-size?)) + reset-stream? (or new-interval? >max-file-size?)] + + (locking lock + + (if new-interval? + (do + ;; Rename main -> .1.gz, etc. + (when-let [prev-timestamp (prev-timestamp_)] + (archive-main-file! main-path interval prev-timestamp + max-num-parts gzip-archives? nil)) + + (when max-num-intervals + (prune-archive-files! main-path interval + max-num-intervals nil))) + + (when >max-file-size? + ;; Rename main -> .1.gz, etc. + (archive-main-file! main-path interval (curr-timestamp_) + max-num-parts gzip-archives? nil))) + + (when reset-stream? (fw :writer/reset!)) + (do (fw output-str)))))))))) + +(comment + (manage-test-files! :create) + (.setLastModified (utils/as-file "test/logs/app6.log") + (enc/as-udt "1999-01-01T01:00:00.00Z")) + + (let [hf + (file-handler + {:path "test/logs/app6.log" + :max-num-intervals 2 + :max-num-parts 2})] + + (hf {:info :level :msg_ "hello"}) (hf))) diff --git a/src/taoensso/telemere/utils.cljc b/src/taoensso/telemere/utils.cljc index dc3b2ca..6fefa32 100644 --- a/src/taoensso/telemere/utils.cljc +++ b/src/taoensso/telemere/utils.cljc @@ -2,8 +2,9 @@ "Misc utils useful for Telemere handlers, middleware, etc." (:refer-clojure :exclude [newline]) (:require - [clojure.string :as str] - [taoensso.encore :as enc :refer [have have?]])) + [clojure.string :as str] + #?(:clj [clojure.java.io :as jio]) + [taoensso.encore :as enc :refer [have have?]])) (comment (require '[taoensso.telemere :as tel]) @@ -145,6 +146,100 @@ (enc/qb 1e6 ; 683 (minify-signal s)))) +;;;; Files + +#?(:clj (defn ^:no-doc as-file ^java.io.File [file] (jio/as-file file))) +#?(:clj + (defn ^:no-doc writeable-file! + "Private, don't use. + Returns writable `java.io.File`, or throws." + ^java.io.File [file] + (let [file (as-file file)] + (when-not (.exists file) + (when-let [parent (.getParentFile file)] (.mkdirs parent)) + (.createNewFile file)) + + (if (.canWrite file) + file + (throw + (ex-info "Unable to prepare writable `java.io.File`" + {:path (.getAbsolutePath file)})))))) + +#?(:clj + (defn ^:no-doc file-stream + "Private, don't use. + Returns a new `java.io.FileOutputStream` for given `java.io.File`, etc." + ^java.io.FileOutputStream [file append?] + (java.io.FileOutputStream. (as-file file) (boolean append?)))) + +#?(:clj + (defn file-writer + "Experimental, subject to change!! + + Opens the specified file and returns a stateful fn of 2 arities: + [content] => Writes given content to file, or no-ops if closed. + [] => Closes the writer. + + Thread safe. Automatically creates file and parent dirs as necessary. + Writers MUST ALWAYS be manually closed after use! + + Useful for handlers that write to files, etc." + [file append?] + (let [file (writeable-file! file) + stream_ (volatile! (file-stream file append?)) + open?_ (enc/latom true) + + close! + (fn [] + (when (compare-and-set! open?_ true false) + (when-let [^java.io.FileOutputStream stream (.deref stream_)] + (.close stream) + (vreset! stream_ nil) + true))) + + reset! + (fn [] + (close!) + (vreset! stream_ (file-stream file append?)) + (reset! open?_ true) + true) + + write-ba! + (fn [^bytes ba-content retrying?] + (when-let [^java.io.FileOutputStream stream (.deref stream_)] + (.write stream ba-content) + (.flush stream) + true)) + + file-exists! + (let [rl (enc/rate-limiter-once-per 250)] + (fn [] + (or (rl) (.exists file) + (throw (java.io.IOException. "File doesn't exist"))))) + + lock (Object.)] + + (fn file-writer + ([] (when (open?_) (locking lock (close!)))) + ([content-or-action] + (case content-or-action ; Undocumented + :writer/open? (open?_) + :writer/file file + :writer/stream (.deref stream_) + :writer/reset! (locking lock (reset!)) + (when (open?_) + (let [content content-or-action + ba (.getBytes (str content) java.nio.charset.StandardCharsets/UTF_8)] + (locking lock + (try + (file-exists!) + (write-ba! ba false) + (catch java.io.IOException _ + (reset!) + (write-ba! ba true)))))))))))) + +(comment (def fw1 (file-writer "test.txt" true)) (fw1 "x") (fw1)) + ;;;; Formatters (defn format-nsecs-fn diff --git a/test/taoensso/telemere_tests.cljc b/test/taoensso/telemere_tests.cljc index aa8e556..7cfe906 100644 --- a/test/taoensso/telemere_tests.cljc +++ b/test/taoensso/telemere_tests.cljc @@ -8,8 +8,10 @@ :refer [signal! with-signal with-signals] :rename {signal! sig!, with-signal with-sig, with-signals with-sigs}] - [taoensso.telemere.utils :as utils] - [taoensso.telemere.timbre-shim :as timbre] + [taoensso.telemere.utils :as utils] + [taoensso.telemere.timbre-shim :as timbre] + [taoensso.telemere.handlers :as handlers] + #?(:clj [taoensso.telemere.handlers.file-handler :as fh]) #?(:clj [taoensso.telemere.slf4j :as slf4j]) #?(:clj [clojure.tools.logging :as ctl]) #?(:clj [jsonista.core :as jsonista]))) @@ -621,6 +623,26 @@ (is (= (utils/error-signal? {:level :fatal}) true)) (is (= (utils/error-signal? {:error? true}) true))]) + #?(:clj + (testing "File writer" + (let [f (java.io.File/createTempFile "file-writer-test" ".txt") + fw (utils/file-writer f false)] + + [(is (true? (fw "1"))) + (is (true? (.delete f))) + (do (Thread/sleep 500) :sleep) ; Wait for `exists` cache to clear + (is (true? (fw "2"))) + (is (= (slurp f) "2")) + + (is (true? (.delete f))) + (is (true? (.createNewFile f))) ; Can break stream without triggering auto reset + + (is (fw :writer/reset!)) + (is (true? (fw "3"))) + (is (= (slurp f) "3")) + (is (true? (fw "3"))) + (is (true? (.delete f)))]))) + (testing "Formatters, etc." [(is (= (utils/error-in-signal->maps {:level :info, :error ex2}) {:level :info, :error [{:type ex-info-type, :msg "Ex2", :data {:k2 "v2"}} @@ -667,9 +689,110 @@ (is (enc/str-starts-with? ((utils/format-signal->str-fn) sig) "2024-06-09T21:15:20.170Z INFO EVENT"))))])]) -;;;; Handlers +;;;; File handler -;; TODO +#?(:clj + (deftest _file-names + [(is (= (fh/get-file-name "/logs/app.log" nil nil false) "/logs/app.log")) + (is (= (fh/get-file-name "/logs/app.log" nil nil true) "/logs/app.log")) + (is (= (fh/get-file-name "/logs/app.log" "ts" nil true) "/logs/app.log-ts")) + (is (= (fh/get-file-name "/logs/app.log" "ts" 1 false) "/logs/app.log-ts.1")) + (is (= (fh/get-file-name "/logs/app.log" "ts" 1 true) "/logs/app.log-ts.1.gz")) + (is (= (fh/get-file-name "/logs/app.log" nil 1 false) "/logs/app.log.1")) + (is (= (fh/get-file-name "/logs/app.log" nil 1 true) "/logs/app.log.1.gz"))])) + +#?(:clj + (deftest _file-timestamps + [(is (= (fh/format-file-timestamp :daily (fh/udt->edy udt0)) "2024-06-09d")) + (is (= (fh/format-file-timestamp :weekly (fh/udt->edy udt0)) "2024-06-03w")) + (is (= (fh/format-file-timestamp :monthly (fh/udt->edy udt0)) "2024-06-01m"))])) + +(comment (fh/manage-test-files! :create)) + +#?(:clj + (deftest _file-handling + [(is (boolean (fh/manage-test-files! :create))) + + (testing "`scan-files`" + ;; Just checking basic counts here, should be sufficient + [(is (= (count (fh/scan-files "test/logs/app1.log" nil nil :sort)) 1) "1 main, 0 parts") + (is (= (count (fh/scan-files "test/logs/app1.log" :daily nil :sort)) 0) "0 stamped") + (is (= (count (fh/scan-files "test/logs/app2.log" nil nil :sort)) 6) "1 main, 5 parts (+gz)") + (is (= (count (fh/scan-files "test/logs/app3.log" nil nil :sort)) 6) "1 main, 5 parts (-gz") + (is (= (count (fh/scan-files "test/logs/app4.log" nil nil :sort)) 11) "1 main, 5 parts (+gz) + 5 parts (-gz)") + (is (= (count (fh/scan-files "test/logs/app5.log" nil nil :sort)) 1) "1 main, 0 unstamped") + (is (= (count (fh/scan-files "test/logs/app5.log" :daily nil :sort)) 5) "5 stamped") + (is (= (count (fh/scan-files "test/logs/app6.log" nil nil :sort)) 1) "1 main, 0 unstamped") + (is (= (count (fh/scan-files "test/logs/app6.log" :daily nil :sort)) 25) "5 stamped * 5 parts") + (is (= (count (fh/scan-files "test/logs/app6.log" :weekly nil :sort)) 5) "5 stamped")]) + + (testing "`archive-main-file!`" + [(is (= (let [df (fh/debugger)] (fh/archive-main-file! "test/logs/app1.log" nil nil 2 :gz df) (df)) + [[:rename "test/logs/app1.log" "test/logs/app1.log.1.gz"]])) + + (is (= (let [df (fh/debugger)] (fh/archive-main-file! "test/logs/app2.log" nil nil 2 :gz df) (df)) + [[:delete "test/logs/app2.log.5.gz"] + [:delete "test/logs/app2.log.4.gz"] + [:delete "test/logs/app2.log.3.gz"] + [:delete "test/logs/app2.log.2.gz"] + [:rename "test/logs/app2.log.1.gz" "test/logs/app2.log.2.gz"] + [:rename "test/logs/app2.log" "test/logs/app2.log.1.gz"]])) + + (is (= (let [df (fh/debugger)] (fh/archive-main-file! "test/logs/app3.log" nil nil 2 :gz df) (df)) + [[:delete "test/logs/app3.log.5"] + [:delete "test/logs/app3.log.4"] + [:delete "test/logs/app3.log.3"] + [:delete "test/logs/app3.log.2"] + [:rename "test/logs/app3.log.1" "test/logs/app3.log.2"] + [:rename "test/logs/app3.log" "test/logs/app3.log.1.gz"]])) + + (is (= (let [df (fh/debugger)] (fh/archive-main-file! "test/logs/app6.log" :daily "2021-01-01d" 2 :gz df) (df)) + [[:delete "test/logs/app6.log-2021-01-01d.5.gz"] + [:delete "test/logs/app6.log-2021-01-01d.4.gz"] + [:delete "test/logs/app6.log-2021-01-01d.3.gz"] + [:delete "test/logs/app6.log-2021-01-01d.2.gz"] + [:rename "test/logs/app6.log-2021-01-01d.1.gz" "test/logs/app6.log-2021-01-01d.2.gz"] + [:rename "test/logs/app6.log" "test/logs/app6.log-2021-01-01d.1.gz"]]))]) + + (testing "`prune-archive-files!`" + [(is (= (let [df (fh/debugger)] (fh/prune-archive-files! "test/logs/app1.log" nil 2 df) (df)) [])) + (is (= (let [df (fh/debugger)] (fh/prune-archive-files! "test/logs/app2.log" nil 2 df) (df)) [])) + (is (= (let [df (fh/debugger)] (fh/prune-archive-files! "test/logs/app5.log" nil 2 df) (df)) [])) + (is (= (let [df (fh/debugger)] (fh/prune-archive-files! "test/logs/app5.log" :daily 2 df) (df)) + [[:delete "test/logs/app5.log-2020-01-01d"] + [:delete "test/logs/app5.log-2020-01-02d"] + [:delete "test/logs/app5.log-2020-02-01d"]])) + + (is (= (let [df (fh/debugger)] (fh/prune-archive-files! "test/logs/app6.log" :daily 2 df) (df)) + [[:delete "test/logs/app6.log-2020-01-01d.5.gz"] + [:delete "test/logs/app6.log-2020-01-01d.4.gz"] + [:delete "test/logs/app6.log-2020-01-01d.3.gz"] + [:delete "test/logs/app6.log-2020-01-01d.2.gz"] + [:delete "test/logs/app6.log-2020-01-01d.1.gz"] + + [:delete "test/logs/app6.log-2020-01-02d.5.gz"] + [:delete "test/logs/app6.log-2020-01-02d.4.gz"] + [:delete "test/logs/app6.log-2020-01-02d.3.gz"] + [:delete "test/logs/app6.log-2020-01-02d.2.gz"] + [:delete "test/logs/app6.log-2020-01-02d.1.gz"] + + [:delete "test/logs/app6.log-2020-02-01d.5.gz"] + [:delete "test/logs/app6.log-2020-02-01d.4.gz"] + [:delete "test/logs/app6.log-2020-02-01d.3.gz"] + [:delete "test/logs/app6.log-2020-02-01d.2.gz"] + [:delete "test/logs/app6.log-2020-02-01d.1.gz"]]) + + "Prune oldest 3 intervals, with 5 parts each")]) + + (is (boolean (fh/manage-test-files! :delete)))])) + +;;;; Other handlers + +(deftest _other-handlers + ;; For now just testing that basic construction succeeds + [#?(:default (is (fn? (handlers/console-handler)))) + #?(:cljs (is (fn? (handlers/raw-console-handler)))) + #?(:clj (is (fn? (handlers/file-handler))))]) ;;;;