From 3f43542adb545a5dd0fbe6286ddc0a04aaa30aef Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 16 Mar 2016 19:22:56 +0700 Subject: [PATCH] Tools housekeeping --- src/taoensso/nippy.clj | 2 + src/taoensso/nippy/benchmarks.clj | 1 - src/taoensso/nippy/compression.clj | 3 +- src/taoensso/nippy/encryption.clj | 3 +- src/taoensso/nippy/tools.clj | 55 +++++++++++-------- src/taoensso/nippy/utils.clj | 88 ++++++++++++++---------------- 6 files changed, 77 insertions(+), 75 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 551af72..b68caa4 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -343,6 +343,7 @@ (.writeLong out (.getMostSignificantBits x)) (.writeLong out (.getLeastSignificantBits x))) +;; TODO Deprecate, move to `thaw` opt? (enc/defonce* ^:dynamic *final-freeze-fallback* nil) (defn freeze-fallback-as-str [out x] (-freeze-to-out {:nippy/unfreezable (enc/pr-edn x) :type (type x)} out)) @@ -513,6 +514,7 @@ (def ^:private class-method-sig (into-array Class [IPersistentMap])) +;; TODO Deprecate, move to `thaw` opt? (enc/defonce* ^:dynamic *custom-readers* "{ (fn [data-input])}" nil) (defn swap-custom-readers! [f] (alter-var-root #'*custom-readers* f)) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 44a9538..89f86af 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -1,5 +1,4 @@ (ns taoensso.nippy.benchmarks - {:author "Peter Taoussanis"} (:require [clojure.data.fressian :as fressian] [taoensso.encore :as enc] [taoensso.nippy :as nippy :refer (freeze thaw)])) diff --git a/src/taoensso/nippy/compression.clj b/src/taoensso/nippy/compression.clj index 1fb8917..d0d3f6a 100644 --- a/src/taoensso/nippy/compression.clj +++ b/src/taoensso/nippy/compression.clj @@ -1,5 +1,4 @@ (ns taoensso.nippy.compression - {:author "Peter Taoussanis"} (:require [taoensso.encore :as enc]) (:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream DataOutputStream])) @@ -13,7 +12,7 @@ ;;;; Default implementations -(def standard-header-ids "These'll support :auto thaw." #{:snappy :lzma2 :lz4}) +(def standard-header-ids "These'll support :auto thaw" #{:snappy :lzma2 :lz4}) (deftype SnappyCompressor [] ICompressor diff --git a/src/taoensso/nippy/encryption.clj b/src/taoensso/nippy/encryption.clj index d8b72cb..f91d248 100644 --- a/src/taoensso/nippy/encryption.clj +++ b/src/taoensso/nippy/encryption.clj @@ -1,6 +1,5 @@ (ns taoensso.nippy.encryption - "Simple no-nonsense crypto with reasonable defaults." - {:author "Peter Taoussanis"} + "Simple no-nonsense crypto with reasonable defaults" (:require [taoensso.encore :as enc])) ;;;; Interface diff --git a/src/taoensso/nippy/tools.clj b/src/taoensso/nippy/tools.clj index ea1c84c..7f06ce3 100644 --- a/src/taoensso/nippy/tools.clj +++ b/src/taoensso/nippy/tools.clj @@ -1,33 +1,42 @@ (ns taoensso.nippy.tools - "Utilities for third-party tools that want to add fully-user-configurable - Nippy support. Used by Carmine and Faraday." - {:author "Peter Taoussanis"} + "Utils for 3rd-party tools that want to add user-configurable Nippy support. + Used by Carmine, Faraday, etc." (:require [taoensso.nippy :as nippy])) +(def ^:dynamic *freeze-opts* nil) +(def ^:dynamic *thaw-opts* nil) + +(defmacro with-freeze-opts [opts & body] `(binding [*freeze-opts* ~opts] ~@body)) +(defmacro with-thaw-opts [opts & body] `(binding [*thaw-opts* ~opts] ~@body)) + (defrecord WrappedForFreezing [value opts]) (defn wrapped-for-freezing? [x] (instance? WrappedForFreezing x)) (defn wrap-for-freezing - "Wraps arg (any freezable data type) so that (tools/freeze ) - will serialize the arg using given options." - [value & [opts]] (->WrappedForFreezing value opts)) + "Ensures that given arg (any freezable data type) is wrapped so that + (tools/freeze ) will serialize as + (nippy/freeze )." + ([x ] (wrap-for-freezing x nil)) + ([x opts] + (if (wrapped-for-freezing? x) + (if (= (:opts x) opts) + x + (WrappedForFreezing. (:value x) opts)) + (WrappedForFreezing. x opts)))) (defn freeze - "Like `nippy/freeze` but takes options from special argument wrapper when - present." - [x & [{:keys [default-opts]}]] - (if (wrapped-for-freezing? x) - (nippy/freeze (:value x) (or (:opts x) default-opts)) - (nippy/freeze x default-opts))) + "Like `nippy/freeze` but merges opts from *freeze-opts*, `wrap-for-freezing`." + ([x ] (freeze x nil)) + ([x default-opts] + (let [default-opts (or (:default-opts default-opts) default-opts)] ; Back compat + (if (wrapped-for-freezing? x) + (nippy/freeze (:value x) (merge default-opts *freeze-opts* (:opts x))) + (nippy/freeze x default-opts))))) -(comment (freeze (wrap-for-freezing "wrapped")) - (freeze "unwrapped")) +(defn thaw + "Like `nippy/thaw` but merges opts from `*thaw-opts*`." + ([ba ] (thaw ba nil)) + ([ba default-opts] + (let [default-opts (or (:default-opts default-opts) default-opts)] ; Back compat + (nippy/thaw ba (merge default-opts *thaw-opts*))))) -(def ^:dynamic *thaw-opts* nil) -(defmacro with-thaw-opts - "Evaluates body using given options for any automatic deserialization in - context." - [opts & body] `(binding [*thaw-opts* ~opts] ~@body)) - -(defn thaw "Like `nippy/thaw` but takes options from *thaw-opts* binding." - [ba & [{:keys [default-opts]}]] - (nippy/thaw ba (merge default-opts *thaw-opts*))) +(comment (thaw (freeze (wrap-for-freezing "wrapped")))) diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 7dfcc91..f67a603 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -1,5 +1,4 @@ (ns taoensso.nippy.utils - {:author "Peter Taoussanis"} (:require [clojure.string :as str] [taoensso.encore :as enc]) (:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable @@ -29,22 +28,17 @@ _ (.writeObject (ObjectOutputStream. bas) x) ba (.toByteArray bas) object (.readObject (ObjectInputStream. - (ByteArrayInputStream. ba)))] + (ByteArrayInputStream. ba)))] (cast class object) true))))) (def readable? (memoize-type-test (fn [x] (-> x enc/pr-edn enc/read-edn) true))) (comment - (serializable? "Hello world") - (serializable? (fn [])) - (readable? "Hello world") - (readable? (fn [])) - - (time (dotimes [_ 10000] (serializable? "Hello world"))) ; Cacheable - (time (dotimes [_ 10000] (serializable? (fn [])))) ; Uncacheable - (time (dotimes [_ 10000] (readable? "Hello world"))) ; Cacheable - (time (dotimes [_ 10000] (readable? (fn []))))) ; Uncacheable + (enc/qb 1000 (serializable? "Hello world")) ; Cacheable + (enc/qb 1000 (serializable? (fn []))) ; Uncacheable + (enc/qb 1000 (readable? "Hello world")) ; Cacheable + (enc/qb 1000 (readable? (fn [])))) ; Uncacheable ;;;; @@ -77,49 +71,49 @@ (comment (is-coll? (clojure.lang.PersistentVector$ChunkedSeq. [1 2 3] 0 0))) (defn freezable? - "Alpha - subject to change, MAY BE BUGGY! - Returns truthy value iff Nippy supports de/serialization of given argument. - Conservative with default options. + "Alpha - subject to change. + Returns truthy iff Nippy *appears* to support freezing the given argument. - `:allow-clojure-reader?` and `:allow-java-serializable?` options may be used - to also enable the relevant roundtrip fallback test(s). These tests are only - **moderately reliable** since they're cached by arg type and don't test for - pre/post serialization equality (there's no good general way of doing so)." - [x & [{:keys [allow-clojure-reader? allow-java-serializable?]}]] - (let [is? #(when (instance? % x) %)] - (if (is-coll? x) - (try - (when (every? freezable? x) (type x)) - (catch Exception _ false)) - (or - (is? clojure.lang.Keyword) - (is? java.lang.String) - (is? java.lang.Long) - (is? java.lang.Double) + `:allow-clojure-reader?` and `:allow-java-serializable?` options may be + used to enable the relevant roundtrip fallback test(s). These tests are + only **moderately reliable** since they're cached by arg type and don't + test for pre/post serialization value equality (there's no good general + way of doing so)." - (is? clojure.lang.BigInt) - (is? clojure.lang.Ratio) + ([x] (freezable? x nil)) + ([x {:keys [allow-clojure-reader? allow-java-serializable?]}] + (let [is? #(when (instance? % x) %)] + (if (is-coll? x) + (when (enc/revery? freezable? x) (type x)) + (or + (is? clojure.lang.Keyword) + (is? java.lang.String) + (is? java.lang.Long) + (is? java.lang.Double) - (is? java.lang.Boolean) - (is? java.lang.Integer) - (is? java.lang.Short) - (is? java.lang.Byte) - (is? java.lang.Character) - (is? java.math.BigInteger) - (is? java.math.BigDecimal) - (is? #=(java.lang.Class/forName "[B")) + (is? clojure.lang.BigInt) + (is? clojure.lang.Ratio) - (is? java.util.Date) - (is? java.util.UUID) + (is? java.lang.Boolean) + (is? java.lang.Integer) + (is? java.lang.Short) + (is? java.lang.Byte) + (is? java.lang.Character) + (is? java.math.BigInteger) + (is? java.math.BigDecimal) + (is? #=(java.lang.Class/forName "[B")) - (when (and allow-clojure-reader? (readable? x)) :clojure-reader) - (when (and allow-java-serializable? - ;; Reports as true but is unreliable: - (not (is? clojure.lang.Fn)) - (serializable? x)) :java-serializable))))) + (is? java.util.Date) + (is? java.util.UUID) + + (when (and allow-clojure-reader? (readable? x)) :clojure-reader) + (when (and allow-java-serializable? + ;; Reports as true but is unreliable: + (not (is? clojure.lang.Fn)) + (serializable? x)) :java-serializable)))))) (comment - (time (dotimes [_ 10000] (freezable? "hello"))) + (enc/qb 10000 (freezable? "hello")) (freezable? [:a :b]) (freezable? [:a (fn [x] (* x x))]) (freezable? (.getBytes "foo"))