luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[nop] Misc housekeeping

Browse source
This commit is contained in:
Peter Taoussanis 2023-10-17 16:41:44 +02:00
parent e2a44abf6f
commit 27b3ed958b
4 changed files with 33 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
src/taoensso/nippy.clj
View file

@ -1,5 +1,5 @@
(ns taoensso.nippy (ns taoensso.nippy
"High-performance serialization library for Clojure" "High-performance serialization library for Clojure."
{:author "Peter Taoussanis (@ptaoussanis)"} {:author "Peter Taoussanis (@ptaoussanis)"}
(:require (:require
[clojure.string :as str] [clojure.string :as str]
@ -545,7 +545,7 @@
(enc/defonce ^{:dynamic true :doc doc} *thaw-serializable-allowlist* (impl/init-serializable-allowlist :thaw default-thaw-serializable-allowlist true))) (enc/defonce ^{:dynamic true :doc doc} *thaw-serializable-allowlist* (impl/init-serializable-allowlist :thaw default-thaw-serializable-allowlist true)))
(enc/defonce ^:dynamic ^:no-doc ^:deprecated *serializable-whitelist* (enc/defonce ^:dynamic ^:no-doc ^:deprecated *serializable-whitelist*
;; Retained for https://github.com/juxt/crux/releases/tag/20.09-1.11.0 ;; Back compatibility for Crux, Ref. <https://github.com/juxt/crux/releases/tag/20.09-1.11.0>
"Prefer `*thaw-serializable-allowlist*`." nil) "Prefer `*thaw-serializable-allowlist*`." nil)
(enc/defaliases (enc/defaliases
@ -666,7 +666,7 @@
(sm-count? len) (do (write-id out id-kw-sm) (write-sm-count out len)) (sm-count? len) (do (write-id out id-kw-sm) (write-sm-count out len))
(md-count? len) (do (write-id out id-kw-md) (write-md-count out len)) (md-count? len) (do (write-id out id-kw-md) (write-md-count out len))
;; :else (do (write-id out id-kw-lg) (write-lg-count out len)) ; Unrealistic ;; :else (do (write-id out id-kw-lg) (write-lg-count out len)) ; Unrealistic
:else (throw (ex-info "Keyword too long" {:full-name s}))) :else (throw (ex-info "Keyword too long" {:name s})))
(.write out ba 0 len))) (.write out ba 0 len)))
@ -678,7 +678,7 @@
(sm-count? len) (do (write-id out id-sym-sm) (write-sm-count out len)) (sm-count? len) (do (write-id out id-sym-sm) (write-sm-count out len))
(md-count? len) (do (write-id out id-sym-md) (write-md-count out len)) (md-count? len) (do (write-id out id-sym-md) (write-md-count out len))
;; :else (do (write-id out id-sym-lg) (write-lg-count out len)) ; Unrealistic ;; :else (do (write-id out id-sym-lg) (write-lg-count out len)) ; Unrealistic
:else (throw (ex-info "Symbol too long" {:full-name s}))) :else (throw (ex-info "Symbol too long" {:name s})))
(.write out ba 0 len))) (.write out ba 0 len)))
@ -854,10 +854,7 @@
(sm-count? len) (do (write-id out id-sz-quarantined-sm) (write-bytes-sm out class-name-ba)) (sm-count? len) (do (write-id out id-sz-quarantined-sm) (write-bytes-sm out class-name-ba))
(md-count? len) (do (write-id out id-sz-quarantined-md) (write-bytes-md out class-name-ba)) (md-count? len) (do (write-id out id-sz-quarantined-md) (write-bytes-md out class-name-ba))
;; :else (do (write-id out id-sz-quarantined-lg) (write-bytes-md out class-name-ba)) ; Unrealistic ;; :else (do (write-id out id-sz-quarantined-lg) (write-bytes-md out class-name-ba)) ; Unrealistic
:else :else (throw (ex-info "Serializable class name too long" {:name class-name})))
(throw
(ex-info "Serializable class name too long"
{:class-name class-name})))
;; Legacy: write object directly to out. ;; Legacy: write object directly to out.
;; (.writeObject (ObjectOutputStream. out) x) ;; (.writeObject (ObjectOutputStream. out) x)
@ -1014,8 +1011,7 @@
(vswap! cache_ assoc idx x) (vswap! cache_ assoc idx x)
x) x)
v)) v))
(throw (ex-info "No cache_ established, can't thaw. See `with-cache`." (throw (ex-info "No cache_ established, can't thaw. See `with-cache`." {}))))))
{}))))))
(comment (comment
(thaw (freeze [(cache "foo") (cache "foo") (cache "foo")])) (thaw (freeze [(cache "foo") (cache "foo") (cache "foo")]))
@ -1027,7 +1023,7 @@
;;;; ;;;;
(id-freezer nil id-nil) (id-freezer nil id-nil)
(id-freezer (type '()) id-list-0) (id-freezer (type ()) id-list-0)
(id-freezer Character id-char (.writeChar out (int x))) (id-freezer Character id-char (.writeChar out (int x)))
(id-freezer Byte id-byte (.writeByte out x)) (id-freezer Byte id-byte (.writeByte out x))
(id-freezer Short id-short (.writeShort out x)) (id-freezer Short id-short (.writeShort out x))
@ -1087,10 +1083,7 @@
(sm-count? len) (do (write-id out id-record-sm) (write-bytes-sm out class-name-ba)) (sm-count? len) (do (write-id out id-record-sm) (write-bytes-sm out class-name-ba))
(md-count? len) (do (write-id out id-record-md) (write-bytes-md out class-name-ba)) (md-count? len) (do (write-id out id-record-md) (write-bytes-md out class-name-ba))
;; :else (do (write-id out id-record-lg) (write-bytes-md out class-name-ba)) ; Unrealistic ;; :else (do (write-id out id-record-lg) (write-bytes-md out class-name-ba)) ; Unrealistic
:else :else (throw (ex-info "Record class name too long" {:name class-name})))
(throw
(ex-info "Record class name too long"
{:class-name class-name})))
(-freeze-without-meta! (into {} x) out))) (-freeze-without-meta! (into {} x) out)))
@ -1161,8 +1154,9 @@
(defn- wrap-header [data-ba head-meta] (defn- wrap-header [data-ba head-meta]
(if-let [head-ba (get-head-ba head-meta)] (if-let [head-ba (get-head-ba head-meta)]
(enc/ba-concat head-ba data-ba) (enc/ba-concat head-ba data-ba)
(throw (ex-info (str "Unrecognized header meta: " head-meta) (throw
{:head-meta head-meta})))) (ex-info (str "Unrecognized header meta: " head-meta)
{:head-meta head-meta}))))
(comment (wrap-header (.getBytes "foo") {:compressor-id :lz4 (comment (wrap-header (.getBytes "foo") {:compressor-id :lz4
:encryptor-id nil})) :encryptor-id nil}))
@ -1182,6 +1176,7 @@
bindings bindings
(-> nil (-> nil
(opt->bindings :freeze-fallback #'*freeze-fallback*) (opt->bindings :freeze-fallback #'*freeze-fallback*)
(opt->bindings :final-freeze-fallback #'*final-freeze-fallback*)
(opt->bindings :auto-freeze-compressor #'*auto-freeze-compressor*) (opt->bindings :auto-freeze-compressor #'*auto-freeze-compressor*)
(opt->bindings :custom-readers #'*custom-readers*) (opt->bindings :custom-readers #'*custom-readers*)
(opt->bindings :incl-metadata? #'*incl-metadata?*) (opt->bindings :incl-metadata? #'*incl-metadata?*)
@ -1223,7 +1218,6 @@
(defn freeze (defn freeze
"Serializes arg (any Clojure data type) to a byte array. "Serializes arg (any Clojure data type) to a byte array.
To freeze custom types, extend the Clojure reader or see `extend-freeze`." To freeze custom types, extend the Clojure reader or see `extend-freeze`."
([x] (freeze x nil)) ([x] (freeze x nil))
([x {:as opts ([x {:as opts
:keys [compressor encryptor password serializable-allowlist incl-metadata?] :keys [compressor encryptor password serializable-allowlist incl-metadata?]
@ -1232,9 +1226,7 @@
(call-with-bindings :freeze opts (call-with-bindings :freeze opts
(fn [] (fn []
(let [;; Intentionally undocumented: (let [no-header? (or (get opts :no-header?) (get opts :skip-header?)) ; Undocumented
no-header? (or (get opts :no-header?)
(get opts :skip-header?))
encryptor (when password encryptor) encryptor (when password encryptor)
baos (ByteArrayOutputStream. 64) baos (ByteArrayOutputStream. 64)
dos (DataOutputStream. baos)] dos (DataOutputStream. baos)]
@ -1470,7 +1462,7 @@
nil nbasis) nil nbasis)
(let [ctors (.getConstructors aclass) (let [ctors (.getConstructors aclass)
^Constructor ctor (aget ctors 0) ; Impl. detail? Ref. https://goo.gl/XWmckR ^Constructor ctor (aget ctors 0) ; Impl. detail? Ref. <https://goo.gl/XWmckR>
] ]
(.newInstance ctor cvalues))) (.newInstance ctor cvalues)))
@ -1576,14 +1568,14 @@
id-map-md (read-kvs-into {} in (read-md-count in)) id-map-md (read-kvs-into {} in (read-md-count in))
id-map-lg (read-kvs-into {} in (read-lg-count in)) id-map-lg (read-kvs-into {} in (read-lg-count in))
id-queue-lg (read-into (PersistentQueue/EMPTY) in (read-lg-count in)) id-queue-lg (read-into PersistentQueue/EMPTY in (read-lg-count in))
id-sorted-set-lg (read-into (sorted-set) in (read-lg-count in)) id-sorted-set-lg (read-into (sorted-set) in (read-lg-count in))
id-sorted-map-lg (read-kvs-into (sorted-map) in (read-lg-count in)) id-sorted-map-lg (read-kvs-into (sorted-map) in (read-lg-count in))
id-list-0 '() id-list-0 ()
id-list-sm (into '() (rseq (read-into [] in (read-sm-count in)))) id-list-sm (into () (rseq (read-into [] in (read-sm-count in))))
id-list-md (into '() (rseq (read-into [] in (read-md-count in)))) id-list-md (into () (rseq (read-into [] in (read-md-count in))))
id-list-lg (into '() (rseq (read-into [] in (read-lg-count in)))) id-list-lg (into () (rseq (read-into [] in (read-lg-count in))))
id-seq-0 (lazy-seq nil) id-seq-0 (lazy-seq nil)
id-seq-sm (or (seq (read-into [] in (read-sm-count in))) (lazy-seq nil)) id-seq-sm (or (seq (read-into [] in (read-sm-count in))) (lazy-seq nil))
@ -1613,15 +1605,11 @@
id-float (.readFloat in) id-float (.readFloat in)
id-double-0 0.0 id-double-0 0.0
id-double (.readDouble in) id-double (.readDouble in)
id-bigdec (BigDecimal. ^BigInteger (read-biginteger in) (.readInt in))
id-ratio (clojure.lang.Ratio. id-bigdec (BigDecimal. ^BigInteger (read-biginteger in) (.readInt in))
(read-biginteger in) id-ratio (clojure.lang.Ratio. (read-biginteger in) (read-biginteger in))
(read-biginteger in))
id-map-entry (clojure.lang.MapEntry. id-map-entry (clojure.lang.MapEntry/create (thaw-from-in! in) (thaw-from-in! in))
(thaw-from-in! in)
(thaw-from-in! in))
id-util-date (java.util.Date. (.readLong in)) id-util-date (java.util.Date. (.readLong in))
id-sql-date (java.sql.Date. (.readLong in)) id-sql-date (java.sql.Date. (.readLong in))
@ -1738,7 +1726,6 @@
- :compressor nil - :compressor nil
- :encryptor nil - :encryptor nil
- :no-header? true" - :no-header? true"
[^bytes ba] [^bytes ba]
(let [dis (DataInputStream. (ByteArrayInputStream. ba))] (let [dis (DataInputStream. (ByteArrayInputStream. ba))]
(with-cache (thaw-from-in! dis)))) (with-cache (thaw-from-in! dis))))
@ -1755,7 +1742,6 @@
:v1-compatibility? - support data frozen by legacy versions of Nippy? :v1-compatibility? - support data frozen by legacy versions of Nippy?
:compressor - :auto (checks header, default) an ICompressor, or nil :compressor - :auto (checks header, default) an ICompressor, or nil
:encryptor - :auto (checks header, default), an IEncryptor, or nil" :encryptor - :auto (checks header, default), an IEncryptor, or nil"
([ba] (thaw ba nil)) ([ba] (thaw ba nil))
([^bytes ba ([^bytes ba
{:as opts {:as opts
@ -1785,6 +1771,7 @@
:bindings :bindings
(enc/assoc-some {} (enc/assoc-some {}
'*freeze-fallback* *freeze-fallback* '*freeze-fallback* *freeze-fallback*
'*final-freeze-fallback* *final-freeze-fallback*
'*auto-freeze-compressor* *auto-freeze-compressor* '*auto-freeze-compressor* *auto-freeze-compressor*
'*custom-readers* *custom-readers* '*custom-readers* *custom-readers*
'*incl-metadata?* *incl-metadata?* '*incl-metadata?* *incl-metadata?*
@ -1812,7 +1799,7 @@
(catch Exception e (ex-fn e))))) (catch Exception e (ex-fn e)))))
;; Hackish + can actually segfault JVM due to Snappy bug, ;; Hackish + can actually segfault JVM due to Snappy bug,
;; Ref. http://goo.gl/mh7Rpy - no better alternatives, unfortunately ;; Ref. <http://goo.gl/mh7Rpy> - no better alternatives, unfortunately
thaw-v1-data thaw-v1-data
(fn [data-ba ex-fn] (fn [data-ba ex-fn]
(thaw-data data-ba :snappy nil (thaw-data data-ba :snappy nil
@ -1884,7 +1871,7 @@
* Keyword - 2 byte overhead, keywords hashed to 16 bit id * Keyword - 2 byte overhead, keywords hashed to 16 bit id
* ℕ∈[1, 128] - 0 byte overhead * ℕ∈[1, 128] - 0 byte overhead
NB: be careful about extending to interfaces, Ref. http://goo.gl/6gGRlU. NB: be careful about extending to interfaces, Ref. <http://goo.gl/6gGRlU>.
(defrecord MyRec [data]) (defrecord MyRec [data])
(extend-freeze MyRec :foo/my-type [x data-output] ; Keyword id (extend-freeze MyRec :foo/my-type [x data-output] ; Keyword id

3
src/taoensso/nippy/compression.clj
View file

@ -67,7 +67,8 @@
(defn- airlift-decompress (defn- airlift-decompress
^bytes [^io.airlift.compress.Decompressor d ^bytes ba max-out-len] ^bytes [^io.airlift.compress.Decompressor d ^bytes ba max-out-len]
(if max-out-len (if max-out-len
(let [ba-max-out (byte-array (int max-out-len)) (let [max-out-len (int max-out-len)
ba-max-out (byte-array max-out-len)
out-len out-len
(.decompress d (.decompress d
ba 0 (alength ba) ba 0 (alength ba)

6
src/taoensso/nippy/crypto.clj
View file

@ -5,8 +5,8 @@
(:require [taoensso.encore :as enc])) (:require [taoensso.encore :as enc]))
;; Note that AES128 may be preferable to AES256 due to known attack ;; Note that AES128 may be preferable to AES256 due to known attack
;; vectors specific to AES256, Ref. https://goo.gl/qU4CCV ;; vectors specific to AES256, Ref. <https://goo.gl/qU4CCV>
;; or for a counter argument, Ref. https://goo.gl/9LA9Yb ;; or for a counter argument, Ref. <https://goo.gl/9LA9Yb>
;;;; Randomness ;;;; Randomness
@ -70,7 +70,7 @@
(get-key-spec ^javax.crypto.spec.SecretKeySpec [_ ba] "Returns a `javax.crypto.spec.SecretKeySpec`.") (get-key-spec ^javax.crypto.spec.SecretKeySpec [_ ba] "Returns a `javax.crypto.spec.SecretKeySpec`.")
(get-param-spec ^java.security.spec.AlgorithmParameterSpec [_ iv-ba] "Returns a `java.security.spec.AlgorithmParameters`.")) (get-param-spec ^java.security.spec.AlgorithmParameterSpec [_ iv-ba] "Returns a `java.security.spec.AlgorithmParameters`."))
;; Prefer GCM > CBC, Ref. https://goo.gl/jpZoj8 ;; Prefer GCM > CBC, Ref. <https://goo.gl/jpZoj8>
(def ^:private gcm-cipher* (enc/thread-local-proxy (javax.crypto.Cipher/getInstance "AES/GCM/NoPadding"))) (def ^:private gcm-cipher* (enc/thread-local-proxy (javax.crypto.Cipher/getInstance "AES/GCM/NoPadding")))
(def ^:private cbc-cipher* (enc/thread-local-proxy (javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding"))) (def ^:private cbc-cipher* (enc/thread-local-proxy (javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding")))

2
test/taoensso/nippy_tests.clj
View file

@ -78,7 +78,7 @@
(do {:password [:salted "pwd"]}))) (do {:password [:salted "pwd"]})))
"CBC auto-encryptor compatibility") "CBC auto-encryptor compatibility")
(testing "Signed long types" (testing "Unsigned long types"
(let [range-ushort+ (+ (long @#'nippy/range-ushort) 128) (let [range-ushort+ (+ (long @#'nippy/range-ushort) 128)
range-uint+ (+ (long @#'nippy/range-uint) 128)] range-uint+ (+ (long @#'nippy/range-uint) 128)]