v2.14.2 hotfix
This commit is contained in:
Peter Taoussanis
parent
640c6dbbb0
commit
ea93fee8e2
7 changed files with 261 additions and 517 deletions
25
project.clj
25
project.clj
|
|
@ -1,4 +1,4 @@
|
|||
(defproject com.taoensso/nippy "2.15.0"
|
||||
(defproject com.taoensso/nippy "2.14.2"
|
||||
:author "Peter Taoussanis <https://www.taoensso.com>"
|
||||
:description "High-performance serialization library for Clojure"
|
||||
:url "https://github.com/ptaoussanis/nippy"
|
||||
|
|
@ -14,11 +14,11 @@
|
|||
|
||||
:dependencies
|
||||
[[org.clojure/clojure "1.5.1"]
|
||||
[org.clojure/tools.reader "1.3.2"]
|
||||
[com.taoensso/encore "2.122.0"]
|
||||
[org.clojure/tools.reader "1.1.1"]
|
||||
[com.taoensso/encore "2.93.0"]
|
||||
[org.iq80.snappy/snappy "0.4"]
|
||||
[org.tukaani/xz "1.8"]
|
||||
[org.lz4/lz4-java "1.7.1"]]
|
||||
[org.tukaani/xz "1.6"]
|
||||
[net.jpountz.lz4/lz4 "1.3"]]
|
||||
|
||||
:profiles
|
||||
{;; :default [:base :system :user :provided :dev]
|
||||
|
|
@ -28,23 +28,22 @@
|
|||
:1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]}
|
||||
:1.8 {:dependencies [[org.clojure/clojure "1.8.0"]]}
|
||||
:1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]}
|
||||
:1.10 {:dependencies [[org.clojure/clojure "1.10.1"]]}
|
||||
:test {:jvm-opts ["-Xms1024m" "-Xmx2048m"]
|
||||
:dependencies [[org.clojure/test.check "1.1.0"]
|
||||
[org.clojure/data.fressian "1.0.0"]
|
||||
[org.xerial.snappy/snappy-java "1.1.7.6"]]}
|
||||
:dev [:1.10 :test :server-jvm
|
||||
:dependencies [[org.clojure/test.check "0.9.0"]
|
||||
[org.clojure/data.fressian "0.2.1"]
|
||||
[org.xerial.snappy/snappy-java "1.1.7.1"]]}
|
||||
:dev [:1.9 :test :server-jvm
|
||||
{:plugins
|
||||
[[lein-pprint "1.3.2"]
|
||||
[[lein-pprint "1.2.0"]
|
||||
[lein-ancient "0.6.15"]
|
||||
[lein-codox "0.10.7"]]}]}
|
||||
[lein-codox "0.10.3"]]}]}
|
||||
|
||||
:codox
|
||||
{:language :clojure
|
||||
:source-uri "https://github.com/ptaoussanis/nippy/blob/master/{filepath}#L{line}"}
|
||||
|
||||
:aliases
|
||||
{"test-all" ["with-profile" "+1.10:+1.9:+1.8:+1.7:+1.6:+1.5" "test"]
|
||||
{"test-all" ["with-profile" "+1.9:+1.8:+1.7:+1.6:+1.5" "test"]
|
||||
"deploy-lib" ["do" "deploy" "clojars," "install"]
|
||||
"start-dev" ["with-profile" "+dev" "repl" ":headless"]}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,19 +13,19 @@
|
|||
[java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream
|
||||
DataOutputStream Serializable ObjectOutputStream ObjectInputStream
|
||||
DataOutput DataInput]
|
||||
[java.lang.reflect Method Field Constructor]
|
||||
[java.net URI]
|
||||
[java.lang.reflect Method]
|
||||
;; [java.net URI] ; TODO
|
||||
[java.util Date UUID]
|
||||
[java.util.regex Pattern]
|
||||
[clojure.lang Keyword Symbol BigInt Ratio
|
||||
APersistentMap APersistentVector APersistentSet
|
||||
IPersistentMap ; IPersistentVector IPersistentSet IPersistentList
|
||||
PersistentQueue PersistentTreeMap PersistentTreeSet PersistentList
|
||||
LazySeq IRecord ISeq IType]))
|
||||
LazySeq IRecord ISeq]))
|
||||
|
||||
(if (vector? enc/encore-version)
|
||||
(enc/assert-min-encore-version [2 121 0])
|
||||
(enc/assert-min-encore-version 2.121))
|
||||
(enc/assert-min-encore-version [2 67 1])
|
||||
(enc/assert-min-encore-version 2.67))
|
||||
|
||||
(comment
|
||||
(set! *unchecked-math* :warn-on-boxed)
|
||||
|
|
@ -59,38 +59,25 @@
|
|||
(def ^:private ^:const head-version "Current Nippy header format version" 1)
|
||||
(def ^:private ^:const head-meta
|
||||
"Final byte of 4-byte Nippy header stores version-dependent metadata"
|
||||
|
||||
;; Currently
|
||||
;; - 5 compressors, #{nil :snappy :lz4 :lzma2 :else}
|
||||
;; - 4 encryptors, #{nil :aes128-cbc-sha512 :aes128-gcm-sha512 :else}
|
||||
|
||||
{(byte 0) {:version 1 :compressor-id nil :encryptor-id nil}
|
||||
(byte 2) {:version 1 :compressor-id nil :encryptor-id :aes128-cbc-sha512}
|
||||
(byte 14) {:version 1 :compressor-id nil :encryptor-id :aes128-gcm-sha512}
|
||||
(byte 4) {:version 1 :compressor-id nil :encryptor-id :else}
|
||||
|
||||
(byte 5) {:version 1 :compressor-id :else :encryptor-id nil}
|
||||
(byte 6) {:version 1 :compressor-id :else :encryptor-id :else}
|
||||
;;
|
||||
(byte 2) {:version 1 :compressor-id nil :encryptor-id :aes128-sha512}
|
||||
;;
|
||||
(byte 1) {:version 1 :compressor-id :snappy :encryptor-id nil}
|
||||
(byte 3) {:version 1 :compressor-id :snappy :encryptor-id :aes128-cbc-sha512}
|
||||
(byte 15) {:version 1 :compressor-id :snappy :encryptor-id :aes128-gcm-sha512}
|
||||
(byte 3) {:version 1 :compressor-id :snappy :encryptor-id :aes128-sha512}
|
||||
(byte 7) {:version 1 :compressor-id :snappy :encryptor-id :else}
|
||||
|
||||
;;
|
||||
;;; :lz4 used for both lz4 and lz4hc compressor (the two are compatible)
|
||||
(byte 8) {:version 1 :compressor-id :lz4 :encryptor-id nil}
|
||||
(byte 9) {:version 1 :compressor-id :lz4 :encryptor-id :aes128-cbc-sha512}
|
||||
(byte 16) {:version 1 :compressor-id :lz4 :encryptor-id :aes128-gcm-sha512}
|
||||
(byte 9) {:version 1 :compressor-id :lz4 :encryptor-id :aes128-sha512}
|
||||
(byte 10) {:version 1 :compressor-id :lz4 :encryptor-id :else}
|
||||
|
||||
;;
|
||||
(byte 11) {:version 1 :compressor-id :lzma2 :encryptor-id nil}
|
||||
(byte 12) {:version 1 :compressor-id :lzma2 :encryptor-id :aes128-cbc-sha512}
|
||||
(byte 17) {:version 1 :compressor-id :lzma2 :encryptor-id :aes128-gcm-sha512}
|
||||
(byte 13) {:version 1 :compressor-id :lzma2 :encryptor-id :else}
|
||||
|
||||
(byte 5) {:version 1 :compressor-id :else :encryptor-id nil}
|
||||
(byte 18) {:version 1 :compressor-id :else :encryptor-id :aes128-cbc-sha512}
|
||||
(byte 19) {:version 1 :compressor-id :else :encryptor-id :aes128-gcm-sha512}
|
||||
(byte 6) {:version 1 :compressor-id :else :encryptor-id :else}})
|
||||
|
||||
(comment (count (sort (keys head-meta))))
|
||||
(byte 12) {:version 1 :compressor-id :lzma2 :encryptor-id :aes128-sha512}
|
||||
(byte 13) {:version 1 :compressor-id :lzma2 :encryptor-id :else}})
|
||||
|
||||
(defmacro ^:private when-debug [& body] (when #_true false `(do ~@body)))
|
||||
|
||||
|
|
@ -119,7 +106,7 @@
|
|||
49 :record-md
|
||||
80 :record-lg ; Used only for back-compatible thawing
|
||||
|
||||
81 :type
|
||||
81 :type ; TODO Implement?
|
||||
|
||||
3 :nil
|
||||
8 :true
|
||||
|
|
@ -138,7 +125,7 @@
|
|||
57 :sym-lg
|
||||
|
||||
58 :regex
|
||||
71 :uri
|
||||
71 :uri ; TODO Implement?
|
||||
|
||||
53 :bytes-0
|
||||
7 :bytes-sm
|
||||
|
|
@ -152,8 +139,6 @@
|
|||
69 :vec-md
|
||||
21 :vec-lg
|
||||
|
||||
115 :objects-lg ; TODO Could include md, sm, 0 later if there's demand
|
||||
|
||||
18 :set-0
|
||||
111 :set-sm
|
||||
32 :set-md
|
||||
|
|
@ -259,21 +244,17 @@
|
|||
|
||||
(enc/defalias encrypt encryption/encrypt)
|
||||
(enc/defalias decrypt encryption/decrypt)
|
||||
|
||||
(enc/defalias aes128-gcm-encryptor encryption/aes128-gcm-encryptor)
|
||||
(enc/defalias aes128-cbc-encryptor encryption/aes128-cbc-encryptor)
|
||||
(enc/defalias aes128-encryptor encryption/aes128-gcm-encryptor) ; Default
|
||||
(enc/defalias aes128-encryptor encryption/aes128-encryptor)
|
||||
|
||||
(enc/defalias freezable? utils/freezable?))
|
||||
|
||||
;;;; Dynamic config
|
||||
;; See also `nippy.tools` ns for further dynamic config support
|
||||
|
||||
;; For back compatibility (nb Timbre's Carmine appender)
|
||||
(enc/defonce ^:dynamic *final-freeze-fallback* "DEPRECATED: prefer `*freeze-fallback`." nil)
|
||||
(enc/defonce ^:dynamic *freeze-fallback* "(fn [data-output x])->freeze, nil => default" nil)
|
||||
;; TODO Switch to thread-local proxies?
|
||||
|
||||
(enc/defonce ^:dynamic *custom-readers* "{<hash-or-byte-id> (fn [data-input])->read}" nil)
|
||||
(enc/defonce ^:dynamic *freeze-fallback* "(fn [data-output x]), nil => default" nil)
|
||||
(enc/defonce ^:dynamic *custom-readers* "{<hash-or-byte-id> (fn [data-input])}" nil)
|
||||
(enc/defonce ^:dynamic *auto-freeze-compressor*
|
||||
"(fn [byte-array])->compressor used by `(freeze <x> {:compressor :auto}),
|
||||
nil => default"
|
||||
|
|
@ -320,7 +301,12 @@
|
|||
source, you can use `(constantly true)` as predicate. This
|
||||
will whitelist everything, allowing Serializable for ANY class.
|
||||
|
||||
Default value as of v2.15.0 is: #{}.
|
||||
**** IMPORTANT *********************************************************
|
||||
To avoid a breaking change in an <x.y.patch> release, the default value
|
||||
in Nippy v2.14.1 is `(constantly true)`, which MAY BE UNSAFE!!!
|
||||
|
||||
PLEASE REVIEW AND ADJUST THIS SETTING AS NECESSARY FOR YOUR ENVIRONMENT!
|
||||
************************************************************************
|
||||
|
||||
PRs welcome for additional known-safe classes to be added to default
|
||||
whitelist.
|
||||
|
|
@ -338,7 +324,8 @@
|
|||
|
||||
[1] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ"
|
||||
|
||||
#{#_"java.lang.Throwable"})
|
||||
#_#{#_"java.lang.Throwable"}
|
||||
(constantly true))
|
||||
|
||||
(defn set-freeze-fallback! [x] (alter-var-root #'*freeze-fallback* (constantly x)))
|
||||
(defn set-auto-freeze-compressor! [x] (alter-var-root #'*auto-freeze-compressor* (constantly x)))
|
||||
|
|
@ -355,6 +342,8 @@
|
|||
See also `*serializable-whitelist*."
|
||||
[f] (alter-var-root #'*serializable-whitelist* f))
|
||||
|
||||
(declare ^:dynamic *final-freeze-fallback*) ; DEPRECATED
|
||||
|
||||
;;;; Freezing
|
||||
|
||||
#_(do
|
||||
|
|
@ -744,12 +733,6 @@
|
|||
|
||||
(-run! (fn [in] (-freeze-with-meta! in out)) s)))))
|
||||
|
||||
(defn- write-objects [^DataOutput out ^objects ary]
|
||||
(let [len (alength ary)]
|
||||
(write-id out id-objects-lg)
|
||||
(write-lg-count out len)
|
||||
(-run! (fn [in] (-freeze-with-meta! in out)) ary)))
|
||||
|
||||
(defn- write-serializable [^DataOutput out x ^String class-name]
|
||||
(when-debug (println (str "write-serializable: " (type x))))
|
||||
(let [class-name-ba (.getBytes class-name charset)
|
||||
|
|
@ -968,17 +951,12 @@
|
|||
(write-biginteger out (.denominator x)))
|
||||
|
||||
(id-freezer Date id-date (.writeLong out (.getTime x)))
|
||||
|
||||
(id-freezer URI id-uri
|
||||
(write-str out (.toString x)))
|
||||
|
||||
(id-freezer UUID id-uuid
|
||||
(.writeLong out (.getMostSignificantBits x))
|
||||
(.writeLong out (.getLeastSignificantBits x)))
|
||||
|
||||
(freezer Boolean (if x (write-id out id-true) (write-id out id-false)))
|
||||
(freezer (Class/forName "[B") (write-bytes out x))
|
||||
(freezer (Class/forName "[Ljava.lang.Object;") (write-objects out x))
|
||||
(freezer (Class/forName "[B") (write-bytes out x))
|
||||
(freezer String (write-str out x))
|
||||
(freezer Keyword (write-kw out x))
|
||||
(freezer Symbol (write-sym out x))
|
||||
|
|
@ -1015,20 +993,6 @@
|
|||
|
||||
(-freeze-without-meta! (into {} x) out)))
|
||||
|
||||
(freezer IType
|
||||
(let [aclass (class x)
|
||||
cname (.getName aclass)]
|
||||
(write-id out id-type)
|
||||
(write-str out cname)
|
||||
(let [basis-method (.getMethod aclass "getBasis" nil)
|
||||
basis (.invoke basis-method nil nil)]
|
||||
(-run!
|
||||
(fn [b]
|
||||
(let [^Field cfield (.getField aclass (name b))]
|
||||
(let [fvalue (.get cfield x)]
|
||||
(-freeze-without-meta! fvalue out))))
|
||||
basis))))
|
||||
|
||||
(freezer Object
|
||||
(when-debug (println (str "freeze-fallback: " (type x))))
|
||||
(if-let [ff *freeze-fallback*]
|
||||
|
|
@ -1044,7 +1008,10 @@
|
|||
(try-write-serializable out x)
|
||||
(try-write-readable out x)
|
||||
|
||||
(when-let [fff *final-freeze-fallback*] (fff out x) true) ; Deprecated
|
||||
;; For back compatibility (nb Timbre's Carmine appender)
|
||||
(when-let [fff *final-freeze-fallback*]
|
||||
(fff out x)
|
||||
true)
|
||||
|
||||
(throw-unfreezable x))))
|
||||
|
||||
|
|
@ -1080,98 +1047,65 @@
|
|||
(with-cache (-freeze-with-meta! x dos))
|
||||
(.toByteArray baos)))
|
||||
|
||||
(defn- call-with-bindings
|
||||
"Allow opts to override config bindings. Undocumented."
|
||||
[opts f]
|
||||
(let [opt->bindings
|
||||
(fn [bindings id var]
|
||||
(if-let [o (find opts id)]
|
||||
(assoc bindings var (val o))
|
||||
(do bindings)))
|
||||
|
||||
bindings
|
||||
(-> nil
|
||||
(opt->bindings :freeze-fallback #'*freeze-fallback*)
|
||||
(opt->bindings :auto-freeze-compressor #'*auto-freeze-compressor*)
|
||||
(opt->bindings :serializable-whitelist #'*serializable-whitelist*)
|
||||
(opt->bindings :custom-readers #'*custom-readers*))]
|
||||
|
||||
(if-not bindings
|
||||
(f) ; Common case
|
||||
(try
|
||||
(push-thread-bindings bindings)
|
||||
(f)
|
||||
(finally
|
||||
(pop-thread-bindings))))))
|
||||
|
||||
(comment
|
||||
(enc/qb 1e4
|
||||
(call-with-bindings {} (fn [] *freeze-fallback*))
|
||||
(call-with-bindings {:freeze-fallback "foo"} (fn [] *freeze-fallback*))))
|
||||
|
||||
(defn freeze
|
||||
"Serializes arg (any Clojure data type) to a byte array. To freeze custom
|
||||
types, extend the Clojure reader or see `extend-freeze`."
|
||||
([x] (freeze x nil))
|
||||
([x {:as opts
|
||||
:keys [compressor encryptor password]
|
||||
([x {:keys [compressor encryptor password]
|
||||
:or {compressor :auto
|
||||
encryptor aes128-gcm-encryptor}}]
|
||||
encryptor aes128-encryptor}
|
||||
:as opts}]
|
||||
(let [;; Intentionally undocumented:
|
||||
no-header? (or (get opts :no-header?)
|
||||
(get opts :skip-header?))
|
||||
encryptor (when password encryptor)
|
||||
baos (ByteArrayOutputStream. 64)
|
||||
dos (DataOutputStream. baos)]
|
||||
|
||||
(call-with-bindings opts
|
||||
(fn []
|
||||
(if (and (nil? compressor) (nil? encryptor))
|
||||
(do ; Optimized case
|
||||
(when-not no-header? ; Avoid `wrap-header`'s array copy:
|
||||
(let [head-ba (get-head-ba {:compressor-id nil :encryptor-id nil})]
|
||||
(.write dos head-ba 0 4)))
|
||||
(with-cache (-freeze-with-meta! x dos))
|
||||
(.toByteArray baos))
|
||||
|
||||
(let [;; Intentionally undocumented:
|
||||
no-header? (or (get opts :no-header?)
|
||||
(get opts :skip-header?))
|
||||
encryptor (when password encryptor)
|
||||
baos (ByteArrayOutputStream. 64)
|
||||
dos (DataOutputStream. baos)]
|
||||
(do
|
||||
(with-cache (-freeze-with-meta! x dos))
|
||||
(let [ba (.toByteArray baos)
|
||||
|
||||
(if (and (nil? compressor) (nil? encryptor))
|
||||
(do ; Optimized case
|
||||
(when-not no-header? ; Avoid `wrap-header`'s array copy:
|
||||
(let [head-ba (get-head-ba {:compressor-id nil :encryptor-id nil})]
|
||||
(.write dos head-ba 0 4)))
|
||||
(with-cache (-freeze-with-meta! x dos))
|
||||
(.toByteArray baos))
|
||||
compressor
|
||||
(if (identical? compressor :auto)
|
||||
(if no-header?
|
||||
lz4-compressor
|
||||
(if-let [fc *auto-freeze-compressor*]
|
||||
(fc ba)
|
||||
;; Intelligently enable compression only if benefit
|
||||
;; is likely to outweigh cost:
|
||||
(when (> (alength ba) 8192) lz4-compressor)))
|
||||
|
||||
(do
|
||||
(with-cache (-freeze-with-meta! x dos))
|
||||
(let [ba (.toByteArray baos)
|
||||
(if (fn? compressor)
|
||||
(compressor ba) ; Assume compressor selector fn
|
||||
compressor ; Assume compressor
|
||||
))
|
||||
|
||||
compressor
|
||||
(if (identical? compressor :auto)
|
||||
(if no-header?
|
||||
lz4-compressor
|
||||
(if-let [fc *auto-freeze-compressor*]
|
||||
(fc ba)
|
||||
;; Intelligently enable compression only if benefit
|
||||
;; is likely to outweigh cost:
|
||||
(when (> (alength ba) 8192) lz4-compressor)))
|
||||
ba (if compressor (compress compressor ba) ba)
|
||||
ba (if encryptor (encrypt encryptor password ba) ba)]
|
||||
|
||||
(if (fn? compressor)
|
||||
(compressor ba) ; Assume compressor selector fn
|
||||
compressor ; Assume compressor
|
||||
))
|
||||
(if no-header?
|
||||
ba
|
||||
(wrap-header ba
|
||||
{:compressor-id
|
||||
(when-let [c compressor]
|
||||
(or (compression/standard-header-ids
|
||||
(compression/header-id c))
|
||||
:else))
|
||||
|
||||
ba (if compressor (compress compressor ba) ba)
|
||||
ba (if encryptor (encrypt encryptor password ba) ba)]
|
||||
|
||||
(if no-header?
|
||||
ba
|
||||
(wrap-header ba
|
||||
{:compressor-id
|
||||
(when-let [c compressor]
|
||||
(or (compression/standard-header-ids
|
||||
(compression/header-id c))
|
||||
:else))
|
||||
|
||||
:encryptor-id
|
||||
(when-let [e encryptor]
|
||||
(or (encryption/standard-header-ids
|
||||
(encryption/header-id e))
|
||||
:else))}))))))))))
|
||||
:encryptor-id
|
||||
(when-let [e encryptor]
|
||||
(or (encryption/standard-header-ids
|
||||
(encryption/header-id e))
|
||||
:else))}))))))))
|
||||
|
||||
;;;; Thawing
|
||||
|
||||
|
|
@ -1209,13 +1143,6 @@
|
|||
|
||||
(enc/reduce-n (fn [acc _] (conj acc (thaw-from-in! in))) to n)))
|
||||
|
||||
(defn- read-objects [^objects ary ^DataInput in]
|
||||
(enc/reduce-n
|
||||
(fn [^objects ary i]
|
||||
(aset ary i (thaw-from-in! in))
|
||||
ary)
|
||||
ary (alength ary)))
|
||||
|
||||
(defn- read-kvs-into [to ^DataInput in ^long n]
|
||||
(if (and (editable? to) (> n 10))
|
||||
(persistent!
|
||||
|
|
@ -1299,30 +1226,6 @@
|
|||
:throwable e
|
||||
:nippy/unthawable {:class-name class-name :content content}}))))
|
||||
|
||||
(defn- read-type [in class-name]
|
||||
(try
|
||||
(let [aclass (clojure.lang.RT/classForName class-name)
|
||||
nbasis
|
||||
(let [basis-method (.getMethod aclass "getBasis" nil)
|
||||
basis (.invoke basis-method nil nil)]
|
||||
(count basis))
|
||||
|
||||
cvalues (object-array nbasis)]
|
||||
|
||||
(enc/reduce-n
|
||||
(fn [_ i] (aset cvalues i (thaw-from-in! in)))
|
||||
nil nbasis)
|
||||
|
||||
(let [ctors (.getConstructors aclass)
|
||||
^Constructor ctor (aget ctors 0) ; Impl. detail? Ref. https://goo.gl/XWmckR
|
||||
]
|
||||
(.newInstance ctor cvalues)))
|
||||
|
||||
(catch Exception e
|
||||
{:type :type
|
||||
:throwable e
|
||||
:nippy/unthawable {:class-name class-name}})))
|
||||
|
||||
(defn thaw-from-in!
|
||||
"Deserializes a frozen object from given DataInput to its original Clojure
|
||||
data type.
|
||||
|
|
@ -1345,8 +1248,6 @@
|
|||
id-record-md (read-record in (read-utf8 in (read-md-count in)))
|
||||
id-record-lg (read-record in (read-utf8 in (read-lg-count in)))
|
||||
|
||||
id-type (read-type in (thaw-from-in! in))
|
||||
|
||||
id-nil nil
|
||||
id-true true
|
||||
id-false false
|
||||
|
|
@ -1370,8 +1271,6 @@
|
|||
id-bytes-md (read-bytes in (read-md-count in))
|
||||
id-bytes-lg (read-bytes in (read-lg-count in))
|
||||
|
||||
id-objects-lg (read-objects (object-array (read-lg-count in)) in)
|
||||
|
||||
id-str-0 ""
|
||||
id-str-sm (read-utf8 in (read-sm-count in))
|
||||
id-str-md (read-utf8 in (read-md-count in))
|
||||
|
|
@ -1435,7 +1334,6 @@
|
|||
(read-biginteger in))
|
||||
|
||||
id-date (Date. (.readLong in))
|
||||
id-uri (URI. (thaw-from-in! in))
|
||||
id-uuid (UUID. (.readLong in) (.readLong in))
|
||||
|
||||
;; Deprecated ------------------------------------------------------
|
||||
|
|
@ -1485,19 +1383,18 @@
|
|||
:lzma2 lzma2-compressor
|
||||
:lz4 lz4-compressor
|
||||
:no-header (throw (ex-info ":auto not supported on headerless data." {}))
|
||||
:else (throw (ex-info ":auto not supported for non-standard compressors." {}))
|
||||
(do (throw (ex-info (str "Unrecognized :auto compressor id: " compressor-id)
|
||||
{:compressor-id compressor-id})))))
|
||||
:else (throw (ex-info ":auto not supported for non-standard compressors." {}))
|
||||
(throw (ex-info (str "Unrecognized :auto compressor id: " compressor-id)
|
||||
{:compressor-id compressor-id}))))
|
||||
|
||||
(defn- get-auto-encryptor [encryptor-id]
|
||||
(case encryptor-id
|
||||
nil nil
|
||||
:aes128-gcm-sha512 aes128-gcm-encryptor
|
||||
:aes128-cbc-sha512 aes128-cbc-encryptor
|
||||
:no-header (throw (ex-info ":auto not supported on headerless data." {}))
|
||||
:else (throw (ex-info ":auto not supported for non-standard encryptors." {}))
|
||||
(do (throw (ex-info (str "Unrecognized :auto encryptor id: " encryptor-id)
|
||||
{:encryptor-id encryptor-id})))))
|
||||
nil nil
|
||||
:aes128-sha512 aes128-encryptor
|
||||
:no-header (throw (ex-info ":auto not supported on headerless data." {}))
|
||||
:else (throw (ex-info ":auto not supported for non-standard encryptors." {}))
|
||||
(throw (ex-info (str "Unrecognized :auto encryptor id: " encryptor-id)
|
||||
{:encryptor-id encryptor-id}))))
|
||||
|
||||
(def ^:private err-msg-unknown-thaw-failure
|
||||
"Decryption/decompression failure, or data unfrozen/damaged.")
|
||||
|
|
@ -1531,83 +1428,80 @@
|
|||
|
||||
([ba] (thaw ba nil))
|
||||
([^bytes ba
|
||||
{:as opts
|
||||
:keys [v1-compatibility? compressor encryptor password]
|
||||
{:keys [v1-compatibility? compressor encryptor password]
|
||||
:or {compressor :auto
|
||||
encryptor :auto}}]
|
||||
encryptor :auto}
|
||||
:as opts}]
|
||||
|
||||
(assert (not (get opts :headerless-meta))
|
||||
":headerless-meta `thaw` opt removed in Nippy v2.7+")
|
||||
|
||||
(call-with-bindings opts
|
||||
(fn []
|
||||
(let [v2+? (not v1-compatibility?)
|
||||
no-header? (get opts :no-header?) ; Intentionally undocumented
|
||||
ex (fn ex
|
||||
([ msg] (ex nil msg))
|
||||
([e msg] (throw (ex-info (str "Thaw failed: " msg)
|
||||
{:opts (assoc opts
|
||||
:compressor compressor
|
||||
:encryptor encryptor)}
|
||||
e))))
|
||||
|
||||
(let [v2+? (not v1-compatibility?)
|
||||
no-header? (get opts :no-header?) ; Intentionally undocumented
|
||||
ex (fn ex
|
||||
([ msg] (ex nil msg))
|
||||
([e msg] (throw (ex-info (str "Thaw failed: " msg)
|
||||
{:opts (assoc opts
|
||||
:compressor compressor
|
||||
:encryptor encryptor)}
|
||||
e))))
|
||||
thaw-data
|
||||
(fn [data-ba compressor-id encryptor-id ex-fn]
|
||||
(let [compressor (if (identical? compressor :auto)
|
||||
(get-auto-compressor compressor-id)
|
||||
compressor)
|
||||
encryptor (if (identical? encryptor :auto)
|
||||
(get-auto-encryptor encryptor-id)
|
||||
encryptor)]
|
||||
|
||||
thaw-data
|
||||
(fn [data-ba compressor-id encryptor-id ex-fn]
|
||||
(let [compressor (if (identical? compressor :auto)
|
||||
(get-auto-compressor compressor-id)
|
||||
compressor)
|
||||
encryptor (if (identical? encryptor :auto)
|
||||
(get-auto-encryptor encryptor-id)
|
||||
encryptor)]
|
||||
(when (and encryptor (not password))
|
||||
(ex "Password required for decryption."))
|
||||
|
||||
(when (and encryptor (not password))
|
||||
(ex "Password required for decryption."))
|
||||
(try
|
||||
(let [ba data-ba
|
||||
ba (if encryptor (decrypt encryptor password ba) ba)
|
||||
ba (if compressor (decompress compressor ba) ba)
|
||||
dis (DataInputStream. (ByteArrayInputStream. ba))]
|
||||
|
||||
(try
|
||||
(let [ba data-ba
|
||||
ba (if encryptor (decrypt encryptor password ba) ba)
|
||||
ba (if compressor (decompress compressor ba) ba)
|
||||
dis (DataInputStream. (ByteArrayInputStream. ba))]
|
||||
(with-cache (thaw-from-in! dis)))
|
||||
|
||||
(with-cache (thaw-from-in! dis)))
|
||||
(catch Exception e (ex-fn e)))))
|
||||
|
||||
(catch Exception e (ex-fn e)))))
|
||||
;; Hackish + can actually segfault JVM due to Snappy bug,
|
||||
;; Ref. http://goo.gl/mh7Rpy - no better alternatives, unfortunately
|
||||
thaw-v1-data
|
||||
(fn [data-ba ex-fn]
|
||||
(thaw-data data-ba :snappy nil
|
||||
(fn [_] (thaw-data data-ba nil nil (fn [_] (ex-fn nil))))))]
|
||||
|
||||
;; Hackish + can actually segfault JVM due to Snappy bug,
|
||||
;; Ref. http://goo.gl/mh7Rpy - no better alternatives, unfortunately
|
||||
thaw-v1-data
|
||||
(fn [data-ba ex-fn]
|
||||
(thaw-data data-ba :snappy nil
|
||||
(fn [_] (thaw-data data-ba nil nil (fn [_] (ex-fn nil))))))]
|
||||
(if no-header?
|
||||
(if v2+?
|
||||
(thaw-data ba :no-header :no-header (fn [e] (ex e err-msg-unknown-thaw-failure)))
|
||||
(thaw-data ba :no-header :no-header
|
||||
(fn [e] (thaw-v1-data ba (fn [_] (ex e err-msg-unknown-thaw-failure))))))
|
||||
|
||||
(if no-header?
|
||||
(if v2+?
|
||||
(thaw-data ba :no-header :no-header (fn [e] (ex e err-msg-unknown-thaw-failure)))
|
||||
(thaw-data ba :no-header :no-header
|
||||
(fn [e] (thaw-v1-data ba (fn [_] (ex e err-msg-unknown-thaw-failure))))))
|
||||
;; At this point we assume that we have a header iff we have v2+ data
|
||||
(if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?]
|
||||
:as head-meta}] (try-parse-header ba)]
|
||||
|
||||
;; At this point we assume that we have a header iff we have v2+ data
|
||||
(if-let [[data-ba {:keys [compressor-id encryptor-id unrecognized-meta?]
|
||||
:as head-meta}] (try-parse-header ba)]
|
||||
;; A well-formed header _appears_ to be present (it's possible though
|
||||
;; unlikely that this is a fluke and data is actually headerless):
|
||||
(if v2+?
|
||||
(if unrecognized-meta?
|
||||
(ex err-msg-unrecognized-header)
|
||||
(thaw-data data-ba compressor-id encryptor-id
|
||||
(fn [e] (ex e err-msg-unknown-thaw-failure))))
|
||||
|
||||
;; A well-formed header _appears_ to be present (it's possible though
|
||||
;; unlikely that this is a fluke and data is actually headerless):
|
||||
(if v2+?
|
||||
(if unrecognized-meta?
|
||||
(ex err-msg-unrecognized-header)
|
||||
(thaw-data data-ba compressor-id encryptor-id
|
||||
(fn [e] (ex e err-msg-unknown-thaw-failure))))
|
||||
(if unrecognized-meta?
|
||||
(thaw-v1-data ba (fn [_] (ex err-msg-unrecognized-header)))
|
||||
(thaw-data data-ba compressor-id encryptor-id
|
||||
(fn [e] (thaw-v1-data ba (fn [_] (ex e err-msg-unknown-thaw-failure)))))))
|
||||
|
||||
(if unrecognized-meta?
|
||||
(thaw-v1-data ba (fn [_] (ex err-msg-unrecognized-header)))
|
||||
(thaw-data data-ba compressor-id encryptor-id
|
||||
(fn [e] (thaw-v1-data ba (fn [_] (ex e err-msg-unknown-thaw-failure)))))))
|
||||
|
||||
;; Well-formed header definitely not present
|
||||
(if v2+?
|
||||
(ex err-msg-unknown-thaw-failure)
|
||||
(thaw-v1-data ba (fn [_] (ex err-msg-unknown-thaw-failure)))))))))))
|
||||
;; Well-formed header definitely not present
|
||||
(if v2+?
|
||||
(ex err-msg-unknown-thaw-failure)
|
||||
(thaw-v1-data ba (fn [_] (ex err-msg-unknown-thaw-failure)))))))))
|
||||
|
||||
(comment
|
||||
(thaw (freeze "hello"))
|
||||
|
|
@ -1699,9 +1593,6 @@
|
|||
;;;; Stress data
|
||||
|
||||
(defrecord StressRecord [data])
|
||||
(deftype StressType [data]
|
||||
Object
|
||||
(equals [a b] (= (.-data a) (.-data ^StressType b))))
|
||||
(def stress-data "Reference data used for tests & benchmarks"
|
||||
{:bytes (byte-array [(byte 1) (byte 2) (byte 3)])
|
||||
:nil nil
|
||||
|
|
@ -1756,13 +1647,10 @@
|
|||
:bigdec (bigdec 3.1415926535897932384626433832795)
|
||||
|
||||
:ratio 22/7
|
||||
:uri (URI. "https://clojure.org/reference/data_structures")
|
||||
:uuid (java.util.UUID/randomUUID)
|
||||
:date (java.util.Date.)
|
||||
:objects (object-array [1 "two" {:data "data"}])
|
||||
|
||||
:stress-record (StressRecord. "data")
|
||||
:stress-type (StressType. "data")
|
||||
|
||||
;; Serializable
|
||||
:throwable (Throwable. "Yolo")
|
||||
|
|
@ -1771,14 +1659,14 @@
|
|||
|
||||
(def stress-data-comparable
|
||||
"Reference data with stuff removed that breaks roundtrip equality"
|
||||
(dissoc stress-data :bytes :throwable :exception :ex-info :regex :objects))
|
||||
(dissoc stress-data :bytes :throwable :exception :ex-info :regex))
|
||||
|
||||
(def stress-data-benchable
|
||||
"Reference data with stuff removed that breaks reader or other utils we'll
|
||||
be benching against"
|
||||
(dissoc stress-data
|
||||
:bytes :throwable :exception :ex-info :queue :queue-empty
|
||||
:byte :stress-record :stress-type :regex :objects))
|
||||
:byte :stress-record :regex))
|
||||
|
||||
;;;; Tools
|
||||
|
||||
|
|
@ -1810,29 +1698,12 @@
|
|||
(inspect-ba (freeze "hello"))
|
||||
(seq (:data-ba (inspect-ba (freeze "hello")))))
|
||||
|
||||
(defn freeze-to-string
|
||||
"Convenience util: like `freeze`, but returns a Base64-encoded string.
|
||||
See also `thaw-from-string`."
|
||||
([x ] (freeze-to-string x nil))
|
||||
([x freeze-opts]
|
||||
(let [ba (freeze x freeze-opts)]
|
||||
(.encodeToString (java.util.Base64/getEncoder)
|
||||
ba))))
|
||||
|
||||
(defn thaw-from-string
|
||||
"Convenience util: like `thaw`, but takes a Base64-encoded string.
|
||||
See also `freeze-to-string`."
|
||||
([s ] (thaw-from-string s nil))
|
||||
([^String s thaw-opts]
|
||||
(let [ba (.decode (java.util.Base64/getDecoder) s)]
|
||||
(thaw ba thaw-opts))))
|
||||
|
||||
(comment (thaw-from-string (freeze-to-string {:a :A :b [:B1 :B2]})))
|
||||
|
||||
(defn freeze-to-file
|
||||
"Convenience util: like `freeze`, but writes to `(clojure.java.io/file <file>)`
|
||||
and returns the byte array written.
|
||||
See also `thaw-from-file`."
|
||||
"Convenience util: writes `(freeze x freeze-opts)` byte array to
|
||||
`(clojure.java.io/file file)` and returns the byte array.
|
||||
|
||||
(freeze-to-file \"my-filename.npy\" my-val) => Serialized byte array"
|
||||
|
||||
([file x ] (freeze-to-file file x nil))
|
||||
([file x freeze-opts]
|
||||
(let [^bytes ba (freeze x freeze-opts)]
|
||||
|
|
@ -1841,12 +1712,14 @@
|
|||
ba)))
|
||||
|
||||
(defn thaw-from-file
|
||||
"Convenience util: like `thaw`, but reads from `(clojure.java.io/file <file>)`.
|
||||
"Convenience util: returns `(thaw ba thaw-opts)` Clojure value for the
|
||||
byte array read from `(clojure.java.io/file file)`.
|
||||
|
||||
(thaw-from-file \"my-filename.npy\") => Deserialized Clojure value
|
||||
|
||||
To thaw from a resource on classpath (e.g in Leiningen `resources` dir):
|
||||
(thaw-from-file (clojure.java.io/resource \"my-resource-name.npy\"))
|
||||
(thaw-from-file (clojure.java.io/resource \"my-resource-name.npy\"))"
|
||||
|
||||
See also `freeze-to-file`."
|
||||
([file ] (thaw-from-file file nil))
|
||||
([file thaw-opts]
|
||||
(let [file (jio/file file),
|
||||
|
|
@ -1863,4 +1736,6 @@
|
|||
|
||||
;;;; Deprecated
|
||||
|
||||
(enc/deprecated (def freeze-fallback-as-str "DEPRECATED" write-unfreezable))
|
||||
(enc/deprecated
|
||||
(enc/defonce ^:dynamic *final-freeze-fallback* "DEPRECATED" nil)
|
||||
(def freeze-fallback-as-str "DEPRECATED" write-unfreezable))
|
||||
|
|
|
|||
|
|
@ -1,153 +0,0 @@
|
|||
(ns taoensso.nippy.crypto
|
||||
"Low-level crypto utils.
|
||||
Private & alpha, very likely to change!"
|
||||
(:refer-clojure :exclude [rand-nth])
|
||||
(:require [taoensso.encore :as enc]))
|
||||
|
||||
;; Note that AES128 may be preferable to AES256 due to known attack
|
||||
;; vectors specific to AES256, Ref. https://goo.gl/qU4CCV
|
||||
;; or for a counter argument, Ref. https://goo.gl/9LA9Yb
|
||||
|
||||
;;;; Randomness
|
||||
|
||||
(do
|
||||
(defn rand-nth [coll] (nth coll (int (* (.nextDouble (enc/srng)) (count coll)))))
|
||||
(defn rand-bytes ^bytes [size] (let [ba (byte-array size)] (.nextBytes (enc/srng) ba) ba))
|
||||
(defn rand-double ^double [] (.nextDouble (enc/srng)))
|
||||
(defn rand-gauss ^double [] (.nextGaussian (enc/srng)))
|
||||
(defn rand-bool [] (.nextBoolean (enc/srng)))
|
||||
(defn rand-long
|
||||
(^long [ ] (.nextLong (enc/srng)))
|
||||
(^long [n] (long (* (long n) (.nextDouble (enc/srng)))))))
|
||||
|
||||
(comment
|
||||
(seq (rand-bytes 16))
|
||||
(rand-nth [:a :b :c :d])
|
||||
(rand-long 100))
|
||||
|
||||
;;;; Hashing
|
||||
|
||||
(def ^:private sha256-md* (enc/thread-local-proxy (java.security.MessageDigest/getInstance "SHA-256")))
|
||||
(def ^:private sha512-md* (enc/thread-local-proxy (java.security.MessageDigest/getInstance "SHA-512")))
|
||||
(defn sha256-md ^java.security.MessageDigest [] (.get ^ThreadLocal sha256-md*))
|
||||
(defn sha512-md ^java.security.MessageDigest [] (.get ^ThreadLocal sha512-md*))
|
||||
(defn sha256-ba ^bytes [ba] (.digest (sha256-md) ba))
|
||||
(defn sha512-ba ^bytes [ba] (.digest (sha512-md) ba))
|
||||
|
||||
(enc/compile-if clojure.lang.Murmur3
|
||||
(defn murmur3 [^String s] (clojure.lang.Murmur3/hashUnencodedChars s))
|
||||
nil)
|
||||
|
||||
;;;; Key derivation (salt+password -> key / hash)
|
||||
;; (fn [salt-ba utf8]) -> bytes
|
||||
|
||||
;; (defn ba->hex [^bytes ba] (org.apache.commons.codec.binary.Hex/encodeHexString ba))
|
||||
(defn take-ba ^bytes [n ^bytes ba] (java.util.Arrays/copyOf ba ^int n)) ; Pads if ba too small
|
||||
(defn utf8->ba ^bytes [^String s] (.getBytes s "UTF-8"))
|
||||
(defn- add-salt ^bytes [?salt-ba ba] (if ?salt-ba (enc/ba-concat ?salt-ba ba) ba))
|
||||
(defn pwd-as-ba ^bytes [utf8-or-ba] (if (string? utf8-or-ba) (utf8->ba utf8-or-ba) (enc/have enc/bytes? utf8-or-ba)))
|
||||
|
||||
(comment (seq (pwd-as-ba "foo")))
|
||||
|
||||
(defn sha512-key-ba
|
||||
"SHA512-based key generator. Good JVM availability without extra dependencies
|
||||
(PBKDF2, bcrypt, scrypt, etc.). Decent security when using many rounds."
|
||||
(^bytes [?salt-ba utf8-or-ba ] (sha512-key-ba ?salt-ba utf8-or-ba 163835 #_(* Short/MAX_VALUE 5)))
|
||||
(^bytes [?salt-ba utf8-or-ba ^long n-rounds]
|
||||
(let [ba (add-salt ?salt-ba (pwd-as-ba utf8-or-ba))
|
||||
md (sha512-md)]
|
||||
(enc/reduce-n (fn [acc in] (.digest md acc)) ba n-rounds))))
|
||||
|
||||
(comment
|
||||
(count (seq (sha512-key-ba (utf8->ba "salt") "password" 1)))
|
||||
(count (seq (sha512-key-ba nil "password" 1))))
|
||||
|
||||
;;;; Crypto
|
||||
|
||||
(defprotocol ICipherKit
|
||||
(get-cipher ^javax.crypto.Cipher [_] "Returns a thread-safe `javax.crypto.Cipher` instance.")
|
||||
(get-iv-size [_] "Returns necessary iv-ba length.")
|
||||
(get-key-spec ^javax.crypto.spec.SecretKeySpec [_ ba] "Returns a `javax.crypto.spec.SecretKeySpec`.")
|
||||
(get-param-spec ^java.security.spec.AlgorithmParameterSpec [_ iv-ba] "Returns a `java.security.spec.AlgorithmParameters`."))
|
||||
|
||||
;; Prefer GCM > CBC, Ref. https://goo.gl/jpZoj8
|
||||
(def ^:private gcm-cipher* (enc/thread-local-proxy (javax.crypto.Cipher/getInstance "AES/GCM/NoPadding")))
|
||||
(def ^:private cbc-cipher* (enc/thread-local-proxy (javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding")))
|
||||
|
||||
(defn gcm-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal gcm-cipher*))
|
||||
(defn cbc-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal cbc-cipher*))
|
||||
;
|
||||
(deftype CipherKit-AES-GCM []
|
||||
ICipherKit
|
||||
(get-cipher [_] (gcm-cipher))
|
||||
(get-iv-size [_] 12)
|
||||
(get-key-spec [_ ba] (javax.crypto.spec.SecretKeySpec. ba "AES"))
|
||||
(get-param-spec [_ iv-ba] (javax.crypto.spec.GCMParameterSpec. 128 iv-ba)))
|
||||
|
||||
(deftype CipherKit-AES-CBC []
|
||||
ICipherKit
|
||||
(get-cipher [_] (cbc-cipher))
|
||||
(get-iv-size [_] 16)
|
||||
(get-key-spec [_ ba] (javax.crypto.spec.SecretKeySpec. ba "AES"))
|
||||
(get-param-spec [_ iv-ba] (javax.crypto.spec.IvParameterSpec. iv-ba)))
|
||||
|
||||
(def cipher-kit-aes-gcm "Default CipherKit for AES GCM" (CipherKit-AES-GCM.))
|
||||
(def cipher-kit-aes-cbc "Default CipherKit for AES CBC" (CipherKit-AES-CBC.))
|
||||
|
||||
;; Output bytes: [ <iv> <?salt> <encrypted>]
|
||||
;; Could also do: [<iv-len> <iv> <salt-len> <?salt> <encrypted>]
|
||||
(defn encrypt
|
||||
[{:keys [cipher-kit ?salt-ba key-ba plain-ba rand-bytes-fn]
|
||||
:or {cipher-kit cipher-kit-aes-gcm
|
||||
rand-bytes-fn rand-bytes}}]
|
||||
|
||||
(let [iv-size (long (get-iv-size cipher-kit))
|
||||
iv-ba (rand-bytes-fn iv-size)
|
||||
prefix-ba (if ?salt-ba (enc/ba-concat iv-ba ?salt-ba) iv-ba)
|
||||
key-spec (get-key-spec cipher-kit key-ba)
|
||||
param-spec (get-param-spec cipher-kit iv-ba)
|
||||
cipher (get-cipher cipher-kit)]
|
||||
|
||||
(.init cipher javax.crypto.Cipher/ENCRYPT_MODE key-spec param-spec)
|
||||
(enc/ba-concat prefix-ba (.doFinal cipher plain-ba))))
|
||||
|
||||
(comment (encrypt {:?salt-ba nil :key-ba (take-ba 16 (sha512-key-ba nil "pwd")) :plain-ba (utf8->ba "data")}))
|
||||
|
||||
(defn decrypt
|
||||
[{:keys [cipher-kit salt-size salt->key-fn enc-ba]
|
||||
:or {cipher-kit cipher-kit-aes-gcm}}]
|
||||
(let [salt-size (long salt-size)
|
||||
iv-size (long (get-iv-size cipher-kit))
|
||||
prefix-size (+ iv-size salt-size)
|
||||
[prefix-ba enc-ba] (enc/ba-split enc-ba prefix-size)
|
||||
[iv-ba salt-ba] (if (pos? salt-size)
|
||||
(enc/ba-split prefix-ba iv-size)
|
||||
[prefix-ba nil])
|
||||
|
||||
key-ba (salt->key-fn salt-ba)
|
||||
key-spec (get-key-spec cipher-kit key-ba)
|
||||
param-spec (get-param-spec cipher-kit iv-ba)
|
||||
cipher (get-cipher cipher-kit)]
|
||||
|
||||
(.init cipher javax.crypto.Cipher/DECRYPT_MODE key-spec param-spec)
|
||||
(.doFinal cipher enc-ba)))
|
||||
|
||||
(comment
|
||||
(do
|
||||
(defn sha512-k16 [?salt-ba pwd] (take-ba 16 (sha512-key-ba ?salt-ba pwd)))
|
||||
(defn roundtrip [kit ?salt-ba key-ba key-fn]
|
||||
(let [salt-size (count ?salt-ba)
|
||||
encr (encrypt {:cipher-kit kit :?salt-ba ?salt-ba :key-ba key-ba :plain-ba (utf8->ba "data")})
|
||||
decr (decrypt {:cipher-kit kit :salt-size salt-size :salt->key-fn key-fn :enc-ba encr})]
|
||||
(String. ^bytes decr "UTF-8")))
|
||||
|
||||
[(let [s (rand-bytes 16)] (roundtrip cipher-kit-aes-gcm s (sha512-k16 s "pwd") #(sha512-k16 % "pwd")))
|
||||
(let [s nil] (roundtrip cipher-kit-aes-gcm s (sha512-k16 s "pwd") #(sha512-k16 % "pwd")))
|
||||
(let [s (rand-bytes 16)] (roundtrip cipher-kit-aes-cbc s (sha512-k16 s "pwd") #(sha512-k16 % "pwd")))
|
||||
(let [s nil] (roundtrip cipher-kit-aes-cbc s (sha512-k16 s "pwd") #(sha512-k16 % "pwd")))])
|
||||
|
||||
(enc/qb 10
|
||||
(let [s (rand-bytes 16)]
|
||||
(roundtrip cipher-kit-aes-gcm s (sha512-k16 s "pwd") #(sha512-k16 % "pwd"))))
|
||||
;; 2394.89
|
||||
)
|
||||
|
|
@ -1,24 +1,63 @@
|
|||
(ns taoensso.nippy.encryption
|
||||
"Simple no-nonsense crypto with reasonable defaults"
|
||||
(:require
|
||||
[taoensso.encore :as enc]
|
||||
[taoensso.nippy.crypto :as crypto]))
|
||||
(:require [taoensso.encore :as enc]))
|
||||
|
||||
(def standard-header-ids
|
||||
"These'll support :auto thaw"
|
||||
#{:aes128-cbc-sha512
|
||||
:aes128-gcm-sha512})
|
||||
;;;; Interface
|
||||
|
||||
(def standard-header-ids "These'll support :auto thaw" #{:aes128-sha512})
|
||||
|
||||
(defprotocol IEncryptor
|
||||
(header-id [encryptor])
|
||||
(encrypt ^bytes [encryptor pwd ba])
|
||||
(decrypt ^bytes [encryptor pwd ba]))
|
||||
|
||||
;;;; Default digests, ciphers, etc.
|
||||
|
||||
(def ^:private aes128-cipher* (enc/thread-local-proxy (javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding")))
|
||||
(def ^:private sha512-md* (enc/thread-local-proxy (java.security.MessageDigest/getInstance "SHA-512")))
|
||||
(def ^:private prng* (enc/thread-local-proxy (java.security.SecureRandom/getInstance "SHA1PRNG")))
|
||||
|
||||
(defn- aes128-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal aes128-cipher*))
|
||||
(defn- sha512-md ^java.security.MessageDigest [] (.get ^ThreadLocal sha512-md*))
|
||||
(defn- prng ^java.security.SecureRandom [] (.get ^ThreadLocal prng*))
|
||||
|
||||
(def ^:private ^:const aes128-block-size (.getBlockSize (aes128-cipher)))
|
||||
(def ^:private ^:const salt-size aes128-block-size)
|
||||
|
||||
(defn- rand-bytes [size] (let [ba (byte-array size)] (.nextBytes (prng) ba) ba))
|
||||
|
||||
;;;; Default key-gen
|
||||
|
||||
(defn- sha512-key
|
||||
"SHA512-based key generator. Good JVM availability without extra dependencies
|
||||
(PBKDF2, bcrypt, scrypt, etc.). Decent security when using many rounds."
|
||||
|
||||
([salt-ba pwd ] (sha512-key salt-ba pwd (* Short/MAX_VALUE (if salt-ba 5 64))))
|
||||
([salt-ba pwd ^long n]
|
||||
(let [md (sha512-md)
|
||||
init-ba (let [pwd-ba (.getBytes ^String pwd "UTF-8")]
|
||||
(if salt-ba (enc/ba-concat salt-ba pwd-ba) pwd-ba))
|
||||
^bytes ba (enc/reduce-n (fn [acc in] (.digest md acc)) init-ba n)]
|
||||
|
||||
(-> ba
|
||||
(java.util.Arrays/copyOf aes128-block-size)
|
||||
(javax.crypto.spec.SecretKeySpec. "AES")))))
|
||||
|
||||
(comment
|
||||
(enc/qb 10
|
||||
(sha512-key nil "hi" (* Short/MAX_VALUE 1)) ; ~40ms per hash (fast)
|
||||
(sha512-key nil "hi" (* Short/MAX_VALUE 5)) ; ~180ms (default)
|
||||
(sha512-key nil "hi" (* Short/MAX_VALUE 32)) ; ~1200ms (conservative)
|
||||
(sha512-key nil "hi" (* Short/MAX_VALUE 128)) ; ~4500ms (paranoid)
|
||||
))
|
||||
|
||||
;;;; Default implementations
|
||||
|
||||
(defn- throw-destructure-ex [typed-password]
|
||||
(throw (ex-info
|
||||
(str "Expected password form: "
|
||||
"[<#{:salted :cached}> <password-string>].\n "
|
||||
"See `aes128-encryptor` docstring for details!")
|
||||
"See `default-aes128-encryptor` docstring for details!")
|
||||
{:typed-password typed-password})))
|
||||
|
||||
(defn- destructure-typed-pwd [typed-password]
|
||||
|
|
@ -31,41 +70,46 @@
|
|||
|
||||
(comment (destructure-typed-pwd [:salted "foo"]))
|
||||
|
||||
(deftype AES128Encryptor [header-id cipher-kit salted-key-fn cached-key-fn]
|
||||
(deftype AES128Encryptor [header-id keyfn cached-keyfn]
|
||||
IEncryptor
|
||||
(header-id [_] header-id)
|
||||
(encrypt [_ typed-pwd plain-ba]
|
||||
(encrypt [_ typed-pwd data-ba]
|
||||
(let [[type pwd] (destructure-typed-pwd typed-pwd)
|
||||
salt? (identical? type :salted)
|
||||
?salt-ba (when salt? (crypto/rand-bytes 16))
|
||||
key-ba
|
||||
(crypto/take-ba 16 ; 128 bit AES
|
||||
(if-let [salt-ba ?salt-ba]
|
||||
(salted-key-fn salt-ba pwd)
|
||||
(cached-key-fn nil pwd)))]
|
||||
iv-ba (rand-bytes aes128-block-size)
|
||||
salt-ba (when salt? (rand-bytes salt-size))
|
||||
prefix-ba (if salt? (enc/ba-concat iv-ba salt-ba) iv-ba)
|
||||
key (if salt?
|
||||
(keyfn salt-ba pwd)
|
||||
(cached-keyfn salt-ba pwd))
|
||||
iv (javax.crypto.spec.IvParameterSpec. iv-ba)
|
||||
cipher (aes128-cipher)]
|
||||
|
||||
(crypto/encrypt
|
||||
{:cipher-kit cipher-kit
|
||||
:?salt-ba ?salt-ba
|
||||
:key-ba key-ba
|
||||
:plain-ba plain-ba})))
|
||||
(.init cipher javax.crypto.Cipher/ENCRYPT_MODE
|
||||
^javax.crypto.spec.SecretKeySpec key iv)
|
||||
(enc/ba-concat prefix-ba (.doFinal cipher data-ba))))
|
||||
|
||||
(decrypt [_ typed-pwd enc-ba]
|
||||
(let [[type pwd] (destructure-typed-pwd typed-pwd)
|
||||
salt? (identical? type :salted)
|
||||
salt->key-fn
|
||||
(if salt?
|
||||
#(salted-key-fn % pwd)
|
||||
#(cached-key-fn % pwd))]
|
||||
(decrypt [_ typed-pwd ba]
|
||||
(let [[type pwd] (destructure-typed-pwd typed-pwd)
|
||||
salt? (identical? type :salted)
|
||||
prefix-size (+ aes128-block-size (if salt? salt-size 0))
|
||||
[prefix-ba data-ba] (enc/ba-split ba prefix-size)
|
||||
[iv-ba salt-ba] (if salt?
|
||||
(enc/ba-split prefix-ba aes128-block-size)
|
||||
[prefix-ba nil])
|
||||
key (if salt?
|
||||
(keyfn salt-ba pwd)
|
||||
(cached-keyfn salt-ba pwd))
|
||||
|
||||
(crypto/decrypt
|
||||
{:cipher-kit cipher-kit
|
||||
:salt-size (if salt? 16 0)
|
||||
:salt->key-fn salt->key-fn
|
||||
:enc-ba enc-ba}))))
|
||||
iv (javax.crypto.spec.IvParameterSpec. iv-ba)
|
||||
cipher (aes128-cipher)]
|
||||
|
||||
(def aes128-gcm-encryptor
|
||||
"Default 128bit AES-GCM encryptor with many-round SHA-512 key-gen.
|
||||
(.init cipher javax.crypto.Cipher/DECRYPT_MODE
|
||||
^javax.crypto.spec.SecretKeySpec key iv)
|
||||
(.doFinal cipher data-ba))))
|
||||
|
||||
(def aes128-encryptor
|
||||
"Default 128bit AES encryptor with many-round SHA-512 key-gen.
|
||||
|
||||
Password form [:salted \"my-password\"]
|
||||
---------------------------------------
|
||||
|
|
@ -100,18 +144,7 @@
|
|||
Faster than `aes128-salted`, and harder to attack any particular key - but
|
||||
increased danger if a key is somehow compromised."
|
||||
|
||||
(AES128Encryptor. :aes128-gcm-sha512
|
||||
crypto/cipher-kit-aes-gcm
|
||||
(do (fn [ salt-ba pwd] (crypto/take-ba 16 (crypto/sha512-key-ba salt-ba pwd (* Short/MAX_VALUE 5)))))
|
||||
(enc/memoize_ (fn [_salt-ba pwd] (crypto/take-ba 16 (crypto/sha512-key-ba nil pwd (* Short/MAX_VALUE 64)))))))
|
||||
|
||||
(def aes128-cbc-encryptor
|
||||
"Default 128bit AES-CBC encryptor with many-round SHA-512 key-gen.
|
||||
See also `aes-128-cbc-encryptor`."
|
||||
(AES128Encryptor. :aes128-cbc-sha512
|
||||
crypto/cipher-kit-aes-cbc
|
||||
(do (fn [ salt-ba pwd] (crypto/take-ba 16 (crypto/sha512-key-ba salt-ba pwd (* Short/MAX_VALUE 5)))))
|
||||
(enc/memoize_ (fn [_salt-ba pwd] (crypto/take-ba 16 (crypto/sha512-key-ba nil pwd (* Short/MAX_VALUE 64)))))))
|
||||
(AES128Encryptor. :aes128-sha512 sha512-key (enc/memoize_ sha512-key)))
|
||||
|
||||
;;;; Default implementation
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
Used by Carmine, Faraday, etc."
|
||||
(:require [taoensso.nippy :as nippy]))
|
||||
|
||||
;; TODO Switch to thread-local proxies?
|
||||
|
||||
(def ^:dynamic *freeze-opts* nil)
|
||||
(def ^:dynamic *thaw-opts* nil)
|
||||
|
||||
|
|
|
|||
|
|
@ -93,9 +93,6 @@
|
|||
test for pre/post serialization value equality (there's no good general
|
||||
way of doing so)."
|
||||
|
||||
;; TODO Not happy with this approach in general, could do with a refactor.
|
||||
;; Maybe return true/false/nil (nil => maybe)?
|
||||
|
||||
([x] (freezable? x nil))
|
||||
([x {:keys [allow-clojure-reader? allow-java-serializable?]}]
|
||||
(if (is-coll? x)
|
||||
|
|
@ -105,7 +102,6 @@
|
|||
(is? x java.lang.String)
|
||||
(is? x java.lang.Long)
|
||||
(is? x java.lang.Double)
|
||||
(nil? x)
|
||||
|
||||
(is? x clojure.lang.BigInt)
|
||||
(is? x clojure.lang.Ratio)
|
||||
|
|
|
|||
|
|
@ -38,9 +38,6 @@
|
|||
#(freeze % {:password [:salted "p"]}))
|
||||
test-data)))
|
||||
|
||||
(is (= (vec (:objects nippy/stress-data))
|
||||
((comp vec thaw freeze) (:objects nippy/stress-data))))
|
||||
|
||||
(is (= test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor})
|
||||
#(freeze % {:compressor nippy/lzma2-compressor}))
|
||||
test-data)))
|
||||
|
|
@ -77,12 +74,7 @@
|
|||
(thaw (org.xerial.snappy.Snappy/uncompress xerial-ba))
|
||||
(thaw (org.xerial.snappy.Snappy/uncompress iq80-ba))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))
|
||||
|
||||
(is ; CBC auto-encryptor compatibility
|
||||
(= "payload"
|
||||
(thaw (freeze "payload" {:password [:salted "pwd"] :encryptor nippy/aes128-cbc-encryptor})
|
||||
(do {:password [:salted "pwd"]})))))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))))
|
||||
|
||||
;;;; Custom types & records
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue