Merge branch 'dev'
This commit is contained in:
Peter Taoussanis
commit
6b2b501589
9 changed files with 806 additions and 677 deletions
18
CHANGELOG.md
18
CHANGELOG.md
|
|
@ -1,5 +1,23 @@
|
|||
> This project uses [Break Versioning](https://github.com/ptaoussanis/encore/blob/master/BREAK-VERSIONING.md) as of **Aug 16, 2014**.
|
||||
|
||||
## v2.11.0-alpha1 / 2015 Dec 1
|
||||
|
||||
> This is a major performance release that **drops default support for thawing Nippy v1 archives** but is otherwise non-breaking
|
||||
|
||||
* **BREAKING**: `thaw` now has `:v1-compatibility?` opt set to false by default (was true before) [1]
|
||||
* **Performance**: optimize serialized size of small maps, sets, vectors, bytes
|
||||
* **Performance**: optimized (no copy) `freeze` when using no compression or encryption
|
||||
* **Implementation**: swap most macros for fns (make low-level utils easier to use)
|
||||
|
||||
```clojure
|
||||
[com.taoensso/nippy "2.11.0-alpha1"]
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
||||
**[1]** Use `(thaw <frozen-byte-array> {:v1-compatibility? true})` to support thawing of data frozen with Nippy v1 (before ~June 2013)
|
||||
|
||||
|
||||
## v2.10.0 / 2015 Sep 30
|
||||
|
||||
> This is a major feature/performance release that **drops support for Clojure 1.4** but is otherwise non-breaking
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
**[API docs][]** | **[CHANGELOG][]** | [other Clojure libs][] | [Twitter][] | [contact/contrib](#contact--contributing) | current [Break Version][]:
|
||||
|
||||
```clojure
|
||||
[com.taoensso/nippy "2.10.0"] ; Stable, see CHANGELOG for details
|
||||
[com.taoensso/nippy "2.10.0"] ; Stable
|
||||
[com.taoensso/nippy "2.11.0-alpha1"] ; Dev, see CHANGELOG for details
|
||||
```
|
||||
|
||||
# Nippy, a Clojure serialization library
|
||||
|
|
|
|||
29
project.clj
29
project.clj
|
|
@ -1,4 +1,4 @@
|
|||
(defproject com.taoensso/nippy "2.10.0"
|
||||
(defproject com.taoensso/nippy "2.11.0-alpha1"
|
||||
:author "Peter Taoussanis <https://www.taoensso.com>"
|
||||
:description "Clojure serialization library"
|
||||
:url "https://github.com/ptaoussanis/nippy"
|
||||
|
|
@ -9,12 +9,13 @@
|
|||
:min-lein-version "2.3.3"
|
||||
:global-vars {*warn-on-reflection* true
|
||||
*assert* true
|
||||
*unchecked-math* :warn-on-boxed}
|
||||
;; *unchecked-math* :warn-on-boxed
|
||||
}
|
||||
|
||||
:dependencies
|
||||
[[org.clojure/clojure "1.5.1"]
|
||||
[org.clojure/tools.reader "0.9.2"]
|
||||
[com.taoensso/encore "2.18.0"]
|
||||
[org.clojure/tools.reader "0.10.0"]
|
||||
[com.taoensso/encore "2.26.1"]
|
||||
[org.iq80.snappy/snappy "0.4"]
|
||||
[org.tukaani/xz "1.5"]
|
||||
[net.jpountz.lz4/lz4 "1.3"]]
|
||||
|
|
@ -25,25 +26,25 @@
|
|||
:1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]}
|
||||
:1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]}
|
||||
:1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]}
|
||||
:1.8 {:dependencies [[org.clojure/clojure "1.8.0-alpha5"]]}
|
||||
:1.8 {:dependencies [[org.clojure/clojure "1.8.0-RC2"]]}
|
||||
:test {:jvm-opts ["-Xms1024m" "-Xmx2048m"]
|
||||
:dependencies [[expectations "2.1.1"]
|
||||
[org.clojure/test.check "0.8.2"]
|
||||
:dependencies [[org.clojure/test.check "0.9.0"]
|
||||
[org.clojure/data.fressian "0.2.1"]
|
||||
[org.xerial.snappy/snappy-java "1.1.2"]]}
|
||||
:dev [:1.7 :test
|
||||
{:plugins
|
||||
[[lein-pprint "1.1.1"]
|
||||
[lein-ancient "0.6.7"]
|
||||
[lein-expectations "0.0.8"]
|
||||
[lein-autoexpect "1.2.2"]
|
||||
[codox "0.8.10"]]}]}
|
||||
[[lein-pprint "1.1.2"]
|
||||
[lein-ancient "0.6.8"]
|
||||
[lein-codox "0.9.0"]]}]}
|
||||
|
||||
:test-paths ["test" "src"]
|
||||
|
||||
:codox
|
||||
{:language :clojure ; [:clojure :clojurescript] ; No support?
|
||||
:source-uri "https://github.com/ptaoussanis/nippy/blob/master/{filepath}#L{line}"}
|
||||
|
||||
:aliases
|
||||
{"test-all" ["with-profile" "+1.5:+1.6:+1.7:+1.8" "expectations"]
|
||||
"test-auto" ["with-profile" "+test" "autoexpect"]
|
||||
{"test-all" ["with-profile" "+1.5:+1.6:+1.7:+1.8" "test"]
|
||||
"deploy-lib" ["do" "deploy" "clojars," "install"]
|
||||
"start-dev" ["with-profile" "+server-jvm" "repl" ":headless"]}
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,10 +1,10 @@
|
|||
(ns taoensso.nippy.benchmarks
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [clojure.data.fressian :as fressian]
|
||||
[taoensso.encore :as encore]
|
||||
(:require [clojure.data.fressian :as fressian]
|
||||
[taoensso.encore :as enc]
|
||||
[taoensso.nippy :as nippy :refer (freeze thaw)]))
|
||||
|
||||
(def data nippy/stress-data-benchable)
|
||||
(def data #_22 nippy/stress-data-benchable)
|
||||
|
||||
(defn fressian-freeze [value]
|
||||
(let [^java.nio.ByteBuffer bb (fressian/write value)
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
|
||||
(comment (fressian-thaw (fressian-freeze data)))
|
||||
|
||||
(defmacro bench* [& body] `(encore/bench 10000 {:warmup-laps 20000} ~@body))
|
||||
(defmacro bench* [& body] `(enc/bench 10000 {:warmup-laps 20000} ~@body))
|
||||
(defn bench1 [freezer thawer & [sizer]]
|
||||
(let [data-frozen (freezer data)
|
||||
time-freeze (bench* (freezer data))
|
||||
|
|
@ -36,36 +36,69 @@
|
|||
(println (str "\nLap " (inc l) "/" laps "..."))
|
||||
|
||||
(when reader? ; Slow
|
||||
(println {:reader (bench1 encore/pr-edn encore/read-edn
|
||||
(println {:reader (bench1 enc/pr-edn enc/read-edn
|
||||
#(count (.getBytes ^String % "UTF-8")))}))
|
||||
|
||||
(println {:default (bench1 #(freeze % {})
|
||||
#(thaw % {}))})
|
||||
(println {:fast (bench1 #(freeze % {:compressor nil
|
||||
:skip-header? true})
|
||||
#(thaw % {:compressor nil
|
||||
:encryptor nil}))})
|
||||
(println {:encrypted (bench1 #(freeze % {:password [:cached "p"]})
|
||||
#(thaw % {:password [:cached "p"]}))})
|
||||
|
||||
(when lzma2? ; Slow as molasses
|
||||
(when lzma2? ; Slow
|
||||
(println {:lzma2 (bench1 #(freeze % {:compressor nippy/lzma2-compressor})
|
||||
#(thaw % {:compressor nippy/lzma2-compressor}))}))
|
||||
#(thaw % {:compressor nippy/lzma2-compressor}))}))
|
||||
|
||||
(when fressian?
|
||||
(println {:fressian (bench1 fressian-freeze fressian-thaw)})))
|
||||
(println {:fressian (bench1 fressian-freeze fressian-thaw)}))
|
||||
|
||||
(println {:encrypted (bench1 #(freeze % {:password [:cached "p"]})
|
||||
#(thaw % {:password [:cached "p"]}))})
|
||||
(println {:default (bench1 #(freeze % {})
|
||||
#(thaw % {}))})
|
||||
(println {:fast1 (bench1 #(freeze % {:compressor nil})
|
||||
#(thaw % {:compressor nil}))})
|
||||
(println {:fast2 (bench1 #(freeze % {:no-header? true
|
||||
:compressor nil})
|
||||
#(thaw % {:no-header? true
|
||||
:compressor nil
|
||||
:encryptor nil}))}))
|
||||
|
||||
(println "\nDone! (Time for cake?)")
|
||||
true)
|
||||
|
||||
(comment (encore/read-edn (encore/pr-edn data))
|
||||
(comment (enc/read-edn (enc/pr-edn data))
|
||||
(bench1 fressian-freeze fressian-thaw))
|
||||
|
||||
(comment
|
||||
(set! *unchecked-math* false)
|
||||
;; (bench {:reader? true :lzma2? true :fressian? true :laps 3})
|
||||
;; (bench {:laps 4})
|
||||
;; (bench {:laps 1 :lzma2? true})
|
||||
;; (bench {:laps 2 :lzma2? true})
|
||||
;; (bench {:laps 2})
|
||||
|
||||
;;; 2015 Oct 6, v2.11.0-alpha4
|
||||
{:reader {:round 73409, :freeze 21823, :thaw 51586, :size 27672}}
|
||||
{:lzma2 {:round 56689, :freeze 37222, :thaw 19467, :size 11252}}
|
||||
{:fressian {:round 10666, :freeze 7737, :thaw 2929, :size 16985}}
|
||||
{:encrypted {:round 6885, :freeze 4227, :thaw 2658, :size 16148}}
|
||||
{:default {:round 6304, :freeze 3824, :thaw 2480, :size 16122}}
|
||||
{:fast1 {:round 5352, :freeze 3272, :thaw 2080, :size 16976}}
|
||||
{:fast2 {:round 5243, :freeze 3238, :thaw 2005, :size 16972}}
|
||||
;;
|
||||
{:reader {:round 26, :freeze 17, :thaw 9, :size 2}}
|
||||
{:lzma2 {:round 3648, :freeze 3150, :thaw 498, :size 68}}
|
||||
{:fressian {:round 19, :freeze 7, :thaw 12, :size 1}}
|
||||
{:encrypted {:round 63, :freeze 40, :thaw 23, :size 36}}
|
||||
{:default {:round 24, :freeze 17, :thaw 7, :size 6}}
|
||||
{:fast1 {:round 19, :freeze 12, :thaw 7, :size 6}}
|
||||
{:fast2 {:round 4, :freeze 2, :thaw 2, :size 2}}
|
||||
|
||||
;;; 2015 Sep 29, after read/write API refactor
|
||||
{:lzma2 {:round 51640, :freeze 33699, :thaw 17941, :size 11240}}
|
||||
{:encrypted {:round 5922, :freeze 3734, :thaw 2188, :size 16132}}
|
||||
{:default {:round 5588, :freeze 3658, :thaw 1930, :size 16113}}
|
||||
{:fast {:round 4533, :freeze 2688, :thaw 1845, :size 16972}}
|
||||
|
||||
;;; 2015 Sep 28, small collection optimizations
|
||||
{:lzma2 {:round 56307, :freeze 36475, :thaw 19832, :size 11244}}
|
||||
{:encrypted {:round 6062, :freeze 3802, :thaw 2260, :size 16148}}
|
||||
{:default {:round 5482, :freeze 3382, :thaw 2100, :size 16128}}
|
||||
{:fast {:round 4729, :freeze 2826, :thaw 1903, :size 16972}}
|
||||
|
||||
;;; 2015 Sep 29, various micro optimizations (incl. &arg elimination)
|
||||
{:reader {:round 63547, :freeze 19374, :thaw 44173, :size 27717}}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
(ns taoensso.nippy.compression
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [taoensso.encore :as encore])
|
||||
(:require [taoensso.encore :as enc])
|
||||
(:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream
|
||||
DataOutputStream]))
|
||||
|
||||
|
|
@ -122,10 +122,10 @@
|
|||
(comment
|
||||
(def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8"))
|
||||
(defn bench1 [compressor]
|
||||
{:time (encore/bench 10000 {:nlaps-warmup 10000}
|
||||
{:time (enc/bench 10000 {:nlaps-warmup 10000}
|
||||
(->> ba-bench (compress compressor) (decompress compressor)))
|
||||
:ratio (encore/round2 (/ (count (compress compressor ba-bench))
|
||||
(count ba-bench)))})
|
||||
:ratio (enc/round2 (/ (count (compress compressor ba-bench))
|
||||
(count ba-bench)))})
|
||||
|
||||
(println
|
||||
{:snappy (bench1 snappy-compressor)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
(ns taoensso.nippy.encryption
|
||||
"Simple no-nonsense crypto with reasonable defaults. Because your Clojure data
|
||||
deserves some privacy."
|
||||
"Simple no-nonsense crypto with reasonable defaults."
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [taoensso.encore :as encore]))
|
||||
(:require [taoensso.encore :as enc]))
|
||||
|
||||
;;;; Interface
|
||||
|
||||
(def standard-header-ids "These'll support :auto thaw." #{:aes128-sha512})
|
||||
(def standard-header-ids "These'll support :auto thaw" #{:aes128-sha512})
|
||||
|
||||
(defprotocol IEncryptor
|
||||
(header-id [encryptor])
|
||||
|
|
@ -16,15 +15,15 @@
|
|||
;;;; Default digests, ciphers, etc.
|
||||
|
||||
(def ^:private aes128-cipher*
|
||||
(encore/thread-local-proxy
|
||||
(enc/thread-local-proxy
|
||||
(javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding")))
|
||||
|
||||
(def ^:private sha512-md*
|
||||
(encore/thread-local-proxy
|
||||
(enc/thread-local-proxy
|
||||
(java.security.MessageDigest/getInstance "SHA-512")))
|
||||
|
||||
(def ^:private prng*
|
||||
(encore/thread-local-proxy
|
||||
(enc/thread-local-proxy
|
||||
(java.security.SecureRandom/getInstance "SHA1PRNG")))
|
||||
|
||||
(defn- aes128-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal aes128-cipher*))
|
||||
|
|
@ -40,11 +39,13 @@
|
|||
|
||||
(defn- sha512-key
|
||||
"SHA512-based key generator. Good JVM availability without extra dependencies
|
||||
(PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds."
|
||||
(PBKDF2, bcrypt, scrypt, etc.). Decent security when using many rounds."
|
||||
;; [salt-ba ^String pwd & [n]]
|
||||
[salt-ba ^String pwd]
|
||||
(let [md (sha512-md)]
|
||||
(loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")]
|
||||
(if salt-ba (encore/ba-concat salt-ba pwd-ba) pwd-ba))
|
||||
(if salt-ba (enc/ba-concat salt-ba pwd-ba) pwd-ba))
|
||||
;; n (or n (* (int Short/MAX_VALUE) (if salt-ba 5 64)))
|
||||
n (* (int Short/MAX_VALUE) (if salt-ba 5 64))]
|
||||
(if-not (zero? n)
|
||||
(recur (.digest md ba) (dec n))
|
||||
|
|
@ -76,32 +77,34 @@
|
|||
|
||||
(defrecord AES128Encryptor [key-gen key-cache]
|
||||
IEncryptor
|
||||
(header-id [_] (if (= key-gen sha512-key) :aes128-sha512 :aes128-other))
|
||||
(header-id [_] (if (identical? key-gen :sha512) :aes128-sha512 :aes128-other))
|
||||
(encrypt [_ typed-pwd data-ba]
|
||||
(let [[type pwd] (destructure-typed-pwd typed-pwd)
|
||||
salt? (identical? type :salted)
|
||||
iv-ba (rand-bytes aes128-block-size)
|
||||
salt-ba (when salt? (rand-bytes salt-size))
|
||||
prefix-ba (if-not salt? iv-ba (encore/ba-concat iv-ba salt-ba))
|
||||
prefix-ba (if-not salt? iv-ba (enc/ba-concat iv-ba salt-ba))
|
||||
key-gen (if (identical? key-gen :sha512) sha512-key key-gen)
|
||||
key (if salt?
|
||||
(key-gen salt-ba pwd)
|
||||
(encore/memoized key-cache key-gen salt-ba pwd))
|
||||
(enc/memoized key-cache key-gen salt-ba pwd))
|
||||
iv (javax.crypto.spec.IvParameterSpec. iv-ba)
|
||||
cipher (aes128-cipher)]
|
||||
(.init cipher javax.crypto.Cipher/ENCRYPT_MODE
|
||||
^javax.crypto.spec.SecretKeySpec key iv)
|
||||
(encore/ba-concat prefix-ba (.doFinal cipher data-ba))))
|
||||
(enc/ba-concat prefix-ba (.doFinal cipher data-ba))))
|
||||
|
||||
(decrypt [_ typed-pwd ba]
|
||||
(let [[type pwd] (destructure-typed-pwd typed-pwd)
|
||||
salt? (= type :salted)
|
||||
prefix-size (+ aes128-block-size (if salt? salt-size 0))
|
||||
[prefix-ba data-ba] (encore/ba-split ba prefix-size)
|
||||
[prefix-ba data-ba] (enc/ba-split ba prefix-size)
|
||||
[iv-ba salt-ba] (if-not salt? [prefix-ba nil]
|
||||
(encore/ba-split prefix-ba aes128-block-size))
|
||||
(enc/ba-split prefix-ba aes128-block-size))
|
||||
key-gen (if (identical? key-gen :sha512) sha512-key key-gen)
|
||||
key (if salt?
|
||||
(key-gen salt-ba pwd)
|
||||
(encore/memoized key-cache key-gen salt-ba pwd))
|
||||
(enc/memoized key-cache key-gen salt-ba pwd))
|
||||
iv (javax.crypto.spec.IvParameterSpec. iv-ba)
|
||||
cipher (aes128-cipher)]
|
||||
(.init cipher javax.crypto.Cipher/DECRYPT_MODE
|
||||
|
|
@ -109,7 +112,7 @@
|
|||
(.doFinal cipher data-ba))))
|
||||
|
||||
(def aes128-encryptor
|
||||
"Default 128bit AES encryptor with multi-round SHA-512 key-gen.
|
||||
"Default 128bit AES encryptor with many-round SHA-512 key-gen.
|
||||
|
||||
Password form [:salted \"my-password\"]
|
||||
---------------------------------------
|
||||
|
|
@ -143,7 +146,7 @@
|
|||
|
||||
Faster than `aes128-salted`, and harder to attack any particular key - but
|
||||
increased danger if a key is somehow compromised."
|
||||
(->AES128Encryptor sha512-key (atom {})))
|
||||
(->AES128Encryptor :sha512 (atom {})))
|
||||
|
||||
;;;; Default implementation
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
(ns taoensso.nippy.utils
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [clojure.string :as str]
|
||||
[taoensso.encore :as encore])
|
||||
[taoensso.encore :as enc])
|
||||
(:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable
|
||||
ObjectOutputStream ObjectInputStream]))
|
||||
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
cacheable? (not (re-find #"__\d+" (str t))) ; gensym form
|
||||
test (fn [] (try (f-test x) (catch Exception _ false)))]
|
||||
(if-not cacheable? (test)
|
||||
@(encore/swap-val! cache t #(if % % (delay (test)))))))))
|
||||
@(enc/swap-val! cache t #(if % % (delay (test)))))))))
|
||||
|
||||
(def serializable?
|
||||
(memoize-type-test
|
||||
|
|
@ -33,7 +33,7 @@
|
|||
(cast class object)
|
||||
true)))))
|
||||
|
||||
(def readable? (memoize-type-test (fn [x] (-> x encore/pr-edn encore/read-edn) true)))
|
||||
(def readable? (memoize-type-test (fn [x] (-> x enc/pr-edn enc/read-edn) true)))
|
||||
|
||||
(comment
|
||||
(serializable? "Hello world")
|
||||
|
|
|
|||
|
|
@ -1,169 +1,201 @@
|
|||
(ns taoensso.nippy.tests.main
|
||||
(:require [clojure.test.check :as check]
|
||||
[clojure.test.check.generators :as check-gen]
|
||||
[clojure.test.check.properties :as check-props]
|
||||
[expectations :as test :refer :all]
|
||||
[taoensso.nippy :as nippy :refer (freeze thaw)]
|
||||
[taoensso.nippy.benchmarks :as benchmarks]))
|
||||
(:require
|
||||
[clojure.test :as test :refer (is are deftest run-tests)]
|
||||
[clojure.test.check :as tc]
|
||||
[clojure.test.check.generators :as tc-gens]
|
||||
[clojure.test.check.properties :as tc-props]
|
||||
[taoensso.encore :as enc :refer ()]
|
||||
[taoensso.nippy :as nippy :refer (freeze thaw)]
|
||||
[taoensso.nippy.benchmarks :as benchmarks]))
|
||||
|
||||
(comment (test/run-tests '[taoensso.nippy.tests.main]))
|
||||
(comment (test/run-tests))
|
||||
|
||||
(def test-data nippy/stress-data-comparable)
|
||||
(defn- before-run {:expectations-options :before-run} [])
|
||||
(defn- after-run {:expectations-options :after-run} [])
|
||||
(def tc-num-tests 120)
|
||||
(def tc-gens
|
||||
"Like `tc-gens/any` but removes NaN (which breaks equality tests)"
|
||||
(tc-gens/recursive-gen tc-gens/container-type #_simple-type
|
||||
(tc-gens/one-of
|
||||
[tc-gens/int tc-gens/large-integer #_tc-gens/double
|
||||
(tc-gens/double* {:NaN? false})
|
||||
tc-gens/char tc-gens/string tc-gens/ratio tc-gens/boolean tc-gens/keyword
|
||||
tc-gens/keyword-ns tc-gens/symbol tc-gens/symbol-ns tc-gens/uuid])))
|
||||
|
||||
(comment (tc-gens/sample tc-gens 10))
|
||||
|
||||
;;;; Core
|
||||
|
||||
(expect (do (println (str "Clojure version: " *clojure-version*)) true))
|
||||
(deftest _core
|
||||
(is (do (println (str "Clojure version: " *clojure-version*)) true))
|
||||
(is (= test-data ((comp thaw freeze) test-data)))
|
||||
(is (= test-data ((comp #(thaw % {:no-header? true
|
||||
:compressor nippy/lz4-compressor
|
||||
:encryptor nil})
|
||||
#(freeze % {:no-header? true}))
|
||||
test-data)))
|
||||
|
||||
(expect test-data ((comp thaw freeze) test-data))
|
||||
(expect test-data ((comp #(thaw % {})
|
||||
#(freeze % {:legacy-mode true}))
|
||||
test-data))
|
||||
(expect test-data ((comp #(thaw % {:password [:salted "p"]})
|
||||
#(freeze % {:password [:salted "p"]}))
|
||||
test-data))
|
||||
(expect test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor})
|
||||
#(freeze % {:compressor nippy/lzma2-compressor}))
|
||||
test-data))
|
||||
(expect test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor
|
||||
:password [:salted "p"]})
|
||||
#(freeze % {:compressor nippy/lzma2-compressor
|
||||
:password [:salted "p"]}))
|
||||
test-data))
|
||||
(expect test-data ((comp #(thaw % {:compressor nippy/lz4-compressor})
|
||||
#(freeze % {:compressor nippy/lz4hc-compressor}))
|
||||
test-data))
|
||||
(is (= test-data ((comp #(thaw % {:password [:salted "p"]})
|
||||
#(freeze % {:password [:salted "p"]}))
|
||||
test-data)))
|
||||
|
||||
(expect ; Try roundtrip anything that simple-check can dream up
|
||||
(:result (check/quick-check 80 ; Time is n-non-linear
|
||||
(check-props/for-all [val check-gen/any]
|
||||
(= val (thaw (freeze val)))))))
|
||||
(is (= test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor})
|
||||
#(freeze % {:compressor nippy/lzma2-compressor}))
|
||||
test-data)))
|
||||
|
||||
;;; Trying to decrypt random (invalid) data can actually crash JVM
|
||||
;; (expect Exception (thaw (freeze test-data {:password "malformed"})))
|
||||
;; (expect Exception (thaw (freeze test-data {:password [:salted "p"]})))
|
||||
;; (expect Exception (thaw (freeze test-data {:password [:salted "p"]})
|
||||
;; {:compressor nil}))
|
||||
(is (= test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor
|
||||
:password [:salted "p"]})
|
||||
#(freeze % {:compressor nippy/lzma2-compressor
|
||||
:password [:salted "p"]}))
|
||||
test-data)))
|
||||
|
||||
(expect ; Snappy lib compatibility (for legacy versions of Nippy)
|
||||
(let [^bytes raw-ba (freeze test-data {:compressor nil})
|
||||
^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba)
|
||||
^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)]
|
||||
(= (thaw raw-ba)
|
||||
(thaw (org.xerial.snappy.Snappy/uncompress xerial-ba))
|
||||
(thaw (org.xerial.snappy.Snappy/uncompress iq80-ba))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))
|
||||
(is (= test-data ((comp #(thaw % {:compressor nippy/lz4-compressor})
|
||||
#(freeze % {:compressor nippy/lz4hc-compressor}))
|
||||
test-data)))
|
||||
|
||||
(is ; Try roundtrip anything that simple-check can dream up
|
||||
(:result (tc/quick-check tc-num-tests
|
||||
(tc-props/for-all [val tc-gens]
|
||||
(= val (thaw (freeze val)))))))
|
||||
|
||||
(is (thrown? Exception (thaw (freeze test-data {:password "malformed"}))))
|
||||
(is (thrown? Exception (thaw (freeze test-data {:password [:salted "p"]})
|
||||
{;; Necessary to prevent against JVM segfault due to
|
||||
;; https://goo.gl/t0OUIo:
|
||||
:v1-compatibility? false})))
|
||||
(is (thrown? Exception (thaw (freeze test-data {:password [:salted "p"]})
|
||||
{:v1-compatibility? false ; Ref. https://goo.gl/t0OUIo
|
||||
:compressor nil})))
|
||||
|
||||
(is ; Snappy lib compatibility (for legacy versions of Nippy)
|
||||
(let [^bytes raw-ba (freeze test-data {:compressor nil})
|
||||
^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba)
|
||||
^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)]
|
||||
(= (thaw raw-ba)
|
||||
(thaw (org.xerial.snappy.Snappy/uncompress xerial-ba))
|
||||
(thaw (org.xerial.snappy.Snappy/uncompress iq80-ba))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
|
||||
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))))
|
||||
|
||||
;;;; Custom types & records
|
||||
|
||||
;;; Extend to custom Type
|
||||
(defrecord MyType [data])
|
||||
(expect Exception (do (nippy/extend-freeze MyType 1 [x s] (.writeUTF s (:data x)))
|
||||
(thaw (freeze (->MyType "val")))))
|
||||
(expect (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s)))
|
||||
(let [type (->MyType "val")] (= type (thaw (freeze type))))))
|
||||
(deftype MyType [data])
|
||||
(defrecord MyRec [data])
|
||||
|
||||
;;; Extend to custom Record
|
||||
(defrecord MyRec [data])
|
||||
(expect (do (nippy/extend-freeze MyRec 2 [x s] (.writeUTF s (str "foo-" (:data x))))
|
||||
(nippy/extend-thaw 2 [s] (->MyRec (.readUTF s)))
|
||||
(= (->MyRec "foo-val") (thaw (freeze (->MyRec "val"))))))
|
||||
(deftest _types
|
||||
;;; Extend to custom Type
|
||||
(is (thrown? Exception ; No thaw extension yet
|
||||
(do (nippy/swap-custom-readers! (constantly {}))
|
||||
(nippy/extend-freeze MyType 1 [x s] (.writeUTF s (.data x)))
|
||||
(thaw (freeze (->MyType "val"))))))
|
||||
(is (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s)))
|
||||
(let [mt (->MyType "val")] (= (.data ^MyType mt)
|
||||
(.data ^MyType (thaw (freeze mt)))))))
|
||||
|
||||
;;; Keyword (prefixed) extensions
|
||||
(expect
|
||||
(do (nippy/extend-freeze MyType :nippy-tests/MyType [x s] (.writeUTF s (:data x)))
|
||||
(nippy/extend-thaw :nippy-tests/MyType [s] (->MyType (.readUTF s)))
|
||||
(let [type (->MyType "val")] (= type (thaw (freeze type))))))
|
||||
;;; Extend to custom Record
|
||||
(is (do (nippy/extend-freeze MyRec 2 [x s] (.writeUTF s (str "foo-" (:data x))))
|
||||
(nippy/extend-thaw 2 [s] (->MyRec (.readUTF s)))
|
||||
(= (->MyRec "foo-val") (thaw (freeze (->MyRec "val"))))))
|
||||
|
||||
;;; Keyword (prefixed) extensions
|
||||
(is
|
||||
(do (nippy/extend-freeze MyRec :nippy-tests/MyRec [x s] (.writeUTF s (:data x)))
|
||||
(nippy/extend-thaw :nippy-tests/MyRec [s] (->MyRec (.readUTF s)))
|
||||
(let [mr (->MyRec "val")] (= mr (thaw (freeze mr)))))))
|
||||
|
||||
;;;; Stable binary representation of vals
|
||||
|
||||
(expect (seq (freeze test-data))
|
||||
(seq (freeze test-data))) ; f(x)=f(y) | x=y
|
||||
(deftest _stable-bin
|
||||
|
||||
;; As above, but try multiple times to catch possible protocol interface races:
|
||||
(expect #(every? true? %)
|
||||
(is (= (seq (freeze test-data))
|
||||
(seq (freeze test-data)))) ; f(x)=f(y) | x=y
|
||||
|
||||
;; As above, but try multiple times to catch possible protocol interface races:
|
||||
(is (every? true?
|
||||
(repeatedly 1000 (fn [] (= (seq (freeze test-data))
|
||||
(seq (freeze test-data))))))
|
||||
(seq (freeze test-data)))))))
|
||||
|
||||
;; NB abandoning - no way to do this reliably w/o appropriate contracts from
|
||||
;; (seq <unordered-coll>):
|
||||
;;
|
||||
;; (expect (seq (-> test-data freeze)) ; f(x)=f(f-1(f(x)))
|
||||
;; (seq (-> test-data freeze thaw freeze)))
|
||||
;;
|
||||
;; As above, but with repeated refreeze to catch possible protocol interface races:
|
||||
;; (expect (= (seq (freeze test-data))
|
||||
;; (seq (reduce (fn [frozen _] (freeze (thaw frozen)))
|
||||
;; (freeze test-data) (range 1000)))))
|
||||
;; NB abandoning - no way to do this reliably w/o appropriate contracts from
|
||||
;; (seq <unordered-coll>):
|
||||
;;
|
||||
;; (is (= (seq (-> test-data freeze))
|
||||
;; (seq (-> test-data freeze thaw freeze)))) ; f(x)=f(f-1(f(x)))
|
||||
;;
|
||||
;; As above, but with repeated refreeze to catch possible protocol interface races:
|
||||
;; (is (= (seq (freeze test-data))
|
||||
;; (seq (reduce (fn [frozen _] (freeze (thaw frozen)))
|
||||
;; (freeze test-data) (range 1000)))))
|
||||
)
|
||||
|
||||
(defn qc-prop-bijection [& [n]]
|
||||
(let [bin->val (atom {})
|
||||
val->bin (atom {})]
|
||||
(merge
|
||||
(check/quick-check (or n 1)
|
||||
(check-props/for-all [val check-gen/any #_check-gen/any-printable]
|
||||
(let [;; Nb need `seq` for Clojure hash equality:
|
||||
bin (hash (seq (freeze val)))]
|
||||
(and
|
||||
(if (contains? val->bin val)
|
||||
(= (get val->bin val) bin) ; x=y => f(x)=f(y) by clj=
|
||||
(do (swap! val->bin assoc val bin)
|
||||
true))
|
||||
(tc/quick-check (or n 1)
|
||||
(tc-props/for-all [val tc-gens]
|
||||
(let [;; Nb need `seq` for Clojure hash equality:
|
||||
bin (hash (seq (freeze val)))]
|
||||
(and
|
||||
(if (contains? val->bin val)
|
||||
(= (get val->bin val) bin) ; x=y => f(x)=f(y) by clj=
|
||||
(do (swap! val->bin assoc val bin)
|
||||
true))
|
||||
|
||||
(if (contains? bin->val bin)
|
||||
(= (get bin->val bin) val) ; f(x)=f(y) => x=y by clj=
|
||||
(do (swap! bin->val assoc bin val)
|
||||
true))))))
|
||||
#_{:bin->val @bin->val
|
||||
:val->bin @val->bin}
|
||||
nil)))
|
||||
(if (contains? bin->val bin)
|
||||
(= (get bin->val bin) val) ; f(x)=f(y) => x=y by clj=
|
||||
(do (swap! bin->val assoc bin val)
|
||||
true))))))
|
||||
#_{:bin->val @bin->val
|
||||
:val->bin @val->bin}
|
||||
nil)))
|
||||
|
||||
(comment
|
||||
(check-gen/sample check-gen/any 10)
|
||||
(tc-gens/sample tc-gens 10)
|
||||
(:result (qc-prop-bijection 80))
|
||||
(let [{:keys [result bin->val val->bin]} (qc-prop-bijection 10)]
|
||||
[result (vals bin->val)]))
|
||||
|
||||
(expect #(:result %) (qc-prop-bijection 80))
|
||||
(deftest _gc-prop-bijection
|
||||
(is (:result (qc-prop-bijection tc-num-tests))))
|
||||
|
||||
;;;; Thread safety
|
||||
|
||||
;; Not sure why, but record equality test fails in futures:
|
||||
(def test-data-threaded (dissoc nippy/stress-data-comparable :stress-record))
|
||||
|
||||
(expect
|
||||
(let [futures
|
||||
(mapv
|
||||
(fn [_]
|
||||
(future
|
||||
(= (thaw (freeze test-data-threaded)) test-data-threaded)))
|
||||
(range 50))]
|
||||
(every? deref futures)))
|
||||
(deftest _thread-safe
|
||||
(is
|
||||
(let [futures
|
||||
(mapv
|
||||
(fn [_]
|
||||
(future
|
||||
(= (thaw (freeze test-data-threaded)) test-data-threaded)))
|
||||
(range 50))]
|
||||
(every? deref futures)))
|
||||
|
||||
(expect
|
||||
(let [futures
|
||||
(mapv
|
||||
(fn [_]
|
||||
(future
|
||||
(= (thaw (freeze test-data-threaded {:password [:salted "password"]})
|
||||
{:password [:salted "password"]})
|
||||
test-data-threaded)))
|
||||
(range 50))]
|
||||
(every? deref futures)))
|
||||
(is
|
||||
(let [futures
|
||||
(mapv
|
||||
(fn [_]
|
||||
(future
|
||||
(= (thaw (freeze test-data-threaded {:password [:salted "password"]})
|
||||
{:password [:salted "password"]})
|
||||
test-data-threaded)))
|
||||
(range 50))]
|
||||
(every? deref futures)))
|
||||
|
||||
(expect
|
||||
(let [futures
|
||||
(mapv
|
||||
(fn [_]
|
||||
(future
|
||||
(= (thaw (freeze test-data-threaded {:password [:cached "password"]})
|
||||
{:password [:cached "password"]})
|
||||
test-data-threaded)))
|
||||
(range 50))]
|
||||
(every? deref futures)))
|
||||
(is
|
||||
(let [futures
|
||||
(mapv
|
||||
(fn [_]
|
||||
(future
|
||||
(= (thaw (freeze test-data-threaded {:password [:cached "password"]})
|
||||
{:password [:cached "password"]})
|
||||
test-data-threaded)))
|
||||
(range 50))]
|
||||
(every? deref futures))))
|
||||
|
||||
;;;; Benchmarks
|
||||
|
||||
(expect (benchmarks/bench {})) ; Also tests :cached passwords
|
||||
(deftest _benchmarks
|
||||
(is (benchmarks/bench {})) ; Also tests :cached passwords
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue