luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'dev'

Browse source
This commit is contained in:
Peter Taoussanis 2015-12-01 16:40:57 +07:00
commit 6b2b501589
9 changed files with 806 additions and 677 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
CHANGELOG.md
View file

@ -1,5 +1,23 @@
> This project uses [Break Versioning](https://github.com/ptaoussanis/encore/blob/master/BREAK-VERSIONING.md) as of **Aug 16, 2014**. > This project uses [Break Versioning](https://github.com/ptaoussanis/encore/blob/master/BREAK-VERSIONING.md) as of **Aug 16, 2014**.
## v2.11.0-alpha1 / 2015 Dec 1
> This is a major performance release that **drops default support for thawing Nippy v1 archives** but is otherwise non-breaking
* **BREAKING**: `thaw` now has `:v1-compatibility?` opt set to false by default (was true before) [1]
* **Performance**: optimize serialized size of small maps, sets, vectors, bytes
* **Performance**: optimized (no copy) `freeze` when using no compression or encryption
* **Implementation**: swap most macros for fns (make low-level utils easier to use)
```clojure
[com.taoensso/nippy "2.11.0-alpha1"]
```
#### Notes
**[1]** Use `(thaw <frozen-byte-array> {:v1-compatibility? true})` to support thawing of data frozen with Nippy v1 (before ~June 2013)
## v2.10.0 / 2015 Sep 30 ## v2.10.0 / 2015 Sep 30
> This is a major feature/performance release that **drops support for Clojure 1.4** but is otherwise non-breaking > This is a major feature/performance release that **drops support for Clojure 1.4** but is otherwise non-breaking

3
README.md
View file

@ -1,7 +1,8 @@
**[API docs][]** | **[CHANGELOG][]** | [other Clojure libs][] | [Twitter][] | [contact/contrib](#contact--contributing) | current [Break Version][]: **[API docs][]** | **[CHANGELOG][]** | [other Clojure libs][] | [Twitter][] | [contact/contrib](#contact--contributing) | current [Break Version][]:
```clojure ```clojure
[com.taoensso/nippy "2.10.0"] ; Stable, see CHANGELOG for details [com.taoensso/nippy "2.10.0"] ; Stable
[com.taoensso/nippy "2.11.0-alpha1"] ; Dev, see CHANGELOG for details
``` ```
# Nippy, a Clojure serialization library # Nippy, a Clojure serialization library

29
project.clj
View file

@ -1,4 +1,4 @@
(defproject com.taoensso/nippy "2.10.0" (defproject com.taoensso/nippy "2.11.0-alpha1"
:author "Peter Taoussanis <https://www.taoensso.com>" :author "Peter Taoussanis <https://www.taoensso.com>"
:description "Clojure serialization library" :description "Clojure serialization library"
:url "https://github.com/ptaoussanis/nippy" :url "https://github.com/ptaoussanis/nippy"
@ -9,12 +9,13 @@
:min-lein-version "2.3.3" :min-lein-version "2.3.3"
:global-vars {*warn-on-reflection* true :global-vars {*warn-on-reflection* true
*assert* true *assert* true
*unchecked-math* :warn-on-boxed} ;; *unchecked-math* :warn-on-boxed
}
:dependencies :dependencies
[[org.clojure/clojure "1.5.1"] [[org.clojure/clojure "1.5.1"]
[org.clojure/tools.reader "0.9.2"] [org.clojure/tools.reader "0.10.0"]
[com.taoensso/encore "2.18.0"] [com.taoensso/encore "2.26.1"]
[org.iq80.snappy/snappy "0.4"] [org.iq80.snappy/snappy "0.4"]
[org.tukaani/xz "1.5"] [org.tukaani/xz "1.5"]
[net.jpountz.lz4/lz4 "1.3"]] [net.jpountz.lz4/lz4 "1.3"]]
@ -25,25 +26,25 @@
:1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]}
:1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]}
:1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} :1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]}
:1.8 {:dependencies [[org.clojure/clojure "1.8.0-alpha5"]]} :1.8 {:dependencies [[org.clojure/clojure "1.8.0-RC2"]]}
:test {:jvm-opts ["-Xms1024m" "-Xmx2048m"] :test {:jvm-opts ["-Xms1024m" "-Xmx2048m"]
:dependencies [[expectations "2.1.1"] :dependencies [[org.clojure/test.check "0.9.0"]
[org.clojure/test.check "0.8.2"]
[org.clojure/data.fressian "0.2.1"] [org.clojure/data.fressian "0.2.1"]
[org.xerial.snappy/snappy-java "1.1.2"]]} [org.xerial.snappy/snappy-java "1.1.2"]]}
:dev [:1.7 :test :dev [:1.7 :test
{:plugins {:plugins
[[lein-pprint "1.1.1"] [[lein-pprint "1.1.2"]
[lein-ancient "0.6.7"] [lein-ancient "0.6.8"]
[lein-expectations "0.0.8"] [lein-codox "0.9.0"]]}]}
[lein-autoexpect "1.2.2"]
[codox "0.8.10"]]}]}
:test-paths ["test" "src"] :test-paths ["test" "src"]
:codox
{:language :clojure ; [:clojure :clojurescript] ; No support?
:source-uri "https://github.com/ptaoussanis/nippy/blob/master/{filepath}#L{line}"}
:aliases :aliases
{"test-all" ["with-profile" "+1.5:+1.6:+1.7:+1.8" "expectations"] {"test-all" ["with-profile" "+1.5:+1.6:+1.7:+1.8" "test"]
"test-auto" ["with-profile" "+test" "autoexpect"]
"deploy-lib" ["do" "deploy" "clojars," "install"] "deploy-lib" ["do" "deploy" "clojars," "install"]
"start-dev" ["with-profile" "+server-jvm" "repl" ":headless"]} "start-dev" ["with-profile" "+server-jvm" "repl" ":headless"]}

793
src/taoensso/nippy.clj
View file

File diff suppressed because it is too large Load diff

67
src/taoensso/nippy/benchmarks.clj
View file

@ -1,10 +1,10 @@
(ns taoensso.nippy.benchmarks (ns taoensso.nippy.benchmarks
{:author "Peter Taoussanis"} {:author "Peter Taoussanis"}
(:require [clojure.data.fressian :as fressian] (:require [clojure.data.fressian :as fressian]
[taoensso.encore :as encore] [taoensso.encore :as enc]
[taoensso.nippy :as nippy :refer (freeze thaw)])) [taoensso.nippy :as nippy :refer (freeze thaw)]))
(def data nippy/stress-data-benchable) (def data #_22 nippy/stress-data-benchable)
(defn fressian-freeze [value] (defn fressian-freeze [value]
(let [^java.nio.ByteBuffer bb (fressian/write value) (let [^java.nio.ByteBuffer bb (fressian/write value)
@ -19,7 +19,7 @@
(comment (fressian-thaw (fressian-freeze data))) (comment (fressian-thaw (fressian-freeze data)))
(defmacro bench* [& body] `(encore/bench 10000 {:warmup-laps 20000} ~@body)) (defmacro bench* [& body] `(enc/bench 10000 {:warmup-laps 20000} ~@body))
(defn bench1 [freezer thawer & [sizer]] (defn bench1 [freezer thawer & [sizer]]
(let [data-frozen (freezer data) (let [data-frozen (freezer data)
time-freeze (bench* (freezer data)) time-freeze (bench* (freezer data))
@ -36,36 +36,69 @@
(println (str "\nLap " (inc l) "/" laps "...")) (println (str "\nLap " (inc l) "/" laps "..."))
(when reader? ; Slow (when reader? ; Slow
(println {:reader (bench1 encore/pr-edn encore/read-edn (println {:reader (bench1 enc/pr-edn enc/read-edn
#(count (.getBytes ^String % "UTF-8")))})) #(count (.getBytes ^String % "UTF-8")))}))
(println {:default (bench1 #(freeze % {}) (when lzma2? ; Slow
#(thaw % {}))})
(println {:fast (bench1 #(freeze % {:compressor nil
:skip-header? true})
#(thaw % {:compressor nil
:encryptor nil}))})
(println {:encrypted (bench1 #(freeze % {:password [:cached "p"]})
#(thaw % {:password [:cached "p"]}))})
(when lzma2? ; Slow as molasses
(println {:lzma2 (bench1 #(freeze % {:compressor nippy/lzma2-compressor}) (println {:lzma2 (bench1 #(freeze % {:compressor nippy/lzma2-compressor})
#(thaw % {:compressor nippy/lzma2-compressor}))})) #(thaw % {:compressor nippy/lzma2-compressor}))}))
(when fressian? (when fressian?
(println {:fressian (bench1 fressian-freeze fressian-thaw)}))) (println {:fressian (bench1 fressian-freeze fressian-thaw)}))
(println {:encrypted (bench1 #(freeze % {:password [:cached "p"]})
#(thaw % {:password [:cached "p"]}))})
(println {:default (bench1 #(freeze % {})
#(thaw % {}))})
(println {:fast1 (bench1 #(freeze % {:compressor nil})
#(thaw % {:compressor nil}))})
(println {:fast2 (bench1 #(freeze % {:no-header? true
:compressor nil})
#(thaw % {:no-header? true
:compressor nil
:encryptor nil}))}))
(println "\nDone! (Time for cake?)") (println "\nDone! (Time for cake?)")
true) true)
(comment (encore/read-edn (encore/pr-edn data)) (comment (enc/read-edn (enc/pr-edn data))
(bench1 fressian-freeze fressian-thaw)) (bench1 fressian-freeze fressian-thaw))
(comment (comment
(set! *unchecked-math* false) (set! *unchecked-math* false)
;; (bench {:reader? true :lzma2? true :fressian? true :laps 3}) ;; (bench {:reader? true :lzma2? true :fressian? true :laps 3})
;; (bench {:laps 4}) ;; (bench {:laps 4})
;; (bench {:laps 1 :lzma2? true}) ;; (bench {:laps 2 :lzma2? true})
;; (bench {:laps 2})
;;; 2015 Oct 6, v2.11.0-alpha4
{:reader {:round 73409, :freeze 21823, :thaw 51586, :size 27672}}
{:lzma2 {:round 56689, :freeze 37222, :thaw 19467, :size 11252}}
{:fressian {:round 10666, :freeze 7737, :thaw 2929, :size 16985}}
{:encrypted {:round 6885, :freeze 4227, :thaw 2658, :size 16148}}
{:default {:round 6304, :freeze 3824, :thaw 2480, :size 16122}}
{:fast1 {:round 5352, :freeze 3272, :thaw 2080, :size 16976}}
{:fast2 {:round 5243, :freeze 3238, :thaw 2005, :size 16972}}
;;
{:reader {:round 26, :freeze 17, :thaw 9, :size 2}}
{:lzma2 {:round 3648, :freeze 3150, :thaw 498, :size 68}}
{:fressian {:round 19, :freeze 7, :thaw 12, :size 1}}
{:encrypted {:round 63, :freeze 40, :thaw 23, :size 36}}
{:default {:round 24, :freeze 17, :thaw 7, :size 6}}
{:fast1 {:round 19, :freeze 12, :thaw 7, :size 6}}
{:fast2 {:round 4, :freeze 2, :thaw 2, :size 2}}
;;; 2015 Sep 29, after read/write API refactor
{:lzma2 {:round 51640, :freeze 33699, :thaw 17941, :size 11240}}
{:encrypted {:round 5922, :freeze 3734, :thaw 2188, :size 16132}}
{:default {:round 5588, :freeze 3658, :thaw 1930, :size 16113}}
{:fast {:round 4533, :freeze 2688, :thaw 1845, :size 16972}}
;;; 2015 Sep 28, small collection optimizations
{:lzma2 {:round 56307, :freeze 36475, :thaw 19832, :size 11244}}
{:encrypted {:round 6062, :freeze 3802, :thaw 2260, :size 16148}}
{:default {:round 5482, :freeze 3382, :thaw 2100, :size 16128}}
{:fast {:round 4729, :freeze 2826, :thaw 1903, :size 16972}}
;;; 2015 Sep 29, various micro optimizations (incl. &arg elimination) ;;; 2015 Sep 29, various micro optimizations (incl. &arg elimination)
{:reader {:round 63547, :freeze 19374, :thaw 44173, :size 27717}} {:reader {:round 63547, :freeze 19374, :thaw 44173, :size 27717}}

6
src/taoensso/nippy/compression.clj
View file

@ -1,6 +1,6 @@
(ns taoensso.nippy.compression (ns taoensso.nippy.compression
{:author "Peter Taoussanis"} {:author "Peter Taoussanis"}
(:require [taoensso.encore :as encore]) (:require [taoensso.encore :as enc])
(:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream (:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream
DataOutputStream])) DataOutputStream]))
@ -122,9 +122,9 @@
(comment (comment
(def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8")) (def ba-bench (.getBytes (apply str (repeatedly 1000 rand)) "UTF-8"))
(defn bench1 [compressor] (defn bench1 [compressor]
{:time (encore/bench 10000 {:nlaps-warmup 10000} {:time (enc/bench 10000 {:nlaps-warmup 10000}
(->> ba-bench (compress compressor) (decompress compressor))) (->> ba-bench (compress compressor) (decompress compressor)))
:ratio (encore/round2 (/ (count (compress compressor ba-bench)) :ratio (enc/round2 (/ (count (compress compressor ba-bench))
(count ba-bench)))}) (count ba-bench)))})
(println (println

39
src/taoensso/nippy/encryption.clj
View file

@ -1,12 +1,11 @@
(ns taoensso.nippy.encryption (ns taoensso.nippy.encryption
"Simple no-nonsense crypto with reasonable defaults. Because your Clojure data "Simple no-nonsense crypto with reasonable defaults."
deserves some privacy."
{:author "Peter Taoussanis"} {:author "Peter Taoussanis"}
(:require [taoensso.encore :as encore])) (:require [taoensso.encore :as enc]))
;;;; Interface ;;;; Interface
(def standard-header-ids "These'll support :auto thaw." #{:aes128-sha512}) (def standard-header-ids "These'll support :auto thaw" #{:aes128-sha512})
(defprotocol IEncryptor (defprotocol IEncryptor
(header-id [encryptor]) (header-id [encryptor])
@ -16,15 +15,15 @@
;;;; Default digests, ciphers, etc. ;;;; Default digests, ciphers, etc.
(def ^:private aes128-cipher* (def ^:private aes128-cipher*
(encore/thread-local-proxy (enc/thread-local-proxy
(javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding"))) (javax.crypto.Cipher/getInstance "AES/CBC/PKCS5Padding")))
(def ^:private sha512-md* (def ^:private sha512-md*
(encore/thread-local-proxy (enc/thread-local-proxy
(java.security.MessageDigest/getInstance "SHA-512"))) (java.security.MessageDigest/getInstance "SHA-512")))
(def ^:private prng* (def ^:private prng*
(encore/thread-local-proxy (enc/thread-local-proxy
(java.security.SecureRandom/getInstance "SHA1PRNG"))) (java.security.SecureRandom/getInstance "SHA1PRNG")))
(defn- aes128-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal aes128-cipher*)) (defn- aes128-cipher ^javax.crypto.Cipher [] (.get ^ThreadLocal aes128-cipher*))
@ -40,11 +39,13 @@
(defn- sha512-key (defn- sha512-key
"SHA512-based key generator. Good JVM availability without extra dependencies "SHA512-based key generator. Good JVM availability without extra dependencies
(PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds." (PBKDF2, bcrypt, scrypt, etc.). Decent security when using many rounds."
;; [salt-ba ^String pwd & [n]]
[salt-ba ^String pwd] [salt-ba ^String pwd]
(let [md (sha512-md)] (let [md (sha512-md)]
(loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")] (loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")]
(if salt-ba (encore/ba-concat salt-ba pwd-ba) pwd-ba)) (if salt-ba (enc/ba-concat salt-ba pwd-ba) pwd-ba))
;; n (or n (* (int Short/MAX_VALUE) (if salt-ba 5 64)))
n (* (int Short/MAX_VALUE) (if salt-ba 5 64))] n (* (int Short/MAX_VALUE) (if salt-ba 5 64))]
(if-not (zero? n) (if-not (zero? n)
(recur (.digest md ba) (dec n)) (recur (.digest md ba) (dec n))
@ -76,32 +77,34 @@
(defrecord AES128Encryptor [key-gen key-cache] (defrecord AES128Encryptor [key-gen key-cache]
IEncryptor IEncryptor
(header-id [_] (if (= key-gen sha512-key) :aes128-sha512 :aes128-other)) (header-id [_] (if (identical? key-gen :sha512) :aes128-sha512 :aes128-other))
(encrypt [_ typed-pwd data-ba] (encrypt [_ typed-pwd data-ba]
(let [[type pwd] (destructure-typed-pwd typed-pwd) (let [[type pwd] (destructure-typed-pwd typed-pwd)
salt? (identical? type :salted) salt? (identical? type :salted)
iv-ba (rand-bytes aes128-block-size) iv-ba (rand-bytes aes128-block-size)
salt-ba (when salt? (rand-bytes salt-size)) salt-ba (when salt? (rand-bytes salt-size))
prefix-ba (if-not salt? iv-ba (encore/ba-concat iv-ba salt-ba)) prefix-ba (if-not salt? iv-ba (enc/ba-concat iv-ba salt-ba))
key-gen (if (identical? key-gen :sha512) sha512-key key-gen)
key (if salt? key (if salt?
(key-gen salt-ba pwd) (key-gen salt-ba pwd)
(encore/memoized key-cache key-gen salt-ba pwd)) (enc/memoized key-cache key-gen salt-ba pwd))
iv (javax.crypto.spec.IvParameterSpec. iv-ba) iv (javax.crypto.spec.IvParameterSpec. iv-ba)
cipher (aes128-cipher)] cipher (aes128-cipher)]
(.init cipher javax.crypto.Cipher/ENCRYPT_MODE (.init cipher javax.crypto.Cipher/ENCRYPT_MODE
^javax.crypto.spec.SecretKeySpec key iv) ^javax.crypto.spec.SecretKeySpec key iv)
(encore/ba-concat prefix-ba (.doFinal cipher data-ba)))) (enc/ba-concat prefix-ba (.doFinal cipher data-ba))))
(decrypt [_ typed-pwd ba] (decrypt [_ typed-pwd ba]
(let [[type pwd] (destructure-typed-pwd typed-pwd) (let [[type pwd] (destructure-typed-pwd typed-pwd)
salt? (= type :salted) salt? (= type :salted)
prefix-size (+ aes128-block-size (if salt? salt-size 0)) prefix-size (+ aes128-block-size (if salt? salt-size 0))
[prefix-ba data-ba] (encore/ba-split ba prefix-size) [prefix-ba data-ba] (enc/ba-split ba prefix-size)
[iv-ba salt-ba] (if-not salt? [prefix-ba nil] [iv-ba salt-ba] (if-not salt? [prefix-ba nil]
(encore/ba-split prefix-ba aes128-block-size)) (enc/ba-split prefix-ba aes128-block-size))
key-gen (if (identical? key-gen :sha512) sha512-key key-gen)
key (if salt? key (if salt?
(key-gen salt-ba pwd) (key-gen salt-ba pwd)
(encore/memoized key-cache key-gen salt-ba pwd)) (enc/memoized key-cache key-gen salt-ba pwd))
iv (javax.crypto.spec.IvParameterSpec. iv-ba) iv (javax.crypto.spec.IvParameterSpec. iv-ba)
cipher (aes128-cipher)] cipher (aes128-cipher)]
(.init cipher javax.crypto.Cipher/DECRYPT_MODE (.init cipher javax.crypto.Cipher/DECRYPT_MODE
@ -109,7 +112,7 @@
(.doFinal cipher data-ba)))) (.doFinal cipher data-ba))))
(def aes128-encryptor (def aes128-encryptor
"Default 128bit AES encryptor with multi-round SHA-512 key-gen. "Default 128bit AES encryptor with many-round SHA-512 key-gen.
Password form [:salted \"my-password\"] Password form [:salted \"my-password\"]
--------------------------------------- ---------------------------------------
@ -143,7 +146,7 @@
Faster than `aes128-salted`, and harder to attack any particular key - but Faster than `aes128-salted`, and harder to attack any particular key - but
increased danger if a key is somehow compromised." increased danger if a key is somehow compromised."
(->AES128Encryptor sha512-key (atom {}))) (->AES128Encryptor :sha512 (atom {})))
;;;; Default implementation ;;;; Default implementation

6
src/taoensso/nippy/utils.clj
View file

@ -1,7 +1,7 @@
(ns taoensso.nippy.utils (ns taoensso.nippy.utils
{:author "Peter Taoussanis"} {:author "Peter Taoussanis"}
(:require [clojure.string :as str] (:require [clojure.string :as str]
[taoensso.encore :as encore]) [taoensso.encore :as enc])
(:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable (:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable
ObjectOutputStream ObjectInputStream])) ObjectOutputStream ObjectInputStream]))
@ -17,7 +17,7 @@
cacheable? (not (re-find #"__\d+" (str t))) ; gensym form cacheable? (not (re-find #"__\d+" (str t))) ; gensym form
test (fn [] (try (f-test x) (catch Exception _ false)))] test (fn [] (try (f-test x) (catch Exception _ false)))]
(if-not cacheable? (test) (if-not cacheable? (test)
@(encore/swap-val! cache t #(if % % (delay (test))))))))) @(enc/swap-val! cache t #(if % % (delay (test)))))))))
(def serializable? (def serializable?
(memoize-type-test (memoize-type-test
@ -33,7 +33,7 @@
(cast class object) (cast class object)
true))))) true)))))
(def readable? (memoize-type-test (fn [x] (-> x encore/pr-edn encore/read-edn) true))) (def readable? (memoize-type-test (fn [x] (-> x enc/pr-edn enc/read-edn) true)))
(comment (comment
(serializable? "Hello world") (serializable? "Hello world")

170
test/taoensso/nippy/tests/main.clj
View file

@ -1,52 +1,72 @@
(ns taoensso.nippy.tests.main (ns taoensso.nippy.tests.main
(:require [clojure.test.check :as check] (:require
[clojure.test.check.generators :as check-gen] [clojure.test :as test :refer (is are deftest run-tests)]
[clojure.test.check.properties :as check-props] [clojure.test.check :as tc]
[expectations :as test :refer :all] [clojure.test.check.generators :as tc-gens]
[clojure.test.check.properties :as tc-props]
[taoensso.encore :as enc :refer ()]
[taoensso.nippy :as nippy :refer (freeze thaw)] [taoensso.nippy :as nippy :refer (freeze thaw)]
[taoensso.nippy.benchmarks :as benchmarks])) [taoensso.nippy.benchmarks :as benchmarks]))
(comment (test/run-tests '[taoensso.nippy.tests.main])) (comment (test/run-tests))
(def test-data nippy/stress-data-comparable) (def test-data nippy/stress-data-comparable)
(defn- before-run {:expectations-options :before-run} []) (def tc-num-tests 120)
(defn- after-run {:expectations-options :after-run} []) (def tc-gens
"Like `tc-gens/any` but removes NaN (which breaks equality tests)"
(tc-gens/recursive-gen tc-gens/container-type #_simple-type
(tc-gens/one-of
[tc-gens/int tc-gens/large-integer #_tc-gens/double
(tc-gens/double* {:NaN? false})
tc-gens/char tc-gens/string tc-gens/ratio tc-gens/boolean tc-gens/keyword
tc-gens/keyword-ns tc-gens/symbol tc-gens/symbol-ns tc-gens/uuid])))
(comment (tc-gens/sample tc-gens 10))
;;;; Core ;;;; Core
(expect (do (println (str "Clojure version: " *clojure-version*)) true)) (deftest _core
(is (do (println (str "Clojure version: " *clojure-version*)) true))
(is (= test-data ((comp thaw freeze) test-data)))
(is (= test-data ((comp #(thaw % {:no-header? true
:compressor nippy/lz4-compressor
:encryptor nil})
#(freeze % {:no-header? true}))
test-data)))
(expect test-data ((comp thaw freeze) test-data)) (is (= test-data ((comp #(thaw % {:password [:salted "p"]})
(expect test-data ((comp #(thaw % {})
#(freeze % {:legacy-mode true}))
test-data))
(expect test-data ((comp #(thaw % {:password [:salted "p"]})
#(freeze % {:password [:salted "p"]})) #(freeze % {:password [:salted "p"]}))
test-data)) test-data)))
(expect test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor})
(is (= test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor})
#(freeze % {:compressor nippy/lzma2-compressor})) #(freeze % {:compressor nippy/lzma2-compressor}))
test-data)) test-data)))
(expect test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor
(is (= test-data ((comp #(thaw % {:compressor nippy/lzma2-compressor
:password [:salted "p"]}) :password [:salted "p"]})
#(freeze % {:compressor nippy/lzma2-compressor #(freeze % {:compressor nippy/lzma2-compressor
:password [:salted "p"]})) :password [:salted "p"]}))
test-data)) test-data)))
(expect test-data ((comp #(thaw % {:compressor nippy/lz4-compressor})
#(freeze % {:compressor nippy/lz4hc-compressor}))
test-data))
(expect ; Try roundtrip anything that simple-check can dream up (is (= test-data ((comp #(thaw % {:compressor nippy/lz4-compressor})
(:result (check/quick-check 80 ; Time is n-non-linear #(freeze % {:compressor nippy/lz4hc-compressor}))
(check-props/for-all [val check-gen/any] test-data)))
(is ; Try roundtrip anything that simple-check can dream up
(:result (tc/quick-check tc-num-tests
(tc-props/for-all [val tc-gens]
(= val (thaw (freeze val))))))) (= val (thaw (freeze val)))))))
;;; Trying to decrypt random (invalid) data can actually crash JVM (is (thrown? Exception (thaw (freeze test-data {:password "malformed"}))))
;; (expect Exception (thaw (freeze test-data {:password "malformed"}))) (is (thrown? Exception (thaw (freeze test-data {:password [:salted "p"]})
;; (expect Exception (thaw (freeze test-data {:password [:salted "p"]}))) {;; Necessary to prevent against JVM segfault due to
;; (expect Exception (thaw (freeze test-data {:password [:salted "p"]}) ;; https://goo.gl/t0OUIo:
;; {:compressor nil})) :v1-compatibility? false})))
(is (thrown? Exception (thaw (freeze test-data {:password [:salted "p"]})
{:v1-compatibility? false ; Ref. https://goo.gl/t0OUIo
:compressor nil})))
(expect ; Snappy lib compatibility (for legacy versions of Nippy) (is ; Snappy lib compatibility (for legacy versions of Nippy)
(let [^bytes raw-ba (freeze test-data {:compressor nil}) (let [^bytes raw-ba (freeze test-data {:compressor nil})
^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba)
^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)]
@ -54,56 +74,64 @@
(thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba))
(thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba))
(thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba)))
(thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))) (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba)))))))
;;;; Custom types & records ;;;; Custom types & records
;;; Extend to custom Type (deftype MyType [data])
(defrecord MyType [data])
(expect Exception (do (nippy/extend-freeze MyType 1 [x s] (.writeUTF s (:data x)))
(thaw (freeze (->MyType "val")))))
(expect (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s)))
(let [type (->MyType "val")] (= type (thaw (freeze type))))))
;;; Extend to custom Record
(defrecord MyRec [data]) (defrecord MyRec [data])
(expect (do (nippy/extend-freeze MyRec 2 [x s] (.writeUTF s (str "foo-" (:data x))))
(deftest _types
;;; Extend to custom Type
(is (thrown? Exception ; No thaw extension yet
(do (nippy/swap-custom-readers! (constantly {}))
(nippy/extend-freeze MyType 1 [x s] (.writeUTF s (.data x)))
(thaw (freeze (->MyType "val"))))))
(is (do (nippy/extend-thaw 1 [s] (->MyType (.readUTF s)))
(let [mt (->MyType "val")] (= (.data ^MyType mt)
(.data ^MyType (thaw (freeze mt)))))))
;;; Extend to custom Record
(is (do (nippy/extend-freeze MyRec 2 [x s] (.writeUTF s (str "foo-" (:data x))))
(nippy/extend-thaw 2 [s] (->MyRec (.readUTF s))) (nippy/extend-thaw 2 [s] (->MyRec (.readUTF s)))
(= (->MyRec "foo-val") (thaw (freeze (->MyRec "val")))))) (= (->MyRec "foo-val") (thaw (freeze (->MyRec "val"))))))
;;; Keyword (prefixed) extensions ;;; Keyword (prefixed) extensions
(expect (is
(do (nippy/extend-freeze MyType :nippy-tests/MyType [x s] (.writeUTF s (:data x))) (do (nippy/extend-freeze MyRec :nippy-tests/MyRec [x s] (.writeUTF s (:data x)))
(nippy/extend-thaw :nippy-tests/MyType [s] (->MyType (.readUTF s))) (nippy/extend-thaw :nippy-tests/MyRec [s] (->MyRec (.readUTF s)))
(let [type (->MyType "val")] (= type (thaw (freeze type)))))) (let [mr (->MyRec "val")] (= mr (thaw (freeze mr)))))))
;;;; Stable binary representation of vals ;;;; Stable binary representation of vals
(expect (seq (freeze test-data)) (deftest _stable-bin
(seq (freeze test-data))) ; f(x)=f(y) | x=y
;; As above, but try multiple times to catch possible protocol interface races: (is (= (seq (freeze test-data))
(expect #(every? true? %) (seq (freeze test-data)))) ; f(x)=f(y) | x=y
;; As above, but try multiple times to catch possible protocol interface races:
(is (every? true?
(repeatedly 1000 (fn [] (= (seq (freeze test-data)) (repeatedly 1000 (fn [] (= (seq (freeze test-data))
(seq (freeze test-data)))))) (seq (freeze test-data)))))))
;; NB abandoning - no way to do this reliably w/o appropriate contracts from ;; NB abandoning - no way to do this reliably w/o appropriate contracts from
;; (seq <unordered-coll>): ;; (seq <unordered-coll>):
;; ;;
;; (expect (seq (-> test-data freeze)) ; f(x)=f(f-1(f(x))) ;; (is (= (seq (-> test-data freeze))
;; (seq (-> test-data freeze thaw freeze))) ;; (seq (-> test-data freeze thaw freeze)))) ; f(x)=f(f-1(f(x)))
;; ;;
;; As above, but with repeated refreeze to catch possible protocol interface races: ;; As above, but with repeated refreeze to catch possible protocol interface races:
;; (expect (= (seq (freeze test-data)) ;; (is (= (seq (freeze test-data))
;; (seq (reduce (fn [frozen _] (freeze (thaw frozen))) ;; (seq (reduce (fn [frozen _] (freeze (thaw frozen)))
;; (freeze test-data) (range 1000))))) ;; (freeze test-data) (range 1000)))))
)
(defn qc-prop-bijection [& [n]] (defn qc-prop-bijection [& [n]]
(let [bin->val (atom {}) (let [bin->val (atom {})
val->bin (atom {})] val->bin (atom {})]
(merge (merge
(check/quick-check (or n 1) (tc/quick-check (or n 1)
(check-props/for-all [val check-gen/any #_check-gen/any-printable] (tc-props/for-all [val tc-gens]
(let [;; Nb need `seq` for Clojure hash equality: (let [;; Nb need `seq` for Clojure hash equality:
bin (hash (seq (freeze val)))] bin (hash (seq (freeze val)))]
(and (and
@ -121,19 +149,21 @@
nil))) nil)))
(comment (comment
(check-gen/sample check-gen/any 10) (tc-gens/sample tc-gens 10)
(:result (qc-prop-bijection 80)) (:result (qc-prop-bijection 80))
(let [{:keys [result bin->val val->bin]} (qc-prop-bijection 10)] (let [{:keys [result bin->val val->bin]} (qc-prop-bijection 10)]
[result (vals bin->val)])) [result (vals bin->val)]))
(expect #(:result %) (qc-prop-bijection 80)) (deftest _gc-prop-bijection
(is (:result (qc-prop-bijection tc-num-tests))))
;;;; Thread safety ;;;; Thread safety
;; Not sure why, but record equality test fails in futures: ;; Not sure why, but record equality test fails in futures:
(def test-data-threaded (dissoc nippy/stress-data-comparable :stress-record)) (def test-data-threaded (dissoc nippy/stress-data-comparable :stress-record))
(expect (deftest _thread-safe
(is
(let [futures (let [futures
(mapv (mapv
(fn [_] (fn [_]
@ -142,7 +172,7 @@
(range 50))] (range 50))]
(every? deref futures))) (every? deref futures)))
(expect (is
(let [futures (let [futures
(mapv (mapv
(fn [_] (fn [_]
@ -153,7 +183,7 @@
(range 50))] (range 50))]
(every? deref futures))) (every? deref futures)))
(expect (is
(let [futures (let [futures
(mapv (mapv
(fn [_] (fn [_]
@ -162,8 +192,10 @@
{:password [:cached "password"]}) {:password [:cached "password"]})
test-data-threaded))) test-data-threaded)))
(range 50))] (range 50))]
(every? deref futures))) (every? deref futures))))
;;;; Benchmarks ;;;; Benchmarks
(expect (benchmarks/bench {})) ; Also tests :cached passwords (deftest _benchmarks
(is (benchmarks/bench {})) ; Also tests :cached passwords
)