EXPERIMENTAL: Make LZ4 the default Nippy compressor (back-compatible for header'ed data)

src/taoensso/nippy.clj

@@ -6,7 +6,8 @@
             [taoensso.encore :as encore]
             [taoensso.nippy
              (utils       :as utils)
-             (compression :as compression :refer (snappy-compressor))
+             (compression :as compression :refer (lz4-compressor
+                                                  snappy-compressor))
              (encryption  :as encryption  :refer (aes128-encryptor))])
   (:import  [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream
              DataOutputStream Serializable ObjectOutputStream ObjectInputStream
@@ -25,13 +26,20 @@
 ;; * Nippy version check (=> supports changes to data schema over time).
 ;; * Encrypted &/or compressed data identification.
 ;;
-(def ^:private ^:const head-version 1)
+(def ^:private ^:const head-version 2)
 (def ^:private head-sig (.getBytes "NPY" "UTF-8"))
 (def ^:private ^:const head-meta "Final byte stores version-dependent metadata."
-  {(byte 0) {:version 1 :compressed? false :encrypted? false}
+  {;;; Nippy <= v2.6 with Snappy as default compressor
+   (byte 0) {:version 1 :compressed? false :encrypted? false}
    (byte 1) {:version 1 :compressed? true  :encrypted? false}
    (byte 2) {:version 1 :compressed? false :encrypted? true}
-   (byte 3) {:version 1 :compressed? true  :encrypted? true}})
+   (byte 3) {:version 1 :compressed? true  :encrypted? true}
+
+   ;;; EXPERIMENTAL: Nippy >= v2.7 with LZ4 as default compressor
+   (byte 4) {:version 2 :compressed? false :encrypted? false}
+   (byte 5) {:version 2 :compressed? true  :encrypted? false}
+   (byte 6) {:version 2 :compressed? false :encrypted? true}
+   (byte 7) {:version 2 :compressed? true  :encrypted? true}})
 
 (defmacro when-debug-mode [& body] (when #_true false `(do ~@body)))
 
@@ -310,8 +318,6 @@
 (comment (wrap-header (.getBytes "foo") {:compressed? true
                                          :encrypted?  false}))
 
-(declare assert-legacy-args) ; Deprecated
-
 (defn freeze-to-out!
   "Low-level API. Serializes arg (any Clojure data type) to a DataOutput."
   [^DataOutput data-output x & _]
@@ -321,12 +327,15 @@
   "Serializes arg (any Clojure data type) to a byte array. For custom types
   extend the Clojure reader or see `extend-freeze`."
   ^bytes [x & [{:keys [password compressor encryptor skip-header?]
-                :or   {compressor snappy-compressor
+                :or   {compressor lz4-compressor
                        encryptor  aes128-encryptor}
                 :as   opts}]]
-  (when (:legacy-mode opts) ; Deprecated
+  (let [;;; Legacy mode is deprecated
-    (assert-legacy-args compressor password))
+        compressor (if-not (:legacy-mode opts) compressor snappy-compressor)
-  (let [skip-header? (or skip-header? (:legacy-mode opts)) ; Deprecated
+        encryptor  (if-not (:legacy-mode opts) encryptor  nil)
+        ;;
+        skip-header? (or skip-header? (:legacy-mode opts) ; Deprecated
+                         )
         bas  (ByteArrayOutputStream.)
         sout (DataOutputStream. bas)]
     (freeze-to-out! sout x)
@@ -492,10 +501,10 @@
   fallback (`{:headerless-meta nil}`) if you're certain you won't be thawing
   headerless data."
   [^bytes ba & [{:keys [password compressor encryptor headerless-meta]
-                 :or   {compressor snappy-compressor
+                 :or   {compressor :auto
-                        encryptor  aes128-encryptor
+                        encryptor  :auto
                         headerless-meta ; Recommend set to nil when possible
-                        {:version     1
+                        {:version     2
                          :compressed? true
                          :encrypted?  false}}
                  :as   opts}]]
@@ -507,19 +516,26 @@
 
         ex (fn [msg & [e]] (throw (Exception. (str "Thaw failed: " msg) e)))
         try-thaw-data
-        (fn [data-ba {:keys [compressed? encrypted?] :as _head-or-headerless-meta}]
+        (fn [data-ba {:keys [compressed? encrypted? version]
-          (let [password   (when encrypted?  password)
+                     :as _head-or-headerless-meta}]
-                compressor (when compressed? compressor)]
+          (let [encryptor  (when (and password encrypted?)
+                             (if-not (identical? encryptor :auto) encryptor
+                               aes128-encryptor))
+                compressor (when compressed?
+                             (if-not (identical? compressor :auto) compressor
+                               (case (int version)
+                                 1 snappy-compressor
+                                 2 lz4-compressor)))]
             (try
               (let [ba data-ba
-                    ba (if password (encryption/decrypt encryptor password ba) ba)
+                    ba (if encryptor  (encryption/decrypt encryptor password ba) ba)
                     ba (if compressor (compression/decompress compressor ba)     ba)
                     sin (DataInputStream. (ByteArrayInputStream. ba))]
                 (thaw-from-in! sin))
 
               (catch Exception e
                 (cond
-                 password   (if head-meta (ex "Wrong password/encryptor?" e)
+                 encryptor  (if head-meta (ex "Wrong password/encryptor?" e)
                                           (ex "Unencrypted data?" e))
                  compressor (if head-meta (ex "Encrypted data or wrong compressor?" e)
                                           (ex "Uncompressed data?" e))
@@ -614,7 +630,7 @@
   (let [compressed? (.readBoolean in)
         ba          (read-bytes in)]
     (thaw ba {:compressor compression/lzma2-compressor
-              :headerless-meta {:version     1
+              :headerless-meta {:version     2
                                 :compressed? compressed?
                                 :encrypted?  false}})))
 
@@ -739,12 +755,6 @@
 (def thaw-from-stream! "DEPRECATED: Use `thaw-from-in!` instead."
   thaw-from-in!)
 
-(defn- assert-legacy-args [compressor password]
-  (when password
-    (throw (AssertionError. "Encryption not supported in legacy mode.")))
-  (when (and compressor (not= compressor snappy-compressor))
-    (throw (AssertionError. "Only Snappy compressor supported in legacy mode."))))
-
 (defn freeze-to-bytes "DEPRECATED: Use `freeze` instead."
   ^bytes [x & {:keys [compress?]
                :or   {compress? true}}]

src/taoensso/nippy/benchmarks.clj

@@ -43,8 +43,12 @@
 
     (println {:default   (bench1 #(freeze % {})
                                  #(thaw   % {}))})
-    (println {:fast      (bench1 #(freeze % {:compressor nil})
+    (println {:fast      (bench1 #(freeze % {:compressor nil
-                                 #(thaw   % {:compressor nil}))})
+                                             :skip-header? true})
+                                 #(thaw   % {:headerless-meta
+                                             {:version 2
+                                              :compressed? false
+                                              :encrypted?  false}}))})
     (println {:encrypted (bench1 #(freeze % {:password [:cached "p"]})
                                  #(thaw   % {:password [:cached "p"]}))})
 
@@ -65,6 +69,11 @@
   ;; (bench {:reader? true :lzma2? true :fressian? true :laps 1})
   ;; (bench {:laps 2})
 
+  ;;; 2014 Apr 5 w/ headerless :fast, LZ4 replacing Snappy as default compressor
+  ;; {:default   {:round 7669,  :freeze 4157, :thaw 3512, :size 16143}}
+  ;; {:fast      {:round 6918,  :freeze 3636, :thaw 3282, :size 16992}}
+  ;; {:encrypted {:round 11814, :freeze 6180, :thaw 5634, :size 16164}}
+
   ;;; 2014 Jan 22: with common-type size optimizations, enlarged stress-data
   ;; {:reader    {:round 109544, :freeze 39523, :thaw 70021, :size 27681}}
   ;; {:default   {:round 9234,   :freeze 5128,  :thaw 4106,  :size 15989}}

src/taoensso/nippy/compression.clj

@@ -104,7 +104,8 @@
   (->LZ4Compressor (.fastCompressor   lz4-factory)
                    (.fastDecompressor lz4-factory)))
 
-(def lz4hc-compressor "Like `lz4-compressor` but trades some speed for ratio."
+(def lz4hc-compressor
+  "Like `lz4-compressor` but trades some write speed for ratio."
   (->LZ4Compressor (.highCompressor   lz4-factory)
                    (.fastDecompressor lz4-factory)))
 

test/taoensso/nippy/tests/main.clj

@@ -16,7 +16,11 @@
 ;;;; Core
 
 (expect test-data ((comp thaw freeze) test-data))
-(expect test-data ((comp thaw #(freeze % {:legacy-mode true})) test-data))
+(expect test-data ((comp #(thaw   % {:headerless-meta {:version     1
+                                                       :compressed? true
+                                                       :encrypted?  false}})
+                         #(freeze % {:legacy-mode true}))
+                   test-data))
 (expect test-data ((comp #(thaw   % {:password [:salted "p"]})
                          #(freeze % {:password [:salted "p"]}))
                    test-data))