diff --git a/CHANGELOG.md b/CHANGELOG.md index 150b7b2..2470b40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ > This project uses [Break Versioning](https://github.com/ptaoussanis/encore/blob/master/BREAK-VERSIONING.md) as of **Aug 16, 2014**. +## v2.9.0-RC1 / 2015 Apr 29 + +> This is a non-breaking **performance release** that can result in significant speed+space improvements for users serializing many small values + +* **Implementation**: eliminate some unnecessary boxed math +* **New**: intelligent allow auto-selection of `freeze` compression scheme using `:auto` compressor (now the default) + + +```clojure +[com.taoensso/nippy "2.9.0-RC1"] +``` + + ## v2.8.0 / 2015 Feb 18 > This is a **maintenance release** with some minor fixes and some dependency updates. diff --git a/README.md b/README.md index 1af31e2..f8e27b3 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ **[API docs][]** | **[CHANGELOG][]** | [other Clojure libs][] | [Twitter][] | [contact/contrib](#contact--contributing) | current [Break Version][]: ```clojure -[com.taoensso/nippy "2.8.0"] ; Please see CHANGELOG for details +[com.taoensso/nippy "2.8.0"] ; Stable +[com.taoensso/nippy "2.9.0-RC1"] ; Dev, please see CHANGELOG for details ``` # Nippy, a Clojure serialization library @@ -148,7 +149,7 @@ There's two default forms of encryption on offer: `:salted` and `:cached`. Each ## Contact & contributing -`lein start-dev` to get a (headless) development repl that you can connect to with [Cider][] (emacs) or your IDE. +`lein start-dev` to get a (headless) development repl that you can connect to with [Cider][] (Emacs) or your IDE. Please use the project's GitHub [issues page][] for project questions/comments/suggestions/whatever **(pull requests welcome!)**. Am very open to ideas if you have any! @@ -159,18 +160,16 @@ Otherwise reach me (Peter Taoussanis) at [taoensso.com][] or on [Twitter][]. Che Copyright © 2012-2014 Peter Taoussanis. Distributed under the [Eclipse Public License][], the same as Clojure. -[API docs]: -[CHANGELOG_]: -[CHANGELOG]: -[other Clojure libs]: -[Twitter]: -[SemVer]: -[Break Version]: -[Leiningen]: -[CDS]: -[ClojureWerkz]: -[issues page]: -[commit history]: -[Cider]: -[taoensso.com]: -[Eclipse Public License]: +[API docs]: http://ptaoussanis.github.io/nippy/ +[CHANGELOG]: https://github.com/ptaoussanis/nippy/releases +[other Clojure libs]: https://www.taoensso.com/clojure +[taoensso.com]: https://www.taoensso.com +[Twitter]: https://twitter.com/ptaoussanis +[issues page]: https://github.com/ptaoussanis/nippy/issues +[commit history]: https://github.com/ptaoussanis/nippy/commits/master +[Break Version]: https://github.com/ptaoussanis/encore/blob/master/BREAK-VERSIONING.md +[Leiningen]: http://leiningen.org/ +[Cider]: https://github.com/clojure-emacs/cider +[CDS]: http://clojure-doc.org/ +[ClojureWerkz]: http://clojurewerkz.org/ +[Eclipse Public License]: https://raw2.github.com/ptaoussanis/nippy/master/LICENSE diff --git a/project.clj b/project.clj index a6b4a1c..295fbd0 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "2.8.0" +(defproject com.taoensso/nippy "2.9.0-RC1" :author "Peter Taoussanis " :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" @@ -8,12 +8,13 @@ :comments "Same as Clojure"} :min-lein-version "2.3.3" :global-vars {*warn-on-reflection* true - *assert* true} + *assert* true + *unchecked-math* :warn-on-boxed} :dependencies [[org.clojure/clojure "1.4.0"] - [org.clojure/tools.reader "0.8.13"] - [com.taoensso/encore "1.21.0"] + [org.clojure/tools.reader "0.9.2"] + [com.taoensso/encore "1.24.1"] [org.iq80.snappy/snappy "0.3"] [org.tukaani/xz "1.5"] [net.jpountz.lz4/lz4 "1.3"]] @@ -23,16 +24,17 @@ :server-jvm {:jvm-opts ^:replace ["-server" "-Xms1024m" "-Xmx2048m"]} :1.5 {:dependencies [[org.clojure/clojure "1.5.1"]]} :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} + :1.7 {:dependencies [[org.clojure/clojure "1.7.0-beta1"]]} :test {:jvm-opts ["-Xms1024m" "-Xmx2048m"] - :dependencies [[expectations "2.0.13"] + :dependencies [[expectations "2.1.1"] [org.clojure/test.check "0.7.0"] - ;; [com.cemerick/double-check "0.5.7"] + ;; [com.cemerick/double-check "0.6.1"] [org.clojure/data.fressian "0.2.0"] - [org.xerial.snappy/snappy-java "1.1.1.6"]]} - :dev [:1.6 :test + [org.xerial.snappy/snappy-java "1.1.1.7"]]} + :dev [:1.7 :test {:plugins [[lein-pprint "1.1.1"] - [lein-ancient "0.5.5"] + [lein-ancient "0.6.7"] [lein-expectations "0.0.8"] [lein-autoexpect "1.2.2"] [codox "0.8.10"]]}]} @@ -40,7 +42,7 @@ :test-paths ["test" "src"] :aliases - {"test-all" ["with-profile" "default:+1.5:+1.6" "expectations"] + {"test-all" ["with-profile" "default:+1.5:+1.6:+1.7" "expectations"] ;; "test-all" ["with-profile" "default:+1.6" "expectations"] "test-auto" ["with-profile" "+test" "autoexpect"] "deploy-lib" ["do" "deploy" "clojars," "install"] diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index e6657ae..eee7472 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -1,6 +1,6 @@ (ns taoensso.nippy - "Simple, high-performance Clojure serialization library. Originally adapted - from Deep-Freeze." + "High-performance JVM Clojure serialization library. Originally adapted from + Deep-Freeze." {:author "Peter Taoussanis"} (:require [clojure.tools.reader.edn :as edn] [taoensso.encore :as encore] @@ -21,7 +21,7 @@ ;;;; Encore version check -(let [min-encore-version 1.21] ; Let's get folks on newer versions here +(let [min-encore-version 1.21] (if-let [assert! (ns-resolve 'taoensso.encore 'assert-min-encore-version)] (assert! min-encore-version) (throw @@ -40,6 +40,8 @@ ;; * Sanity check (confirm that data appears to be Nippy data). ;; * Nippy version check (=> supports changes to data schema over time). ;; * Supports :auto thaw compressor, encryptor. +;; * Supports :auto freeze compressor (since this depends on :auto thaw +;; compressor). ;; (def ^:private ^:const head-version 1) (def ^:private head-sig (.getBytes "NPY" "UTF-8")) @@ -55,6 +57,7 @@ (byte 3) {:version 1 :compressor-id :snappy :encryptor-id :aes128-sha512} (byte 7) {:version 1 :compressor-id :snappy :encryptor-id :else} ;; + ;;; :lz4 used for both lz4 and lz4hc compressor (the two are compatible) (byte 8) {:version 1 :compressor-id :lz4 :encryptor-id nil} (byte 9) {:version 1 :compressor-id :lz4 :encryptor-id :aes128-sha512} (byte 10) {:version 1 :compressor-id :lz4 :encryptor-id :else} @@ -209,7 +212,7 @@ (doseq [i# ~'x] (freeze-to-out ~'out i#))) (let [bas# (ByteArrayOutputStream.) sout# (DataOutputStream. bas#) - cnt# (reduce (fn [cnt# i#] + cnt# (reduce (fn [^long cnt# i#] (freeze-to-out sout# i#) (unchecked-inc cnt#)) 0 ~'x) @@ -369,11 +372,30 @@ [^DataOutput data-output x & _] (freeze-to-out data-output x)) +(defn default-freeze-compressor-selector + "Strategy: + * Prioritize speed, but allow lz4. + * Skip lz4 unless it's likely that lz4's space benefit will outweigh its + space overhead." + [^bytes ba] + (let [ba-len (alength ba)] + (cond + ;; (> ba-len 1024) lzma2-compressor + ;; (> ba-len 512) lz4hc-compressor + (> ba-len 128) lz4-compressor + :else nil))) + +(encore/defonce* default-freeze-compressor-selector_ + "EXPERIMENTAL. + Determines the global default default compressor selector + (fn [^bytes ba])->compressor used by `(freeze {:compressor :auto <...>})." + (atom default-freeze-compressor-selector)) + (defn freeze "Serializes arg (any Clojure data type) to a byte array. To freeze custom types, extend the Clojure reader or see `extend-freeze`." ^bytes [x & [{:keys [compressor encryptor password skip-header?] - :or {compressor lz4-compressor + :or {compressor :auto encryptor aes128-encryptor} :as opts}]] (let [legacy-mode? (:legacy-mode opts) ; DEPRECATED Nippy v1-compatible freeze @@ -384,8 +406,20 @@ dos (DataOutputStream. baos)] (freeze-to-out! dos x) (let [ba (.toByteArray baos) + + compressor + (if (identical? compressor :auto) + (if skip-header? + lz4-compressor + (@default-freeze-compressor-selector_ ba)) + (if (fn? compressor) + (compressor ba) ; Assume compressor selector fn + compressor ; Assume compressor + )) + ba (if-not compressor ba (compress compressor ba)) ba (if-not encryptor ba (encrypt encryptor password ba))] + (if skip-header? ba (wrap-header ba {:compressor-id (when-let [c compressor] @@ -417,8 +451,10 @@ `(let [in# ~in] (encore/repeatedly-into* ~coll (.readInt in#) (thaw-from-in in#)))) (defmacro ^:private read-kvs [in coll] - `(let [in# ~in] (encore/repeatedly-into* ~coll (/ (.readInt in#) 2) - [(thaw-from-in in#) (thaw-from-in in#)]))) + `(let [in# ~in] + (encore/repeatedly-into* ~coll (quot (.readInt in#) 2) + [(thaw-from-in in#) (thaw-from-in in#)]))) + (declare ^:private custom-readers) (defn- read-custom! [type-id in] @@ -660,8 +696,8 @@ [custom-type-id] (assert-custom-type-id custom-type-id) (if-not (keyword? custom-type-id) - (int (- custom-type-id)) - (let [hash-id (hash custom-type-id) + (int (- ^long custom-type-id)) + (let [^long hash-id (hash custom-type-id) short-hash-id (if (pos? hash-id) (mod hash-id Short/MAX_VALUE) (mod hash-id Short/MIN_VALUE))] @@ -722,7 +758,8 @@ ;;; Some useful custom types - EXPERIMENTAL -(defrecord Compressable-LZMA2 [value]) +;; Mostly deprecated by :auto compressor selection +(defrecord Compressable-LZMA2 [value]) ; Why was this `LZMA2`, not `lzma2`? (extend-freeze Compressable-LZMA2 128 [x out] (let [ba (freeze (:value x) {:skip-header? true :compressor nil}) ba-len (alength ba) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index 9da799c..31321ef 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -64,7 +64,12 @@ (comment ;; (bench {:reader? true :lzma2? true :fressian? true :laps 1}) - ;; (bench {:laps 2}) + ;; (bench {:laps 4}) + + ;;; 2015 Apr 17 w/ smart compressor selection, Clojure 1.7.0-beta1 + {:default {:round 6163, :freeze 4095, :thaw 2068, :size 16121}} + {:fast {:round 5417, :freeze 3480, :thaw 1937, :size 17013}} + {:encrypted {:round 10950, :freeze 6400, :thaw 4550, :size 16148}} ;;; 2014 Apr 7 w/ some additional implementation tuning {:default {:round 6533, :freeze 3618, :thaw 2915, :size 16139}}