diff --git a/README.md b/README.md index 5e724af..8d81842 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Current [semantic](http://semver.org/) version: ```clojure -[com.taoensso/nippy "1.0.1"] +[com.taoensso/nippy "1.1.0"] ``` # Nippy, a Clojure serialization library @@ -25,7 +25,7 @@ Nippy is an attempt to provide a drop-in, high-performance alternative to the re Depend on Nippy in your `project.clj`: ```clojure -[com.taoensso/nippy "1.0.1"] +[com.taoensso/nippy "1.1.0"] ``` and `require` the library: @@ -118,7 +118,7 @@ CDS (Clojure Documentation Site) is a contributor-friendly community project aim ## Contact & Contribution -Reach me (Peter Taoussanis) at *ptaoussanis at gmail.com* for questions/comments/suggestions/whatever. I'm very open to ideas if you have any! I'm also on Twitter: [@ptaoussanis](https://twitter.com/#!/ptaoussanis). +Reach me (Peter Taoussanis) at [taoensso.com](https://www.taoensso.com) for questions/comments/suggestions/whatever. I'm very open to ideas if you have any! I'm also on Twitter: [@ptaoussanis](https://twitter.com/#!/ptaoussanis). ## License diff --git a/project.clj b/project.clj index cd0911a..a72f766 100644 --- a/project.clj +++ b/project.clj @@ -1,14 +1,14 @@ -(defproject com.taoensso/nippy "1.0.1" +(defproject com.taoensso/nippy "1.1.0" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License"} - :dependencies [[org.clojure/clojure "1.3.0"] - [org.xerial.snappy/snappy-java "1.0.5-M3"]] + :dependencies [[org.clojure/clojure "1.3.0"] + [org.iq80.snappy/snappy "0.3"]] :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-alpha3"]]} :dev {:dependencies []} - :test {:dependencies []}} + :test {:dependencies [[org.xerial.snappy/snappy-java "1.0.5-M3"]]}} :aliases {"test-all" ["with-profile" "test,1.3:test,1.4:test,1.5" "test"]} :min-lein-version "2.0.0" :warn-on-reflection true) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 78b8ed6..16b7fb0 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -5,7 +5,6 @@ (:require [taoensso.nippy.utils :as utils]) (:import [java.io DataInputStream DataOutputStream ByteArrayOutputStream ByteArrayInputStream] - [org.xerial.snappy Snappy] [clojure.lang IPersistentList IPersistentVector IPersistentMap IPersistentSet PersistentQueue IPersistentCollection Keyword BigInt Ratio])) @@ -82,8 +81,6 @@ (defprotocol Freezable (freeze [this stream])) -(comment (meta '^:DataOutputStream s)) - (defmacro freezer "Helper to extend Freezable protocol." [type id & body] @@ -152,19 +149,22 @@ (defn freeze-to-stream! "Serializes x to given output stream." - [data-output-stream x] - (binding [*print-dup* true] ; For `pr-str` - (freeze-to-stream!* data-output-stream x))) + ([data-output-stream x] ; For <= 1.0.1 compatibility + (freeze-to-stream! data-output-stream x true)) + ([data-output-stream x print-dup?] + (binding [*print-dup* print-dup?] ; For `pr-str` + (freeze-to-stream!* data-output-stream x)))) (defn freeze-to-bytes "Serializes x to a byte array and returns the array." - ^bytes [x & {:keys [compress?] - :or {compress? true}}] + ^bytes [x & {:keys [compress? print-dup?] + :or {compress? true + print-dup? true}}] (let [ba (ByteArrayOutputStream.) stream (DataOutputStream. ba)] - (freeze-to-stream! stream x) + (freeze-to-stream! stream x print-dup?) (let [ba (.toByteArray ba)] - (if compress? (Snappy/compress ba) ba)))) + (if compress? (utils/compress-bytes ba) ba)))) ;;;; Thawing @@ -199,10 +199,7 @@ id-list (apply list (coll-thaw! s)) ; TODO OOMs for big colls id-vector (into [] (coll-thaw! s)) id-set (into #{} (coll-thaw! s)) - ;; id-map (apply hash-map (coll-thaw! s)) ; OOMs for big colls - ;; id-map (into {} (map vec (partition 2 (coll-thaw! s))) ; ~6.4x time - ;; id-map (into {} (utils/pairs (coll-thaw! s))) ; ~1.8x time - id-map (into {} (coll-thaw-pairs! s)) ; ~0.8x time + id-map (into {} (coll-thaw-pairs! s)) id-coll (doall (coll-thaw! s)) id-queue (into (PersistentQueue/EMPTY) (coll-thaw! s)) @@ -245,7 +242,7 @@ [ba & {:keys [read-eval? compressed?] :or {read-eval? false ; For `read-string` injection safety - NB!!! compressed? true}}] - (-> (if compressed? (Snappy/uncompress ba) ba) + (-> (if compressed? (utils/uncompress-bytes ba) ba) (ByteArrayInputStream.) (DataInputStream.) (thaw-from-stream! read-eval?))) diff --git a/benchmarks/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj similarity index 86% rename from benchmarks/benchmarks.clj rename to src/taoensso/nippy/benchmarks.clj index 4f56786..dcc6758 100644 --- a/benchmarks/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -14,6 +14,8 @@ (def roundtrip (comp thaw-from-bytes freeze-to-bytes)) (def reader-roundtrip (comp reader-thaw reader-freeze)) +(defn autobench [] (bench (roundtrip data))) + (comment ;;; Times @@ -43,4 +45,10 @@ (let [frozen (reader-freeze data)] (count (.getBytes frozen "UTF8"))) (let [frozen (freeze-to-bytes data)] (count frozen)) ;; 22788, 12224 + + ;;; Snappy implementations + (println (bench (roundtrip data))) + ;; No Snappy: 6163 6064 6042 6176 + ;; Snappy JNI: 6489 6446 6542 6412 + ;; Snappy native array copy: 6569 6419 6414 6590 ) \ No newline at end of file diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 17be404..30488e1 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -1,6 +1,7 @@ (ns taoensso.nippy.utils {:author "Peter Taoussanis"} - (:require [clojure.string :as str])) + (:require [clojure.string :as str]) + (:import org.iq80.snappy.Snappy)) (defmacro case-eval "Like `case` but evaluates test constants for their compile-time value." @@ -55,4 +56,7 @@ (defn version-sufficient? [version-str min-version-str] (try (>= (version-compare version-str min-version-str) 0) - (catch Exception _ false))) \ No newline at end of file + (catch Exception _ false))) + +(defn compress-bytes [^bytes ba] (Snappy/compress ba)) +(defn uncompress-bytes [^bytes ba] (Snappy/uncompress ba 0 (alength ba))) \ No newline at end of file diff --git a/test/test_nippy/main.clj b/test/test_nippy/main.clj index ec4e41a..185bd68 100644 --- a/test/test_nippy/main.clj +++ b/test/test_nippy/main.clj @@ -1,10 +1,28 @@ (ns test-nippy.main (:use [clojure.test]) - (:require [taoensso.nippy :as nippy])) + (:require [taoensso.nippy :as nippy] + [taoensso.nippy.benchmarks :as benchmarks])) ;; Remove stuff from stress-data that breaks roundtrip equality (def test-data (dissoc nippy/stress-data :bytes)) (def roundtrip (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) -(deftest test-roundtrip (is (= test-data (roundtrip test-data)))) \ No newline at end of file +(deftest test-roundtrip (is (= test-data (roundtrip test-data)))) + +(println "Benchmarking roundtrips (x3)") +(println "----------------------------") +(println (benchmarks/autobench)) +(println (benchmarks/autobench)) +(println (benchmarks/autobench)) + +(deftest test-snappy-library-compatibility + (let [thaw #(nippy/thaw-from-bytes % :compressed? false) + ^bytes raw-ba (nippy/freeze-to-bytes test-data :compress? false) + ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) + ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] + (is (= (thaw raw-ba) + (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) + (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) + (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) + (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))) \ No newline at end of file