Merge branch 'dev'

README.md

@@ -1,7 +1,7 @@
 Current [semantic](http://semver.org/) version:
 
 ```clojure
-[com.taoensso/nippy "1.0.1"]
+[com.taoensso/nippy "1.1.0"]
 ```
 
 # Nippy, a Clojure serialization library
@@ -25,7 +25,7 @@ Nippy is an attempt to provide a drop-in, high-performance alternative to the re
 Depend on Nippy in your `project.clj`:
 
 ```clojure
-[com.taoensso/nippy "1.0.1"]
+[com.taoensso/nippy "1.1.0"]
 ```
 
 and `require` the library:
@@ -118,7 +118,7 @@ CDS (Clojure Documentation Site) is a contributor-friendly community project aim
 
 ## Contact & Contribution
 
-Reach me (Peter Taoussanis) at *ptaoussanis at gmail.com* for questions/comments/suggestions/whatever. I'm very open to ideas if you have any! I'm also on Twitter: [@ptaoussanis](https://twitter.com/#!/ptaoussanis).
+Reach me (Peter Taoussanis) at [taoensso.com](https://www.taoensso.com) for questions/comments/suggestions/whatever. I'm very open to ideas if you have any! I'm also on Twitter: [@ptaoussanis](https://twitter.com/#!/ptaoussanis).
 
 ## License
 

project.clj

@@ -1,14 +1,14 @@
-(defproject com.taoensso/nippy "1.0.1"
+(defproject com.taoensso/nippy "1.1.0"
   :description "Clojure serialization library"
   :url "https://github.com/ptaoussanis/nippy"
   :license {:name "Eclipse Public License"}
-  :dependencies [[org.clojure/clojure           "1.3.0"]
-                 [org.xerial.snappy/snappy-java "1.0.5-M3"]]
+  :dependencies [[org.clojure/clojure    "1.3.0"]
+                 [org.iq80.snappy/snappy "0.3"]]
   :profiles {:1.3  {:dependencies [[org.clojure/clojure "1.3.0"]]}
              :1.4  {:dependencies [[org.clojure/clojure "1.4.0"]]}
              :1.5  {:dependencies [[org.clojure/clojure "1.5.0-alpha3"]]}
              :dev  {:dependencies []}
-             :test {:dependencies []}}
+             :test {:dependencies [[org.xerial.snappy/snappy-java "1.0.5-M3"]]}}
   :aliases {"test-all" ["with-profile" "test,1.3:test,1.4:test,1.5" "test"]}
   :min-lein-version "2.0.0"
   :warn-on-reflection true)

src/taoensso/nippy.clj

@@ -5,7 +5,6 @@
   (:require [taoensso.nippy.utils :as utils])
   (:import  [java.io DataInputStream DataOutputStream ByteArrayOutputStream
              ByteArrayInputStream]
-            [org.xerial.snappy Snappy]
             [clojure.lang IPersistentList IPersistentVector IPersistentMap
              IPersistentSet PersistentQueue IPersistentCollection Keyword
              BigInt Ratio]))
@@ -82,8 +81,6 @@
 
 (defprotocol Freezable (freeze [this stream]))
 
-(comment (meta '^:DataOutputStream s))
-
 (defmacro freezer
   "Helper to extend Freezable protocol."
   [type id & body]
@@ -152,19 +149,22 @@
 
 (defn freeze-to-stream!
   "Serializes x to given output stream."
-  [data-output-stream x]
-  (binding [*print-dup* true] ; For `pr-str`
-    (freeze-to-stream!* data-output-stream x)))
+  ([data-output-stream x] ; For <= 1.0.1 compatibility
+     (freeze-to-stream! data-output-stream x true))
+  ([data-output-stream x print-dup?]
+     (binding [*print-dup* print-dup?] ; For `pr-str`
+       (freeze-to-stream!* data-output-stream x))))
 
 (defn freeze-to-bytes
   "Serializes x to a byte array and returns the array."
-  ^bytes [x & {:keys [compress?]
-               :or   {compress? true}}]
+  ^bytes [x & {:keys [compress?  print-dup?]
+               :or   {compress?  true
+                      print-dup? true}}]
   (let [ba     (ByteArrayOutputStream.)
         stream (DataOutputStream. ba)]
-    (freeze-to-stream! stream x)
+    (freeze-to-stream! stream x print-dup?)
     (let [ba (.toByteArray ba)]
-      (if compress? (Snappy/compress ba) ba))))
+      (if compress? (utils/compress-bytes ba) ba))))
 
 ;;;; Thawing
 
@@ -199,10 +199,7 @@
      id-list    (apply list (coll-thaw! s)) ; TODO OOMs for big colls
      id-vector  (into  [] (coll-thaw! s))
      id-set     (into #{} (coll-thaw! s))
-     ;; id-map  (apply hash-map (coll-thaw! s)) ; OOMs for big colls
-     ;; id-map  (into {} (map vec (partition 2 (coll-thaw! s))) ; ~6.4x time
-     ;; id-map  (into {} (utils/pairs (coll-thaw! s)))          ; ~1.8x time
-     id-map     (into  {} (coll-thaw-pairs! s))                 ; ~0.8x time
+     id-map     (into  {} (coll-thaw-pairs! s))
      id-coll    (doall (coll-thaw! s))
      id-queue   (into  (PersistentQueue/EMPTY) (coll-thaw! s))
 
@@ -245,7 +242,7 @@
   [ba & {:keys [read-eval? compressed?]
          :or   {read-eval?  false ; For `read-string` injection safety - NB!!!
                 compressed? true}}]
-  (-> (if compressed? (Snappy/uncompress ba) ba)
+  (-> (if compressed? (utils/uncompress-bytes ba) ba)
       (ByteArrayInputStream.)
       (DataInputStream.)
       (thaw-from-stream! read-eval?)))

src/taoensso/nippy/benchmarks.clj

@@ -14,6 +14,8 @@
 (def roundtrip        (comp thaw-from-bytes freeze-to-bytes))
 (def reader-roundtrip (comp reader-thaw reader-freeze))
 
+(defn autobench [] (bench (roundtrip data)))
+
 (comment
 
   ;;; Times
@@ -43,4 +45,10 @@
   (let [frozen (reader-freeze data)]   (count (.getBytes frozen "UTF8")))
   (let [frozen (freeze-to-bytes data)] (count frozen))
   ;; 22788, 12224
+
+  ;;; Snappy implementations
+  (println (bench (roundtrip data)))
+  ;;                No Snappy: 6163 6064 6042 6176
+  ;;               Snappy JNI: 6489 6446 6542 6412
+  ;; Snappy native array copy: 6569 6419 6414 6590
   )

src/taoensso/nippy/utils.clj

@@ -1,6 +1,7 @@
 (ns taoensso.nippy.utils
   {:author "Peter Taoussanis"}
-  (:require [clojure.string :as str]))
+  (:require [clojure.string :as str])
+  (:import  org.iq80.snappy.Snappy))
 
 (defmacro case-eval
   "Like `case` but evaluates test constants for their compile-time value."
@@ -55,4 +56,7 @@
 (defn version-sufficient?
   [version-str min-version-str]
   (try (>= (version-compare version-str min-version-str) 0)
-       (catch Exception _ false)))
+       (catch Exception _ false)))
+
+(defn compress-bytes   [^bytes ba] (Snappy/compress   ba))
+(defn uncompress-bytes [^bytes ba] (Snappy/uncompress ba 0 (alength ba)))

test/test_nippy/main.clj

@@ -1,10 +1,28 @@
 (ns test-nippy.main
   (:use [clojure.test])
-  (:require [taoensso.nippy :as nippy]))
+  (:require [taoensso.nippy :as nippy]
+            [taoensso.nippy.benchmarks :as benchmarks]))
 
 ;; Remove stuff from stress-data that breaks roundtrip equality
 (def test-data (dissoc nippy/stress-data :bytes))
 
 (def roundtrip (comp nippy/thaw-from-bytes nippy/freeze-to-bytes))
 
-(deftest test-roundtrip (is (= test-data (roundtrip test-data))))
+(deftest test-roundtrip (is (= test-data (roundtrip test-data))))
+
+(println "Benchmarking roundtrips (x3)")
+(println "----------------------------")
+(println (benchmarks/autobench))
+(println (benchmarks/autobench))
+(println (benchmarks/autobench))
+
+(deftest test-snappy-library-compatibility
+  (let [thaw             #(nippy/thaw-from-bytes % :compressed? false)
+        ^bytes raw-ba    (nippy/freeze-to-bytes test-data :compress? false)
+        ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba)
+        ^bytes iq80-ba   (org.iq80.snappy.Snappy/compress   raw-ba)]
+    (is (= (thaw raw-ba)
+           (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba))
+           (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba))
+           (thaw (org.iq80.snappy.Snappy/uncompress   iq80-ba    0 (alength iq80-ba)))
+           (thaw (org.iq80.snappy.Snappy/uncompress   xerial-ba  0 (alength xerial-ba)))))))