From 03684b6d28814fd5d94bd725d112305c4940296d Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Thu, 6 Dec 2012 16:38:07 +0700 Subject: [PATCH 1/8] Housekeeping --- README.md | 2 +- src/taoensso/nippy.clj | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5e724af..561ae8d 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ CDS (Clojure Documentation Site) is a contributor-friendly community project aim ## Contact & Contribution -Reach me (Peter Taoussanis) at *ptaoussanis at gmail.com* for questions/comments/suggestions/whatever. I'm very open to ideas if you have any! I'm also on Twitter: [@ptaoussanis](https://twitter.com/#!/ptaoussanis). +Reach me (Peter Taoussanis) at [taoensso.com](https://www.taoensso.com) for questions/comments/suggestions/whatever. I'm very open to ideas if you have any! I'm also on Twitter: [@ptaoussanis](https://twitter.com/#!/ptaoussanis). ## License diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 78b8ed6..0037a2d 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -82,8 +82,6 @@ (defprotocol Freezable (freeze [this stream])) -(comment (meta '^:DataOutputStream s)) - (defmacro freezer "Helper to extend Freezable protocol." [type id & body] @@ -199,10 +197,7 @@ id-list (apply list (coll-thaw! s)) ; TODO OOMs for big colls id-vector (into [] (coll-thaw! s)) id-set (into #{} (coll-thaw! s)) - ;; id-map (apply hash-map (coll-thaw! s)) ; OOMs for big colls - ;; id-map (into {} (map vec (partition 2 (coll-thaw! s))) ; ~6.4x time - ;; id-map (into {} (utils/pairs (coll-thaw! s))) ; ~1.8x time - id-map (into {} (coll-thaw-pairs! s)) ; ~0.8x time + id-map (into {} (coll-thaw-pairs! s)) id-coll (doall (coll-thaw! s)) id-queue (into (PersistentQueue/EMPTY) (coll-thaw! s)) From f056abc0e8f6ee91b57d05a993ca6547c4ce4c5a Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Wed, 9 Jan 2013 14:11:52 +0700 Subject: [PATCH 2/8] Add optional `print-dup?` arg to `freeze-to-stream!` (default to true) --- src/taoensso/nippy.clj | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 0037a2d..5f17d41 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -150,17 +150,20 @@ (defn freeze-to-stream! "Serializes x to given output stream." - [data-output-stream x] - (binding [*print-dup* true] ; For `pr-str` - (freeze-to-stream!* data-output-stream x))) + ([data-output-stream x] ; For <= 1.0.1 compatibility + (freeze-to-stream! data-output-stream x true)) + ([data-output-stream x print-dup?] + (binding [*print-dup* print-dup?] ; For `pr-str` + (freeze-to-stream!* data-output-stream x)))) (defn freeze-to-bytes "Serializes x to a byte array and returns the array." - ^bytes [x & {:keys [compress?] - :or {compress? true}}] + ^bytes [x & {:keys [compress? print-dup?] + :or {compress? true + print-dup? true}}] (let [ba (ByteArrayOutputStream.) stream (DataOutputStream. ba)] - (freeze-to-stream! stream x) + (freeze-to-stream! stream x print-dup?) (let [ba (.toByteArray ba)] (if compress? (Snappy/compress ba) ba)))) From fdff6aacd110efd014419692dbd925d85a4f85ff Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sun, 20 Jan 2013 19:37:08 +0700 Subject: [PATCH 3/8] Add autobench to tests --- {benchmarks => src/taoensso/nippy}/benchmarks.clj | 2 ++ test/test_nippy/main.clj | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) rename {benchmarks => src/taoensso/nippy}/benchmarks.clj (97%) diff --git a/benchmarks/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj similarity index 97% rename from benchmarks/benchmarks.clj rename to src/taoensso/nippy/benchmarks.clj index 4f56786..eeed80a 100644 --- a/benchmarks/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -14,6 +14,8 @@ (def roundtrip (comp thaw-from-bytes freeze-to-bytes)) (def reader-roundtrip (comp reader-thaw reader-freeze)) +(defn autobench [] (bench (roundtrip data))) + (comment ;;; Times diff --git a/test/test_nippy/main.clj b/test/test_nippy/main.clj index ec4e41a..bec0e6a 100644 --- a/test/test_nippy/main.clj +++ b/test/test_nippy/main.clj @@ -1,10 +1,17 @@ (ns test-nippy.main (:use [clojure.test]) - (:require [taoensso.nippy :as nippy])) + (:require [taoensso.nippy :as nippy] + [taoensso.nippy.benchmarks :as benchmarks])) ;; Remove stuff from stress-data that breaks roundtrip equality (def test-data (dissoc nippy/stress-data :bytes)) (def roundtrip (comp nippy/thaw-from-bytes nippy/freeze-to-bytes)) -(deftest test-roundtrip (is (= test-data (roundtrip test-data)))) \ No newline at end of file +(deftest test-roundtrip (is (= test-data (roundtrip test-data)))) + +(println "Benchmarking roundtrips (x3)") +(println "----------------------------") +(println (benchmarks/autobench)) +(println (benchmarks/autobench)) +(println (benchmarks/autobench)) From 3e178dabf3574a17e26a710b23690555028ef2c9 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sun, 20 Jan 2013 16:53:29 +0700 Subject: [PATCH 4/8] EXPERIMENTAL: Swap Snappy implementation org.xerial.snappy/snappy-java -> org.iq80.snappy/snappy This (native Java) implementation appears to be about as fast as the old (JNI) implementation, but has better support across more platforms. Going to evaluate stability on this branch for possible later merging into master. --- project.clj | 4 ++-- src/taoensso/nippy.clj | 5 ++--- src/taoensso/nippy/benchmarks.clj | 6 ++++++ src/taoensso/nippy/utils.clj | 16 ++++++++++++++-- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/project.clj b/project.clj index cd0911a..09184ed 100644 --- a/project.clj +++ b/project.clj @@ -2,8 +2,8 @@ :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License"} - :dependencies [[org.clojure/clojure "1.3.0"] - [org.xerial.snappy/snappy-java "1.0.5-M3"]] + :dependencies [[org.clojure/clojure "1.3.0"] + [org.iq80.snappy/snappy "0.2"]] :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-alpha3"]]} diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 5f17d41..16b7fb0 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -5,7 +5,6 @@ (:require [taoensso.nippy.utils :as utils]) (:import [java.io DataInputStream DataOutputStream ByteArrayOutputStream ByteArrayInputStream] - [org.xerial.snappy Snappy] [clojure.lang IPersistentList IPersistentVector IPersistentMap IPersistentSet PersistentQueue IPersistentCollection Keyword BigInt Ratio])) @@ -165,7 +164,7 @@ stream (DataOutputStream. ba)] (freeze-to-stream! stream x print-dup?) (let [ba (.toByteArray ba)] - (if compress? (Snappy/compress ba) ba)))) + (if compress? (utils/compress-bytes ba) ba)))) ;;;; Thawing @@ -243,7 +242,7 @@ [ba & {:keys [read-eval? compressed?] :or {read-eval? false ; For `read-string` injection safety - NB!!! compressed? true}}] - (-> (if compressed? (Snappy/uncompress ba) ba) + (-> (if compressed? (utils/uncompress-bytes ba) ba) (ByteArrayInputStream.) (DataInputStream.) (thaw-from-stream! read-eval?))) diff --git a/src/taoensso/nippy/benchmarks.clj b/src/taoensso/nippy/benchmarks.clj index eeed80a..dcc6758 100644 --- a/src/taoensso/nippy/benchmarks.clj +++ b/src/taoensso/nippy/benchmarks.clj @@ -45,4 +45,10 @@ (let [frozen (reader-freeze data)] (count (.getBytes frozen "UTF8"))) (let [frozen (freeze-to-bytes data)] (count frozen)) ;; 22788, 12224 + + ;;; Snappy implementations + (println (bench (roundtrip data))) + ;; No Snappy: 6163 6064 6042 6176 + ;; Snappy JNI: 6489 6446 6542 6412 + ;; Snappy native array copy: 6569 6419 6414 6590 ) \ No newline at end of file diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 17be404..7df1b64 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -1,6 +1,7 @@ (ns taoensso.nippy.utils {:author "Peter Taoussanis"} - (:require [clojure.string :as str])) + (:require [clojure.string :as str]) + (:import org.iq80.snappy.Snappy)) (defmacro case-eval "Like `case` but evaluates test constants for their compile-time value." @@ -55,4 +56,15 @@ (defn version-sufficient? [version-str min-version-str] (try (>= (version-compare version-str min-version-str) 0) - (catch Exception _ false))) \ No newline at end of file + (catch Exception _ false))) + +;; TODO Unnecessarily complicated. Waiting on http://goo.gl/7mbR3 merge. +(defn compress-bytes [ba] + (let [ba-size (alength ^bytes ba) + ba-out (byte-array (Snappy/maxCompressedLength ba-size)) + ba-out-size (Snappy/compress ba (int 0) (int ba-size) ba-out (int 0))] + (java.util.Arrays/copyOf ba-out ba-out-size))) + +(defn uncompress-bytes [ba] (Snappy/uncompress ba 0 (alength ^bytes ba))) + +(comment (String. (uncompress-bytes (compress-bytes (.getBytes "Test"))))) \ No newline at end of file From a1676c80635281130dd60ef5f8637839d0c25747 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Sun, 20 Jan 2013 17:16:03 +0700 Subject: [PATCH 5/8] Bump version (1.1.0-beta2) --- README.md | 4 ++-- project.clj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 561ae8d..e69d4d4 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Current [semantic](http://semver.org/) version: ```clojure -[com.taoensso/nippy "1.0.1"] +[com.taoensso/nippy "1.1.0-beta2"] ``` # Nippy, a Clojure serialization library @@ -25,7 +25,7 @@ Nippy is an attempt to provide a drop-in, high-performance alternative to the re Depend on Nippy in your `project.clj`: ```clojure -[com.taoensso/nippy "1.0.1"] +[com.taoensso/nippy "1.1.0-beta2"] ``` and `require` the library: diff --git a/project.clj b/project.clj index 09184ed..fd30d1d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "1.0.1" +(defproject com.taoensso/nippy "1.1.0-beta2" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License"} From eb9e828647d4a0c2e5ba0a98ee43006b57349aa9 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 5 Feb 2013 20:24:14 +0700 Subject: [PATCH 6/8] Bump dependency (snappy 0.3), simplify `utils/compress-bytes` --- project.clj | 4 ++-- src/taoensso/nippy/utils.clj | 12 ++---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/project.clj b/project.clj index fd30d1d..2175f99 100644 --- a/project.clj +++ b/project.clj @@ -1,9 +1,9 @@ -(defproject com.taoensso/nippy "1.1.0-beta2" +(defproject com.taoensso/nippy "1.1.0-beta3" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License"} :dependencies [[org.clojure/clojure "1.3.0"] - [org.iq80.snappy/snappy "0.2"]] + [org.iq80.snappy/snappy "0.3"]] :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-alpha3"]]} diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 7df1b64..30488e1 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -58,13 +58,5 @@ (try (>= (version-compare version-str min-version-str) 0) (catch Exception _ false))) -;; TODO Unnecessarily complicated. Waiting on http://goo.gl/7mbR3 merge. -(defn compress-bytes [ba] - (let [ba-size (alength ^bytes ba) - ba-out (byte-array (Snappy/maxCompressedLength ba-size)) - ba-out-size (Snappy/compress ba (int 0) (int ba-size) ba-out (int 0))] - (java.util.Arrays/copyOf ba-out ba-out-size))) - -(defn uncompress-bytes [ba] (Snappy/uncompress ba 0 (alength ^bytes ba))) - -(comment (String. (uncompress-bytes (compress-bytes (.getBytes "Test"))))) \ No newline at end of file +(defn compress-bytes [^bytes ba] (Snappy/compress ba)) +(defn uncompress-bytes [^bytes ba] (Snappy/uncompress ba 0 (alength ba))) \ No newline at end of file From 25b84f6488f233df1f2e58fb8d4f05af4e03c6be Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 5 Feb 2013 20:41:26 +0700 Subject: [PATCH 7/8] Add Snappy library compatibility tests --- project.clj | 2 +- test/test_nippy/main.clj | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2175f99..6e88198 100644 --- a/project.clj +++ b/project.clj @@ -8,7 +8,7 @@ :1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-alpha3"]]} :dev {:dependencies []} - :test {:dependencies []}} + :test {:dependencies [[org.xerial.snappy/snappy-java "1.0.5-M3"]]}} :aliases {"test-all" ["with-profile" "test,1.3:test,1.4:test,1.5" "test"]} :min-lein-version "2.0.0" :warn-on-reflection true) diff --git a/test/test_nippy/main.clj b/test/test_nippy/main.clj index bec0e6a..185bd68 100644 --- a/test/test_nippy/main.clj +++ b/test/test_nippy/main.clj @@ -15,3 +15,14 @@ (println (benchmarks/autobench)) (println (benchmarks/autobench)) (println (benchmarks/autobench)) + +(deftest test-snappy-library-compatibility + (let [thaw #(nippy/thaw-from-bytes % :compressed? false) + ^bytes raw-ba (nippy/freeze-to-bytes test-data :compress? false) + ^bytes xerial-ba (org.xerial.snappy.Snappy/compress raw-ba) + ^bytes iq80-ba (org.iq80.snappy.Snappy/compress raw-ba)] + (is (= (thaw raw-ba) + (thaw (org.xerial.snappy.Snappy/uncompress xerial-ba)) + (thaw (org.xerial.snappy.Snappy/uncompress iq80-ba)) + (thaw (org.iq80.snappy.Snappy/uncompress iq80-ba 0 (alength iq80-ba))) + (thaw (org.iq80.snappy.Snappy/uncompress xerial-ba 0 (alength xerial-ba))))))) \ No newline at end of file From 93301850faa3906051b58da3185f37cc4faaaac8 Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Tue, 5 Feb 2013 20:42:31 +0700 Subject: [PATCH 8/8] Bump version (1.1.0) --- README.md | 4 ++-- project.clj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e69d4d4..8d81842 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Current [semantic](http://semver.org/) version: ```clojure -[com.taoensso/nippy "1.1.0-beta2"] +[com.taoensso/nippy "1.1.0"] ``` # Nippy, a Clojure serialization library @@ -25,7 +25,7 @@ Nippy is an attempt to provide a drop-in, high-performance alternative to the re Depend on Nippy in your `project.clj`: ```clojure -[com.taoensso/nippy "1.1.0-beta2"] +[com.taoensso/nippy "1.1.0"] ``` and `require` the library: diff --git a/project.clj b/project.clj index 6e88198..a72f766 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject com.taoensso/nippy "1.1.0-beta3" +(defproject com.taoensso/nippy "1.1.0" :description "Clojure serialization library" :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License"}