diff --git a/.travis.yml b/.travis.yml index e52c379..0d9ade9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: clojure lein: lein2 -script: lein2 all test +script: lein2 test-all jdk: - openjdk7 - openjdk6 - - oraclejdk7 \ No newline at end of file + - oraclejdk7 diff --git a/README.md b/README.md index dd7effd..2058aeb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Current [semantic](http://semver.org/) version: ```clojure -[com.taoensso/nippy "0.9.1"] +[com.taoensso/nippy "0.9.2"] ``` # Nippy, a serialization library for Clojure @@ -36,7 +36,7 @@ Nippy uses [Snappy](http://code.google.com/p/snappy-java/) which currently has a Depend on Nippy in your `project.clj`: ```clojure -[com.taoensso/nippy "0.9.1"] +[com.taoensso/nippy "0.9.2"] ``` and `require` the library: diff --git a/benchmarks/benchmarks.clj b/benchmarks/benchmarks.clj index d83c1ad..b0b1825 100644 --- a/benchmarks/benchmarks.clj +++ b/benchmarks/benchmarks.clj @@ -1,9 +1,12 @@ (ns taoensso.nippy.benchmarks {:author "Peter Taoussanis"} - (:use [taoensso.nippy :as nippy :only (freeze-to-bytes thaw-from-bytes)])) + (:use [taoensso.nippy :as nippy :only (freeze-to-bytes thaw-from-bytes)]) + (:require [taoensso.nippy.utils :as utils])) ;; Remove stuff from stress-data that breaks reader -(def bench-data (dissoc nippy/stress-data :queue :queue-empty :bytes)) +(def data (dissoc nippy/stress-data :queue :queue-empty :bytes)) + +(defmacro bench [& body] `(utils/bench 10000 (do ~@body) :warmup-laps 1000)) (defn reader-freeze [x] (binding [*print-dup* false] (pr-str x))) (defn reader-thaw [x] (binding [*read-eval* false] (read-string x))) @@ -11,30 +14,20 @@ (def roundtrip (comp thaw-from-bytes freeze-to-bytes)) (def reader-roundtrip (comp reader-thaw reader-freeze)) -(defmacro time-requests - "Warms up, then executes given number of requests and returns total execution - times in msecs." - [num-requests & body] - `(do (dotimes [_# (int (/ ~num-requests 4))] ~@body) ; Warm-up - (let [start-time# (System/nanoTime)] - (dotimes [_# ~num-requests] ~@body) - (Math/round (/ (- (System/nanoTime) start-time#) 1000000.0))))) - (comment ;;; Times (println "---\n" - (let [num 10000] - {:reader {:freeze (time-requests num (reader-freeze bench-data)) - :thaw (let [frozen (reader-freeze bench-data)] - (time-requests num (reader-thaw frozen))) - :round (time-requests num (reader-roundtrip bench-data))} + {:reader {:freeze (bench (reader-freeze data)) + :thaw (let [frozen (reader-freeze data)] + (bench (reader-thaw frozen))) + :round (bench (reader-roundtrip data))} - :nippy {:freeze (time-requests num (freeze-to-bytes bench-data)) - :thaw (let [frozen (freeze-to-bytes bench-data)] - (time-requests num (thaw-from-bytes frozen))) - :round (time-requests num (roundtrip bench-data))}})) + :nippy {:freeze (bench (freeze-to-bytes data)) + :thaw (let [frozen (freeze-to-bytes data)] + (bench (thaw-from-bytes frozen))) + :round (bench (roundtrip data))}}) ;; Clojure 1.3.0, Nippy 0.9.0 ;; {:reader {:freeze 23573, :thaw 31923, :round 53253}, @@ -42,7 +35,7 @@ ;; (float (/ 53253 7522)) = 7.079633 ;;; Data size - (let [frozen (reader-freeze bench-data)] (count (.getBytes frozen "UTF8"))) - (let [frozen (freeze-to-bytes bench-data)] (count frozen)) + (let [frozen (reader-freeze data)] (count (.getBytes frozen "UTF8"))) + (let [frozen (freeze-to-bytes data)] (count frozen)) ;; 22711, 12168 ) \ No newline at end of file diff --git a/project.clj b/project.clj index 7021567..746d8c0 100644 --- a/project.clj +++ b/project.clj @@ -1,19 +1,20 @@ -(defproject com.taoensso/nippy "0.9.1" +(defproject com.taoensso/nippy "0.9.2" :description "Simple, high-performance Clojure serialization library." :url "https://github.com/ptaoussanis/nippy" :license {:name "Eclipse Public License"} :dependencies [[org.clojure/clojure "1.3.0"] [org.xerial.snappy/snappy-java "1.0.4.1"]] - :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} - :1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} - :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} - :dev {:dependencies []}} + :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} + :1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} + :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} + :dev {:dependencies []} + :test {:dependencies []}} :repositories {"sonatype" {:url "http://oss.sonatype.org/content/repositories/releases" :snapshots false :releases {:checksum :fail :update :always}} "sonatype-snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots" :snapshots true :releases {:checksum :fail :update :always}}} - :aliases {"all" ["with-profile" "1.3:1.4:1.5"]} + :aliases {"test-all" ["with-profile" "test,1.3:test,1.4:test,1.5" "test"]} :min-lein-version "2.0.0" :warn-on-reflection true) \ No newline at end of file diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 14a3745..a14e473 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -14,18 +14,20 @@ (def ^:const schema-header "\u0000~0.9.0") -(def ^:const id-reader (int 1)) ; Fallback: *print-dup* pr-str output +;; 1 (def ^:const id-bytes (int 2)) (def ^:const id-nil (int 3)) (def ^:const id-boolean (int 4)) +(def ^:const id-reader (int 5)) ; Fallback: *print-dup* pr-str output (def ^:const id-char (int 10)) -(def ^:const id-string (int 11)) +;; 11 (def ^:const id-keyword (int 12)) +(def ^:const id-string (int 13)) (def ^:const id-list (int 20)) (def ^:const id-vector (int 21)) -(def ^:const id-old-map (int 22)) ; DEPRECATED as of 0.9.0 +;; 22 (def ^:const id-set (int 23)) (def ^:const id-coll (int 24)) ; Fallback: non-specific collection (def ^:const id-meta (int 25)) @@ -44,29 +46,36 @@ (def ^:const id-ratio (int 70)) +;;; DEPRECATED (old types will be supported only for thawing) +(def ^:const id-old-reader (int 1)) ; as of 0.9.2, for +64k support +(def ^:const id-old-string (int 11)) ; as of 0.9.2, for +64k support +(def ^:const id-old-map (int 22)) ; as of 0.9.0, for more efficient thaw + ;;;; Shared low-level stream stuff (defn- write-id! [^DataOutputStream stream ^Integer id] (.writeByte stream id)) (defn- write-bytes! + "Writes arbitrary byte data, preceded by its length." [^DataOutputStream stream ^bytes ba] (let [size (alength ba)] (.writeInt stream size) ; Encode size of byte array (.write stream ba 0 size))) +(defn- write-biginteger! + "Wrapper around `write-bytes!` for common case of writing a BigInteger." + [^DataOutputStream stream ^BigInteger x] + (write-bytes! stream (.toByteArray x))) + (defn- read-bytes! + "Reads arbitrary byte data, preceded by its length." ^bytes [^DataInputStream stream] (let [size (.readInt stream) ba (byte-array size)] (.read stream ba 0 size) ba)) -(defn- write-as-bytes! - "Write arbitrary object as bytes using reflection." - [^DataOutputStream stream obj] - (write-bytes! stream (.toByteArray obj))) - (defn- read-biginteger! - "Wrapper around read-bytes! for common case of reading to a BigInteger. + "Wrapper around `read-bytes!` for common case of reading a BigInteger. Note that as of Clojure 1.3, java.math.BigInteger ≠ clojure.lang.BigInt." ^BigInteger [^DataInputStream stream] (BigInteger. (read-bytes! stream))) @@ -99,7 +108,7 @@ (freezer Boolean id-boolean (.writeBoolean s x)) (freezer Character id-char (.writeChar s (int x))) -(freezer String id-string (.writeUTF s x)) +(freezer String id-string (write-bytes! s (.getBytes x "UTF-8"))) (freezer Keyword id-keyword (.writeUTF s (if-let [ns (namespace x)] (str ns "/" (name x)) (name x)))) @@ -121,21 +130,21 @@ (freezer Short id-short (.writeShort s x)) (freezer Integer id-integer (.writeInt s x)) (freezer Long id-long (.writeLong s x)) -(freezer BigInt id-bigint (write-as-bytes! s (.toBigInteger x))) -(freezer BigInteger id-bigint (write-as-bytes! s x)) +(freezer BigInt id-bigint (write-biginteger! s (.toBigInteger x))) +(freezer BigInteger id-bigint (write-biginteger! s x)) (freezer Float id-float (.writeFloat s x)) (freezer Double id-double (.writeDouble s x)) (freezer BigDecimal id-bigdec - (write-as-bytes! s (.unscaledValue x)) + (write-biginteger! s (.unscaledValue x)) (.writeInt s (.scale x))) (freezer Ratio id-ratio - (write-as-bytes! s (.numerator x)) - (write-as-bytes! s (.denominator x))) + (write-biginteger! s (.numerator x)) + (write-biginteger! s (.denominator x))) ;; Use Clojure's own reader as final fallback -(freezer Object id-reader (.writeUTF s (pr-str x))) +(freezer Object id-reader (write-bytes! s (.getBytes (pr-str x) "UTF-8"))) (defn- freeze-to-stream!* [^DataOutputStream s x] (if-let [m (meta x)] @@ -175,13 +184,13 @@ (utils/case-eval type-id - id-reader (read-string (.readUTF s)) + id-reader (read-string (String. (read-bytes! s) "UTF-8")) id-bytes (read-bytes! s) id-nil nil id-boolean (.readBoolean s) id-char (.readChar s) - id-string (.readUTF s) + id-string (String. (read-bytes! s) "UTF-8") id-keyword (keyword (.readUTF s)) id-list (apply list (coll-thaw! s)) @@ -191,10 +200,6 @@ id-coll (doall (coll-thaw! s)) id-queue (into (PersistentQueue/EMPTY) (coll-thaw! s)) - ;; DEPRECATED as of 0.9.0 - id-old-map (apply hash-map (repeatedly (* 2 (.readInt s)) - (partial thaw-from-stream!* s))) - id-meta (let [m (thaw-from-stream!* s)] (with-meta (thaw-from-stream!* s) m)) id-byte (.readByte s) @@ -210,6 +215,12 @@ id-ratio (/ (bigint (read-biginteger! s)) (bigint (read-biginteger! s))) + ;;; DEPRECATED + id-old-reader (read-string (.readUTF s)) + id-old-string (.readUTF s) + id-old-map (apply hash-map (repeatedly (* 2 (.readInt s)) + (partial thaw-from-stream!* s))) + (throw (Exception. (str "Failed to thaw unknown type ID: " type-id)))))) ;; TODO Scheduled for Carmine version 1.0.0 @@ -254,7 +265,7 @@ :string-utf8 "ಬಾ ಇಲ್ಲಿ ಸಂಭವಿಸ" :string-long (apply str (range 1000)) :keyword :keyword - :ns-keyword ::keyword + :keyword-ns ::keyword :list (list 1 2 3 4 5 (list 6 7 8 (list 9 10))) :list-quoted '(1 2 3 4 5 (6 7 8 (9 10))) diff --git a/src/taoensso/nippy/utils.clj b/src/taoensso/nippy/utils.clj index 6668f1c..fce02b6 100644 --- a/src/taoensso/nippy/utils.clj +++ b/src/taoensso/nippy/utils.clj @@ -10,4 +10,29 @@ `(case ~e ~@(map-indexed (fn [i# form#] (if (even? i#) (eval form#) form#)) clauses) - ~(when default default)))) \ No newline at end of file + ~(when default default)))) + +(defmacro time-ns + "Returns number of nanoseconds it takes to execute body." + [& body] + `(let [t0# (System/nanoTime)] + ~@body + (- (System/nanoTime) t0#))) + +(defmacro bench + "Repeatedly executes form and returns time taken to complete execution." + [num-laps form & {:keys [warmup-laps num-threads as-ms?] + :or {as-ms? true}}] + `(try (when ~warmup-laps (dotimes [_# ~warmup-laps] ~form)) + (let [nanosecs# + (if-not ~num-threads + (time-ns (dotimes [_# ~num-laps] ~form)) + (let [laps-per-thread# (int (/ ~num-laps ~num-threads))] + (time-ns + (->> (fn [] (future (dotimes [_# laps-per-thread#] ~form))) + (repeatedly ~num-threads) + doall + (map deref) + dorun))))] + (if ~as-ms? (Math/round (/ nanosecs# 1000000.0)) nanosecs#)) + (catch Exception e# (str "DNF: " (.getMessage e#))))) \ No newline at end of file