luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'dev': v0.9.2.

Browse source
This commit is contained in:
Peter Taoussanis 2012-07-21 02:00:23 +07:00
commit 88d0990339
6 changed files with 86 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.travis.yml
View file

@ -1,6 +1,6 @@
language: clojure
lein: lein2
script: lein2 all test
script: lein2 test-all
jdk:
- openjdk7
- openjdk6

4
README.md
View file

@ -1,7 +1,7 @@
Current [semantic](http://semver.org/) version:
```clojure
[com.taoensso/nippy "0.9.1"]
[com.taoensso/nippy "0.9.2"]
```
# Nippy, a serialization library for Clojure
@ -36,7 +36,7 @@ Nippy uses [Snappy](http://code.google.com/p/snappy-java/) which currently has a
Depend on Nippy in your `project.clj`:
```clojure
[com.taoensso/nippy "0.9.1"]
[com.taoensso/nippy "0.9.2"]
```
and `require` the library:

37
benchmarks/benchmarks.clj
View file

@ -1,9 +1,12 @@
(ns taoensso.nippy.benchmarks
{:author "Peter Taoussanis"}
(:use [taoensso.nippy :as nippy :only (freeze-to-bytes thaw-from-bytes)]))
(:use [taoensso.nippy :as nippy :only (freeze-to-bytes thaw-from-bytes)])
(:require [taoensso.nippy.utils :as utils]))
;; Remove stuff from stress-data that breaks reader
(def bench-data (dissoc nippy/stress-data :queue :queue-empty :bytes))
(def data (dissoc nippy/stress-data :queue :queue-empty :bytes))
(defmacro bench [& body] `(utils/bench 10000 (do ~@body) :warmup-laps 1000))
(defn reader-freeze [x] (binding [*print-dup* false] (pr-str x)))
(defn reader-thaw [x] (binding [*read-eval* false] (read-string x)))
@ -11,30 +14,20 @@
(def roundtrip (comp thaw-from-bytes freeze-to-bytes))
(def reader-roundtrip (comp reader-thaw reader-freeze))
(defmacro time-requests
"Warms up, then executes given number of requests and returns total execution
times in msecs."
[num-requests & body]
`(do (dotimes [_# (int (/ ~num-requests 4))] ~@body) ; Warm-up
(let [start-time# (System/nanoTime)]
(dotimes [_# ~num-requests] ~@body)
(Math/round (/ (- (System/nanoTime) start-time#) 1000000.0)))))
(comment
;;; Times
(println
"---\n"
(let [num 10000]
{:reader {:freeze (time-requests num (reader-freeze bench-data))
:thaw (let [frozen (reader-freeze bench-data)]
(time-requests num (reader-thaw frozen)))
:round (time-requests num (reader-roundtrip bench-data))}
{:reader {:freeze (bench (reader-freeze data))
:thaw (let [frozen (reader-freeze data)]
(bench (reader-thaw frozen)))
:round (bench (reader-roundtrip data))}
:nippy {:freeze (time-requests num (freeze-to-bytes bench-data))
:thaw (let [frozen (freeze-to-bytes bench-data)]
(time-requests num (thaw-from-bytes frozen)))
:round (time-requests num (roundtrip bench-data))}}))
:nippy {:freeze (bench (freeze-to-bytes data))
:thaw (let [frozen (freeze-to-bytes data)]
(bench (thaw-from-bytes frozen)))
:round (bench (roundtrip data))}})
;; Clojure 1.3.0, Nippy 0.9.0
;; {:reader {:freeze 23573, :thaw 31923, :round 53253},
@ -42,7 +35,7 @@
;; (float (/ 53253 7522)) = 7.079633
;;; Data size
(let [frozen (reader-freeze bench-data)] (count (.getBytes frozen "UTF8")))
(let [frozen (freeze-to-bytes bench-data)] (count frozen))
(let [frozen (reader-freeze data)] (count (.getBytes frozen "UTF8")))
(let [frozen (freeze-to-bytes data)] (count frozen))
;; 22711, 12168
)

13
project.clj
View file

@ -1,19 +1,20 @@
(defproject com.taoensso/nippy "0.9.1"
(defproject com.taoensso/nippy "0.9.2"
:description "Simple, high-performance Clojure serialization library."
:url "https://github.com/ptaoussanis/nippy"
:license {:name "Eclipse Public License"}
:dependencies [[org.clojure/clojure "1.3.0"]
[org.xerial.snappy/snappy-java "1.0.4.1"]]
:profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]}
:1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]}
:1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]}
:dev {:dependencies []}}
:profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]}
:1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]}
:1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]}
:dev {:dependencies []}
:test {:dependencies []}}
:repositories {"sonatype" {:url "http://oss.sonatype.org/content/repositories/releases"
:snapshots false
:releases {:checksum :fail :update :always}}
"sonatype-snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots"
:snapshots true
:releases {:checksum :fail :update :always}}}
:aliases {"all" ["with-profile" "1.3:1.4:1.5"]}
:aliases {"test-all" ["with-profile" "test,1.3:test,1.4:test,1.5" "test"]}
:min-lein-version "2.0.0"
:warn-on-reflection true)

57
src/taoensso/nippy.clj
View file

@ -14,18 +14,20 @@
(def ^:const schema-header "\u0000~0.9.0")
(def ^:const id-reader (int 1)) ; Fallback: *print-dup* pr-str output
;; 1
(def ^:const id-bytes (int 2))
(def ^:const id-nil (int 3))
(def ^:const id-boolean (int 4))
(def ^:const id-reader (int 5)) ; Fallback: *print-dup* pr-str output
(def ^:const id-char (int 10))
(def ^:const id-string (int 11))
;; 11
(def ^:const id-keyword (int 12))
(def ^:const id-string (int 13))
(def ^:const id-list (int 20))
(def ^:const id-vector (int 21))
(def ^:const id-old-map (int 22)) ; DEPRECATED as of 0.9.0
;; 22
(def ^:const id-set (int 23))
(def ^:const id-coll (int 24)) ; Fallback: non-specific collection
(def ^:const id-meta (int 25))
@ -44,29 +46,36 @@
(def ^:const id-ratio (int 70))
;;; DEPRECATED (old types will be supported only for thawing)
(def ^:const id-old-reader (int 1)) ; as of 0.9.2, for +64k support
(def ^:const id-old-string (int 11)) ; as of 0.9.2, for +64k support
(def ^:const id-old-map (int 22)) ; as of 0.9.0, for more efficient thaw
;;;; Shared low-level stream stuff
(defn- write-id! [^DataOutputStream stream ^Integer id] (.writeByte stream id))
(defn- write-bytes!
"Writes arbitrary byte data, preceded by its length."
[^DataOutputStream stream ^bytes ba]
(let [size (alength ba)]
(.writeInt stream size) ; Encode size of byte array
(.write stream ba 0 size)))
(defn- write-biginteger!
"Wrapper around `write-bytes!` for common case of writing a BigInteger."
[^DataOutputStream stream ^BigInteger x]
(write-bytes! stream (.toByteArray x)))
(defn- read-bytes!
"Reads arbitrary byte data, preceded by its length."
^bytes [^DataInputStream stream]
(let [size (.readInt stream)
ba (byte-array size)]
(.read stream ba 0 size) ba))
(defn- write-as-bytes!
"Write arbitrary object as bytes using reflection."
[^DataOutputStream stream obj]
(write-bytes! stream (.toByteArray obj)))
(defn- read-biginteger!
"Wrapper around read-bytes! for common case of reading to a BigInteger.
"Wrapper around `read-bytes!` for common case of reading a BigInteger.
Note that as of Clojure 1.3, java.math.BigInteger ≠ clojure.lang.BigInt."
^BigInteger [^DataInputStream stream]
(BigInteger. (read-bytes! stream)))
@ -99,7 +108,7 @@
(freezer Boolean id-boolean (.writeBoolean s x))
(freezer Character id-char (.writeChar s (int x)))
(freezer String id-string (.writeUTF s x))
(freezer String id-string (write-bytes! s (.getBytes x "UTF-8")))
(freezer Keyword id-keyword (.writeUTF s (if-let [ns (namespace x)]
(str ns "/" (name x))
(name x))))
@ -121,21 +130,21 @@
(freezer Short id-short (.writeShort s x))
(freezer Integer id-integer (.writeInt s x))
(freezer Long id-long (.writeLong s x))
(freezer BigInt id-bigint (write-as-bytes! s (.toBigInteger x)))
(freezer BigInteger id-bigint (write-as-bytes! s x))
(freezer BigInt id-bigint (write-biginteger! s (.toBigInteger x)))
(freezer BigInteger id-bigint (write-biginteger! s x))
(freezer Float id-float (.writeFloat s x))
(freezer Double id-double (.writeDouble s x))
(freezer BigDecimal id-bigdec
(write-as-bytes! s (.unscaledValue x))
(write-biginteger! s (.unscaledValue x))
(.writeInt s (.scale x)))
(freezer Ratio id-ratio
(write-as-bytes! s (.numerator x))
(write-as-bytes! s (.denominator x)))
(write-biginteger! s (.numerator x))
(write-biginteger! s (.denominator x)))
;; Use Clojure's own reader as final fallback
(freezer Object id-reader (.writeUTF s (pr-str x)))
(freezer Object id-reader (write-bytes! s (.getBytes (pr-str x) "UTF-8")))
(defn- freeze-to-stream!* [^DataOutputStream s x]
(if-let [m (meta x)]
@ -175,13 +184,13 @@
(utils/case-eval
type-id
id-reader (read-string (.readUTF s))
id-reader (read-string (String. (read-bytes! s) "UTF-8"))
id-bytes (read-bytes! s)
id-nil nil
id-boolean (.readBoolean s)
id-char (.readChar s)
id-string (.readUTF s)
id-string (String. (read-bytes! s) "UTF-8")
id-keyword (keyword (.readUTF s))
id-list (apply list (coll-thaw! s))
@ -191,10 +200,6 @@
id-coll (doall (coll-thaw! s))
id-queue (into (PersistentQueue/EMPTY) (coll-thaw! s))
;; DEPRECATED as of 0.9.0
id-old-map (apply hash-map (repeatedly (* 2 (.readInt s))
(partial thaw-from-stream!* s)))
id-meta (let [m (thaw-from-stream!* s)] (with-meta (thaw-from-stream!* s) m))
id-byte (.readByte s)
@ -210,6 +215,12 @@
id-ratio (/ (bigint (read-biginteger! s))
(bigint (read-biginteger! s)))
;;; DEPRECATED
id-old-reader (read-string (.readUTF s))
id-old-string (.readUTF s)
id-old-map (apply hash-map (repeatedly (* 2 (.readInt s))
(partial thaw-from-stream!* s)))
(throw (Exception. (str "Failed to thaw unknown type ID: " type-id))))))
;; TODO Scheduled for Carmine version 1.0.0
@ -254,7 +265,7 @@
:string-utf8 "ಬಾ ಇಲ್ಲಿ ಸಂಭವಿಸ"
:string-long (apply str (range 1000))
:keyword :keyword
:ns-keyword ::keyword
:keyword-ns ::keyword
:list (list 1 2 3 4 5 (list 6 7 8 (list 9 10)))
:list-quoted '(1 2 3 4 5 (6 7 8 (9 10)))

25
src/taoensso/nippy/utils.clj
View file

@ -11,3 +11,28 @@
~@(map-indexed (fn [i# form#] (if (even? i#) (eval form#) form#))
clauses)
~(when default default))))
(defmacro time-ns
"Returns number of nanoseconds it takes to execute body."
[& body]
`(let [t0# (System/nanoTime)]
~@body
(- (System/nanoTime) t0#)))
(defmacro bench
"Repeatedly executes form and returns time taken to complete execution."
[num-laps form & {:keys [warmup-laps num-threads as-ms?]
:or {as-ms? true}}]
`(try (when ~warmup-laps (dotimes [_# ~warmup-laps] ~form))
(let [nanosecs#
(if-not ~num-threads
(time-ns (dotimes [_# ~num-laps] ~form))
(let [laps-per-thread# (int (/ ~num-laps ~num-threads))]
(time-ns
(->> (fn [] (future (dotimes [_# laps-per-thread#] ~form)))
(repeatedly ~num-threads)
doall
(map deref)
dorun))))]
(if ~as-ms? (Math/round (/ nanosecs# 1000000.0)) nanosecs#))
(catch Exception e# (str "DNF: " (.getMessage e#)))))