luciano/nippy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Misc hk

Browse source
This commit is contained in:
Peter Taoussanis 2016-05-07 11:18:19 +07:00
parent cac9123794
commit 0df6a7b0f3
3 changed files with 71 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
project.clj
View file

@ -15,7 +15,7 @@
:dependencies :dependencies
[[org.clojure/clojure "1.5.1"] [[org.clojure/clojure "1.5.1"]
[org.clojure/tools.reader "0.10.0"] [org.clojure/tools.reader "0.10.0"]
[com.taoensso/encore "2.49.0"] [com.taoensso/encore "2.52.1"]
[org.iq80.snappy/snappy "0.4"] [org.iq80.snappy/snappy "0.4"]
[org.tukaani/xz "1.5"] [org.tukaani/xz "1.5"]
[net.jpountz.lz4/lz4 "1.3"]] [net.jpountz.lz4/lz4 "1.3"]]

90
src/taoensso/nippy.clj
View file

@ -1,7 +1,7 @@
(ns taoensso.nippy (ns taoensso.nippy
"High-performance serialization library for Clojure" "High-performance serialization library for Clojure"
{:author "Peter Taoussanis (@ptaoussanis)"} {:author "Peter Taoussanis (@ptaoussanis)"}
(:require [taoensso.encore :as enc] (:require [taoensso.encore :as enc :refer (cond*)]
[taoensso.nippy [taoensso.nippy
(utils :as utils) (utils :as utils)
(compression :as compression) (compression :as compression)
@ -10,7 +10,7 @@
DataOutputStream Serializable ObjectOutputStream ObjectInputStream DataOutputStream Serializable ObjectOutputStream ObjectInputStream
DataOutput DataInput] DataOutput DataInput]
[java.lang.reflect Method] [java.lang.reflect Method]
[java.net URI] ;; [java.net URI] ; TODO
[java.util Date UUID] [java.util Date UUID]
[java.util.regex Pattern] [java.util.regex Pattern]
[clojure.lang Keyword Symbol BigInt Ratio [clojure.lang Keyword Symbol BigInt Ratio
@ -20,8 +20,8 @@
LazySeq IRecord ISeq])) LazySeq IRecord ISeq]))
(if (vector? enc/encore-version) (if (vector? enc/encore-version)
(enc/assert-min-encore-version [2 49 0]) (enc/assert-min-encore-version [2 52 1])
(enc/assert-min-encore-version 2.49)) (enc/assert-min-encore-version 2.52))
(comment (comment
(set! *unchecked-math* :warn-on-boxed) (set! *unchecked-math* :warn-on-boxed)
@ -322,7 +322,7 @@
(if (zero? len) (if (zero? len)
(write-id out id-bytes-0) (write-id out id-bytes-0)
(do (do
(cond (cond*
(sm-count? len) (sm-count? len)
(do (write-id out id-bytes-sm) (do (write-id out id-bytes-sm)
(write-sm-count out len)) (write-sm-count out len))
@ -349,7 +349,7 @@
(write-id out id-str-0) (write-id out id-str-0)
(let [ba (.getBytes s charset) (let [ba (.getBytes s charset)
len (alength ba)] len (alength ba)]
(cond (cond*
(sm-count? len) (sm-count? len)
(do (write-id out id-str-sm) (do (write-id out id-str-sm)
(write-sm-count out len)) (write-sm-count out len))
@ -368,7 +368,7 @@
(let [s (if-let [ns (namespace kw)] (str ns "/" (name kw)) (name kw)) (let [s (if-let [ns (namespace kw)] (str ns "/" (name kw)) (name kw))
ba (.getBytes s charset) ba (.getBytes s charset)
len (alength ba)] len (alength ba)]
(cond (cond*
(sm-count? len) (sm-count? len)
(do (write-id out id-kw-sm) (do (write-id out id-kw-sm)
(write-sm-count out len)) (write-sm-count out len))
@ -383,7 +383,7 @@
(let [s (if-let [ns (namespace s)] (str ns "/" (name s)) (name s)) (let [s (if-let [ns (namespace s)] (str ns "/" (name s)) (name s))
ba (.getBytes s charset) ba (.getBytes s charset)
len (alength ba)] len (alength ba)]
(cond (cond*
(sm-count? len) (sm-count? len)
(do (write-id out id-sym-sm) (do (write-id out id-sym-sm)
(write-sm-count out len)) (write-sm-count out len))
@ -395,12 +395,12 @@
(.write out ba 0 len))) (.write out ba 0 len)))
(defn- write-long [^DataOutput out ^long n] (defn- write-long [^DataOutput out ^long n]
(cond (cond*
(zero? n) (zero? n)
(write-id out id-long-zero) (write-id out id-long-zero)
(> n 0) (> n 0)
(cond (cond*
(<= n 127 #_Byte/MAX_VALUE) (<= n 127 #_Byte/MAX_VALUE)
(do (write-id out id-long-sm) (do (write-id out id-long-sm)
(.writeByte out n)) (.writeByte out n))
@ -418,7 +418,7 @@
(.writeLong out n))) (.writeLong out n)))
:else :else
(cond (cond*
(>= n -128 #_Byte/MIN_VALUE) (>= n -128 #_Byte/MIN_VALUE)
(do (write-id out id-long-sm) (do (write-id out id-long-sm)
(.writeByte out n)) (.writeByte out n))
@ -445,9 +445,9 @@
(if (zero? cnt) (if (zero? cnt)
(write-id out id-vec-0) (write-id out id-vec-0)
(do (do
(cond (cond*
(sm-count? cnt) (sm-count? cnt)
(cond (cond*
(== cnt 2) (write-id out id-vec-2) (== cnt 2) (write-id out id-vec-2)
(== cnt 3) (write-id out id-vec-3) (== cnt 3) (write-id out id-vec-3)
:else :else
@ -480,7 +480,7 @@
(if (zero? cnt) (if (zero? cnt)
(write-id out id-empty) (write-id out id-empty)
(do (do
(cond (cond*
(sm-count? cnt) (sm-count? cnt)
(do (write-id out id-sm) (do (write-id out id-sm)
(write-sm-count out cnt)) (write-sm-count out cnt))
@ -513,7 +513,7 @@
(if (zero? cnt) (if (zero? cnt)
(write-id out id-empty) (write-id out id-empty)
(do (do
(cond (cond*
(sm-count? cnt) (sm-count? cnt)
(do (write-id out id-sm) (do (write-id out id-sm)
(write-sm-count out cnt)) (write-sm-count out cnt))
@ -549,7 +549,7 @@
(if (zero? cnt) (if (zero? cnt)
(write-id out id-empty) (write-id out id-empty)
(do (do
(cond (cond*
(sm-count? cnt) (sm-count? cnt)
(do (write-id out id-sm) (do (write-id out id-sm)
(write-sm-count out cnt)) (write-sm-count out cnt))
@ -582,7 +582,7 @@
(if (zero? cnt) (if (zero? cnt)
(write-id out id-map-0) (write-id out id-map-0)
(do (do
(cond (cond*
(sm-count? cnt) (sm-count? cnt)
(do (write-id out id-map-sm) (do (write-id out id-map-sm)
(write-sm-count out cnt)) (write-sm-count out cnt))
@ -608,7 +608,7 @@
(if (zero? cnt) (if (zero? cnt)
(write-id out id-set-0) (write-id out id-set-0)
(do (do
(cond (cond*
(sm-count? cnt) (sm-count? cnt)
(do (write-id out id-set-sm) (do (write-id out id-set-sm)
(write-sm-count out cnt)) (write-sm-count out cnt))
@ -628,7 +628,7 @@
(let [cname (.getName (class x)) ; Reflect (let [cname (.getName (class x)) ; Reflect
cname-ba (.getBytes cname charset) cname-ba (.getBytes cname charset)
len (alength cname-ba)] len (alength cname-ba)]
(cond (cond*
(sm-count? len) (sm-count? len)
(do (write-id out id-serializable-sm) (do (write-id out id-serializable-sm)
(write-bytes-sm out cname-ba)) (write-bytes-sm out cname-ba))
@ -644,7 +644,7 @@
(let [edn (enc/pr-edn x) (let [edn (enc/pr-edn x)
edn-ba (.getBytes ^String edn charset) edn-ba (.getBytes ^String edn charset)
len (alength edn-ba)] len (alength edn-ba)]
(cond (cond*
(sm-count? len) (sm-count? len)
(do (write-id out id-reader-sm) (do (write-id out id-reader-sm)
(write-bytes-sm out edn-ba)) (write-bytes-sm out edn-ba))
@ -731,39 +731,29 @@
(freezer CacheWrapped (freezer CacheWrapped
(let [x-val (:value x)] (let [x-val (:value x)]
(if-let [cache_ *cache_*] (if-let [cache_ *cache_*]
(let [cache @cache_ (let [cache @cache_
?idx (get cache x-val) ?idx (get cache x-val)
idx (or ?idx ^int idx (or ?idx
(let [idx (count cache)] (let [idx (count cache)]
(enc/-vol-swap! cache_ assoc x-val idx) (enc/-vol-swap! cache_ assoc x-val idx)
idx)) idx))
first-occurance? (nil? ?idx)] first-occurance? (nil? ?idx)]
(cond (cond*
(sm-count? idx) (sm-count? idx)
(cond (case (int idx)
(== idx 0) 0 (do (write-id out id-cached-0)
(do (write-id out id-cached-0) (when first-occurance? (-freeze-to-out! x-val out)))
(when first-occurance? (-freeze-to-out! x-val out))) 1 (do (write-id out id-cached-1)
(when first-occurance? (-freeze-to-out! x-val out)))
2 (do (write-id out id-cached-2)
(when first-occurance? (-freeze-to-out! x-val out)))
3 (do (write-id out id-cached-3)
(when first-occurance? (-freeze-to-out! x-val out)))
4 (do (write-id out id-cached-4)
(when first-occurance? (-freeze-to-out! x-val out)))
(== idx 1)
(do (write-id out id-cached-1)
(when first-occurance? (-freeze-to-out! x-val out)))
(== idx 2)
(do (write-id out id-cached-2)
(when first-occurance? (-freeze-to-out! x-val out)))
(== idx 3)
(do (write-id out id-cached-3)
(when first-occurance? (-freeze-to-out! x-val out)))
(== idx 4)
(do (write-id out id-cached-4)
(when first-occurance? (-freeze-to-out! x-val out)))
:else
(do (write-id out id-cached-sm) (do (write-id out id-cached-sm)
(write-sm-count out idx) (write-sm-count out idx)
(when first-occurance? (-freeze-to-out! x-val out)))) (when first-occurance? (-freeze-to-out! x-val out))))
@ -842,7 +832,7 @@
(let [cname (.getName (class x)) ; Reflect (let [cname (.getName (class x)) ; Reflect
cname-ba (.getBytes cname charset) cname-ba (.getBytes cname charset)
len (alength cname-ba)] len (alength cname-ba)]
(cond (cond*
(sm-count? len) (sm-count? len)
(do (write-id out id-record-sm) (do (write-id out id-record-sm)
(write-bytes-sm out cname-ba)) (write-bytes-sm out cname-ba))
@ -1488,7 +1478,7 @@
(when (enc/bytes? ba) (when (enc/bytes? ba)
(let [[first2bytes nextbytes] (enc/ba-split ba 2) (let [[first2bytes nextbytes] (enc/ba-split ba 2)
?known-wrapper ?known-wrapper
(cond (cond*
(enc/ba= first2bytes (.getBytes "\u0000<" charset)) :carmine/bin (enc/ba= first2bytes (.getBytes "\u0000<" charset)) :carmine/bin
(enc/ba= first2bytes (.getBytes "\u0000>" charset)) :carmine/clj) (enc/ba= first2bytes (.getBytes "\u0000>" charset)) :carmine/clj)

56
src/taoensso/nippy/utils.clj
View file

@ -70,6 +70,8 @@
(comment (is-coll? (clojure.lang.PersistentVector$ChunkedSeq. [1 2 3] 0 0))) (comment (is-coll? (clojure.lang.PersistentVector$ChunkedSeq. [1 2 3] 0 0)))
(defmacro ^:private is? [x c] `(when (instance? ~c ~x) ~c))
(defn freezable? (defn freezable?
"Alpha - subject to change. "Alpha - subject to change.
Returns truthy iff Nippy *appears* to support freezing the given argument. Returns truthy iff Nippy *appears* to support freezing the given argument.
@ -82,38 +84,40 @@
([x] (freezable? x nil)) ([x] (freezable? x nil))
([x {:keys [allow-clojure-reader? allow-java-serializable?]}] ([x {:keys [allow-clojure-reader? allow-java-serializable?]}]
(let [is? #(when (instance? % x) %)] (if (is-coll? x)
(if (is-coll? x) (when (enc/revery? freezable? x) (type x))
(when (enc/revery? freezable? x) (type x)) (or
(or (is? x clojure.lang.Keyword)
(is? clojure.lang.Keyword) (is? x java.lang.String)
(is? java.lang.String) (is? x java.lang.Long)
(is? java.lang.Long) (is? x java.lang.Double)
(is? java.lang.Double)
(is? clojure.lang.BigInt) (is? x clojure.lang.BigInt)
(is? clojure.lang.Ratio) (is? x clojure.lang.Ratio)
(is? java.lang.Boolean) (is? x java.lang.Boolean)
(is? java.lang.Integer) (is? x java.lang.Integer)
(is? java.lang.Short) (is? x java.lang.Short)
(is? java.lang.Byte) (is? x java.lang.Byte)
(is? java.lang.Character) (is? x java.lang.Character)
(is? java.math.BigInteger) (is? x java.math.BigInteger)
(is? java.math.BigDecimal) (is? x java.math.BigDecimal)
(is? #=(java.lang.Class/forName "[B")) (is? x #=(java.lang.Class/forName "[B"))
(is? java.util.Date) (is? x clojure.lang.Symbol)
(is? java.util.UUID)
(when (and allow-clojure-reader? (readable? x)) :clojure-reader) (is? x java.util.Date)
(when (and allow-java-serializable? (is? x java.util.UUID)
;; Reports as true but is unreliable: (is? x java.util.regex.Pattern)
(not (is? clojure.lang.Fn))
(serializable? x)) :java-serializable)))))) (when (and allow-clojure-reader? (readable? x)) :clojure-reader)
(when (and allow-java-serializable?
;; Reports as true but is unreliable:
(not (is? x clojure.lang.Fn))
(serializable? x)) :java-serializable)))))
(comment (comment
(enc/qb 10000 (freezable? "hello")) (enc/qb 10000 (freezable? "hello")) ; 0.79
(freezable? [:a :b]) (freezable? [:a :b])
(freezable? [:a (fn [x] (* x x))]) (freezable? [:a (fn [x] (* x x))])
(freezable? (.getBytes "foo")) (freezable? (.getBytes "foo"))