NB migrate utils stuff to encore lib
This commit is contained in:
Peter Taoussanis
parent
c58e2f617c
commit
54389cc3a0
8 changed files with 31 additions and 134 deletions
|
|
@ -44,7 +44,7 @@
|
|||
"test-auto" ["with-profile" "+test" "autoexpect"]
|
||||
"start-dev" ["with-profile" "+dev" "repl" ":headless"]
|
||||
"codox" ["with-profile" "+test" "doc"]
|
||||
"deploy-lib" ["with-profile" "+dev,+build" "deploy" "clojars"]}
|
||||
"deploy-lib" ["with-profile" "+dev,+build" "do" "deploy" "clojars," "install"]}
|
||||
|
||||
:repositories
|
||||
{"sonatype"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
from Deep-Freeze."
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [clojure.tools.reader.edn :as edn]
|
||||
[taoensso.encore :as encore]
|
||||
[taoensso.nippy
|
||||
(utils :as utils)
|
||||
(compression :as compression :refer (snappy-compressor))
|
||||
|
|
@ -302,8 +303,8 @@
|
|||
|
||||
(defn- wrap-header [data-ba metadata]
|
||||
(if-let [meta-id (head-meta-id (assoc metadata :version head-version))]
|
||||
(let [head-ba (utils/ba-concat head-sig (byte-array [meta-id]))]
|
||||
(utils/ba-concat head-ba data-ba))
|
||||
(let [head-ba (encore/ba-concat head-sig (byte-array [meta-id]))]
|
||||
(encore/ba-concat head-ba data-ba))
|
||||
(throw (Exception. (str "Unrecognized header metadata: " metadata)))))
|
||||
|
||||
(comment (wrap-header (.getBytes "foo") {:compressed? true
|
||||
|
|
@ -356,10 +357,10 @@
|
|||
`(long (BigInteger. (read-bytes ~in :small))))
|
||||
|
||||
(defmacro ^:private read-coll [in coll]
|
||||
`(let [in# ~in] (utils/repeatedly-into ~coll (.readInt in#) (thaw-from-in in#))))
|
||||
`(let [in# ~in] (encore/repeatedly-into* ~coll (.readInt in#) (thaw-from-in in#))))
|
||||
|
||||
(defmacro ^:private read-kvs [in coll]
|
||||
`(let [in# ~in] (utils/repeatedly-into ~coll (/ (.readInt in#) 2)
|
||||
`(let [in# ~in] (encore/repeatedly-into* ~coll (/ (.readInt in#) 2)
|
||||
[(thaw-from-in in#) (thaw-from-in in#)])))
|
||||
|
||||
(declare ^:private custom-readers)
|
||||
|
|
@ -371,7 +372,7 @@
|
|||
(when-debug-mode
|
||||
(println (format "DEBUG - thawing type-id: %s" type-id)))
|
||||
|
||||
(utils/case-eval type-id
|
||||
(encore/case-eval type-id
|
||||
|
||||
id-reader
|
||||
(let [edn (read-utf8 in)]
|
||||
|
|
@ -446,7 +447,7 @@
|
|||
;;; DEPRECATED
|
||||
id-old-reader (edn/read-string (.readUTF in))
|
||||
id-old-string (.readUTF in)
|
||||
id-old-map (apply hash-map (utils/repeatedly-into []
|
||||
id-old-map (apply hash-map (encore/repeatedly-into* []
|
||||
(* 2 (.readInt in)) (thaw-from-in in)))
|
||||
id-old-keyword (keyword (.readUTF in))
|
||||
|
||||
|
|
@ -472,9 +473,9 @@
|
|||
(thaw-from-in data-input))
|
||||
|
||||
(defn- try-parse-header [ba]
|
||||
(when-let [[head-ba data-ba] (utils/ba-split ba 4)]
|
||||
(let [[head-sig* [meta-id]] (utils/ba-split head-ba 3)]
|
||||
(when (utils/ba= head-sig* head-sig) ; Appears to be well-formed
|
||||
(when-let [[head-ba data-ba] (encore/ba-split ba 4)]
|
||||
(let [[head-sig* [meta-id]] (encore/ba-split head-ba 3)]
|
||||
(when (encore/ba= head-sig* head-sig) ; Appears to be well-formed
|
||||
[data-ba (head-meta meta-id {:unrecognized-meta? true})]))))
|
||||
|
||||
(defn thaw
|
||||
|
|
@ -699,16 +700,16 @@
|
|||
|
||||
;;;; Tools
|
||||
|
||||
(utils/defalias freezeable? utils/freezable?)
|
||||
(encore/defalias freezeable? utils/freezable?)
|
||||
|
||||
(defn inspect-ba "Alpha - subject to change."
|
||||
[ba & [thaw-opts]]
|
||||
(if-not (utils/bytes? ba) :not-ba
|
||||
(let [[first2bytes nextbytes] (utils/ba-split ba 2)
|
||||
(if-not (encore/bytes? ba) :not-ba
|
||||
(let [[first2bytes nextbytes] (encore/ba-split ba 2)
|
||||
known-wrapper
|
||||
(cond
|
||||
(utils/ba= first2bytes (.getBytes "\u0000<" "UTF8")) :carmine/bin
|
||||
(utils/ba= first2bytes (.getBytes "\u0000>" "UTF8")) :carmine/clj)
|
||||
(encore/ba= first2bytes (.getBytes "\u0000<" "UTF8")) :carmine/bin
|
||||
(encore/ba= first2bytes (.getBytes "\u0000>" "UTF8")) :carmine/clj)
|
||||
|
||||
unwrapped-ba (if known-wrapper nextbytes ba)
|
||||
[data-ba nippy-header] (or (try-parse-header unwrapped-ba)
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
{:author "Peter Taoussanis"}
|
||||
(:require [clojure.tools.reader.edn :as edn]
|
||||
[clojure.data.fressian :as fressian]
|
||||
[taoensso.encore :as encore]
|
||||
[taoensso.nippy :as nippy :refer (freeze thaw)]
|
||||
[taoensso.nippy.compression :as compression]
|
||||
[taoensso.nippy.utils :as utils]))
|
||||
[taoensso.nippy.compression :as compression]))
|
||||
|
||||
(def data nippy/stress-data-benchable)
|
||||
|
||||
|
|
@ -21,7 +21,7 @@
|
|||
|
||||
(comment (fressian-thaw (fressian-freeze data)))
|
||||
|
||||
(defmacro bench* [& body] `(utils/bench 10000 {:warmup-laps 20000} ~@body))
|
||||
(defmacro bench* [& body] `(encore/bench 10000 {:warmup-laps 20000} ~@body))
|
||||
(defn bench1 [freezer thawer & [sizer]]
|
||||
(let [data-frozen (freezer data)
|
||||
time-freeze (bench* (freezer data))
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
(ns taoensso.nippy.compression
|
||||
"Alpha - subject to change."
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [taoensso.nippy.utils :as utils])
|
||||
(:import [java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream
|
||||
DataOutputStream]))
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
Simple no-nonsense crypto with reasonable defaults. Because your Clojure data
|
||||
deserves some privacy."
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [taoensso.nippy.utils :as utils]))
|
||||
(:require [taoensso.encore :as encore]))
|
||||
|
||||
;;;; Interface
|
||||
|
||||
|
|
@ -32,7 +32,7 @@
|
|||
(PBKDF2, bcrypt, scrypt, etc.). Decent security with multiple rounds."
|
||||
[salt-ba ^String pwd]
|
||||
(loop [^bytes ba (let [pwd-ba (.getBytes pwd "UTF-8")]
|
||||
(if salt-ba (utils/ba-concat salt-ba pwd-ba) pwd-ba))
|
||||
(if salt-ba (encore/ba-concat salt-ba pwd-ba) pwd-ba))
|
||||
n (* (int Short/MAX_VALUE) (if salt-ba 5 64))]
|
||||
(if-not (zero? n)
|
||||
(recur (.digest sha512-md ba) (dec n))
|
||||
|
|
@ -70,22 +70,22 @@
|
|||
salt? (= type :salted)
|
||||
iv-ba (rand-bytes aes128-block-size)
|
||||
salt-ba (when salt? (rand-bytes salt-size))
|
||||
prefix-ba (if-not salt? iv-ba (utils/ba-concat iv-ba salt-ba))
|
||||
key (utils/memoized (when-not salt? (:key-cache this))
|
||||
prefix-ba (if-not salt? iv-ba (encore/ba-concat iv-ba salt-ba))
|
||||
key (encore/memoized (when-not salt? (:key-cache this))
|
||||
sha512-key salt-ba pwd)
|
||||
iv (javax.crypto.spec.IvParameterSpec. iv-ba)]
|
||||
(.init aes128-cipher javax.crypto.Cipher/ENCRYPT_MODE
|
||||
^javax.crypto.spec.SecretKeySpec key iv)
|
||||
(utils/ba-concat prefix-ba (.doFinal aes128-cipher data-ba))))
|
||||
(encore/ba-concat prefix-ba (.doFinal aes128-cipher data-ba))))
|
||||
|
||||
(decrypt [this typed-pwd ba]
|
||||
(let [[type pwd] (destructure-typed-pwd typed-pwd)
|
||||
salt? (= type :salted)
|
||||
prefix-size (+ aes128-block-size (if salt? salt-size 0))
|
||||
[prefix-ba data-ba] (utils/ba-split ba prefix-size)
|
||||
[prefix-ba data-ba] (encore/ba-split ba prefix-size)
|
||||
[iv-ba salt-ba] (if-not salt? [prefix-ba nil]
|
||||
(utils/ba-split prefix-ba aes128-block-size))
|
||||
key (utils/memoized (when-not salt? (:key-cache this))
|
||||
(encore/ba-split prefix-ba aes128-block-size))
|
||||
key (encore/memoized (when-not salt? (:key-cache this))
|
||||
sha512-key salt-ba pwd)
|
||||
iv (javax.crypto.spec.IvParameterSpec. iv-ba)]
|
||||
(.init aes128-cipher javax.crypto.Cipher/DECRYPT_MODE
|
||||
|
|
|
|||
|
|
@ -3,8 +3,7 @@
|
|||
Utilities for third-party tools that want to add fully-user-configurable Nippy
|
||||
support. Used by Carmine and Faraday."
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [taoensso.nippy :as nippy]
|
||||
[taoensso.nippy.utils :as utils]))
|
||||
(:require [taoensso.nippy :as nippy]))
|
||||
|
||||
(defrecord WrappedForFreezing [value opts])
|
||||
(defn wrapped-for-freezing? [x] (instance? WrappedForFreezing x))
|
||||
|
|
|
|||
|
|
@ -1,114 +1,10 @@
|
|||
(ns taoensso.nippy.utils
|
||||
{:author "Peter Taoussanis"}
|
||||
(:require [clojure.string :as str]
|
||||
(:require [clojure.string :as str]
|
||||
[clojure.tools.reader.edn :as edn])
|
||||
(:import [java.io ByteArrayInputStream ByteArrayOutputStream Serializable
|
||||
ObjectOutputStream ObjectInputStream]))
|
||||
|
||||
(defmacro defalias
|
||||
"Defines an alias for a var, preserving metadata. Adapted from
|
||||
clojure.contrib/def.clj, Ref. http://goo.gl/xpjeH"
|
||||
[name target & [doc]]
|
||||
`(let [^clojure.lang.Var v# (var ~target)]
|
||||
(alter-meta! (def ~name (.getRawRoot v#))
|
||||
#(merge % (apply dissoc (meta v#) [:column :line :file :test :name])
|
||||
(when-let [doc# ~doc] {:doc doc#})))
|
||||
(var ~name)))
|
||||
|
||||
(defmacro case-eval
|
||||
"Like `case` but evaluates test constants for their compile-time value."
|
||||
[e & clauses]
|
||||
(let [;; Don't evaluate default expression!
|
||||
default (when (odd? (count clauses)) (last clauses))
|
||||
clauses (if default (butlast clauses) clauses)]
|
||||
`(case ~e
|
||||
~@(map-indexed (fn [i# form#] (if (even? i#) (eval form#) form#))
|
||||
clauses)
|
||||
~(when default default))))
|
||||
|
||||
(defmacro repeatedly-into
|
||||
"Like `repeatedly` but faster and `conj`s items into given collection."
|
||||
[coll n & body]
|
||||
`(let [coll# ~coll
|
||||
n# ~n]
|
||||
(if (instance? clojure.lang.IEditableCollection coll#)
|
||||
(loop [v# (transient coll#) idx# 0]
|
||||
(if (>= idx# n#)
|
||||
(persistent! v#)
|
||||
(recur (conj! v# ~@body)
|
||||
(inc idx#))))
|
||||
(loop [v# coll#
|
||||
idx# 0]
|
||||
(if (>= idx# n#)
|
||||
v#
|
||||
(recur (conj v# ~@body)
|
||||
(inc idx#)))))))
|
||||
|
||||
(defmacro time-ns "Returns number of nanoseconds it takes to execute body."
|
||||
[& body] `(let [t0# (System/nanoTime)] ~@body (- (System/nanoTime) t0#)))
|
||||
|
||||
(defmacro bench
|
||||
"Repeatedly executes body and returns time taken to complete execution."
|
||||
[nlaps {:keys [nlaps-warmup nthreads as-ns?]
|
||||
:or {nlaps-warmup 0
|
||||
nthreads 1}} & body]
|
||||
`(let [nlaps# ~nlaps
|
||||
nlaps-warmup# ~nlaps-warmup
|
||||
nthreads# ~nthreads]
|
||||
(try (dotimes [_# nlaps-warmup#] ~@body)
|
||||
(let [nanosecs#
|
||||
(if (= nthreads# 1)
|
||||
(time-ns (dotimes [_# nlaps#] ~@body))
|
||||
(let [nlaps-per-thread# (int (/ nlaps# nthreads#))]
|
||||
(time-ns
|
||||
(->> (fn [] (future (dotimes [_# nlaps-per-thread#] ~@body)))
|
||||
(repeatedly nthreads#)
|
||||
(doall)
|
||||
(map deref)
|
||||
(dorun)))))]
|
||||
(if ~as-ns? nanosecs# (Math/round (/ nanosecs# 1000000.0))))
|
||||
(catch Exception e# (format "DNF: %s" (.getMessage e#))))))
|
||||
|
||||
(defn memoized
|
||||
"Like `(partial memoize* {})` but takes an explicit cache atom (possibly nil)
|
||||
and immediately applies memoized f to given arguments."
|
||||
[cache f & args]
|
||||
(if-not cache
|
||||
(apply f args)
|
||||
(if-let [dv (@cache args)]
|
||||
@dv
|
||||
(locking cache ; For thread racing
|
||||
(if-let [dv (@cache args)] ; Retry after lock acquisition!
|
||||
@dv
|
||||
(let [dv (delay (apply f args))]
|
||||
(swap! cache assoc args dv)
|
||||
@dv))))))
|
||||
|
||||
(comment (memoized nil +)
|
||||
(memoized nil + 5 12))
|
||||
|
||||
(def ^:const bytes-class (Class/forName "[B"))
|
||||
(defn bytes? [x] (instance? bytes-class x))
|
||||
(defn ba= [^bytes x ^bytes y] (java.util.Arrays/equals x y))
|
||||
|
||||
(defn ba-concat ^bytes [^bytes ba1 ^bytes ba2]
|
||||
(let [s1 (alength ba1)
|
||||
s2 (alength ba2)
|
||||
out (byte-array (+ s1 s2))]
|
||||
(System/arraycopy ba1 0 out 0 s1)
|
||||
(System/arraycopy ba2 0 out s1 s2)
|
||||
out))
|
||||
|
||||
(defn ba-split [^bytes ba ^Integer idx]
|
||||
(let [s (alength ba)]
|
||||
(when (> s idx)
|
||||
[(java.util.Arrays/copyOf ba idx)
|
||||
(java.util.Arrays/copyOfRange ba idx s)])))
|
||||
|
||||
(comment (String. (ba-concat (.getBytes "foo") (.getBytes "bar")))
|
||||
(let [[x y] (ba-split (.getBytes "foobar") 5)]
|
||||
[(String. x) (String. y)]))
|
||||
|
||||
;;;; Fallback type tests
|
||||
;; Unfortunately the only reliable way we can tell if something's
|
||||
;; really serializable/readable is to actually try a full roundtrip.
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
[taoensso.nippy.compression :as compression]
|
||||
[taoensso.nippy.benchmarks :as benchmarks]))
|
||||
|
||||
(comment (test/run-tests '[taoensso.nippy.tests.main]))
|
||||
|
||||
(def test-data nippy/stress-data-comparable)
|
||||
(defn- before-run {:expectations-options :before-run} [])
|
||||
(defn- after-run {:expectations-options :after-run} [])
|
||||
|
|
|
|||
Loading…
Reference in a new issue