2012-07-06 19:12:59 +00:00
( ns taoensso.nippy
2016-04-07 05:49:26 +00:00
"High-performance serialization library for Clojure"
2015-09-17 03:55:09 +00:00
{ :author "Peter Taoussanis (@ptaoussanis)" }
2016-10-28 03:25:46 +00:00
( :require
2020-08-25 20:07:33 +00:00
[ clojure.string :as str ]
2017-02-13 16:35:18 +00:00
[ clojure.java.io :as jio ]
2020-07-24 20:50:05 +00:00
[ taoensso.encore :as enc ]
2016-10-28 03:25:46 +00:00
[ taoensso.nippy
[ utils :as utils ]
[ compression :as compression ]
[ encryption :as encryption ] ] )
( :import
2023-01-25 14:24:50 +00:00
[ java.nio.charset StandardCharsets ]
2016-10-28 03:25:46 +00:00
[ java.io ByteArrayInputStream ByteArrayOutputStream DataInputStream
DataOutputStream Serializable ObjectOutputStream ObjectInputStream
DataOutput DataInput ]
2020-07-24 17:38:16 +00:00
[ java.lang.reflect Method Field Constructor ]
[ java.net URI ]
2023-07-27 12:56:34 +00:00
[ java.util # _Date UUID ]
2016-10-28 03:25:46 +00:00
[ java.util.regex Pattern ]
[ clojure.lang Keyword Symbol BigInt Ratio
APersistentMap APersistentVector APersistentSet
IPersistentMap ; IPersistentVector IPersistentSet IPersistentList
PersistentQueue PersistentTreeMap PersistentTreeSet PersistentList
2020-07-24 17:38:16 +00:00
LazySeq IRecord ISeq IType ] ) )
2012-07-06 19:12:59 +00:00
2023-09-25 09:24:58 +00:00
( enc/assert-min-encore-version [ 3 67 0 ] )
2016-04-07 05:49:26 +00:00
( comment
( set! *unchecked-math* :warn-on-boxed )
( set! *unchecked-math* false )
( thaw ( freeze stress-data ) ) )
2015-02-18 10:22:37 +00:00
2016-10-28 03:35:21 +00:00
;;;; TODO
;; - Performance would benefit from ^:static support / direct linking / etc.
2016-07-16 12:09:38 +00:00
2014-07-04 13:05:18 +00:00
;;;; Nippy data format
2015-09-28 09:25:43 +00:00
;; * 4-byte header (Nippy v2.x+) (may be disabled but incl. by default) [1]
2016-04-07 05:49:26 +00:00
;; { * 1-byte type id
;; * Arb-length payload determined by freezer for this type [2] } ...
2014-07-04 13:05:18 +00:00
;;
2015-09-29 04:33:49 +00:00
;; [1] Inclusion of header is *strongly* recommended. Purpose:
2015-09-28 09:25:43 +00:00
;; * Sanity check (confirm that data appears to be Nippy data)
;; * Nippy version check (=> supports changes to data schema over time)
;; * Supports :auto thaw compressor, encryptor
2015-04-19 03:48:01 +00:00
;; * Supports :auto freeze compressor (since this depends on :auto thaw
2015-09-28 09:25:43 +00:00
;; compressor)
2014-01-21 07:21:56 +00:00
;;
2016-07-16 12:09:38 +00:00
;; [2] See `IFreezable1` protocol for type-specific payload formats,
2016-04-07 05:49:26 +00:00
;; `thaw-from-in!` for reference type-specific thaw implementations
;;
2023-01-25 14:24:50 +00:00
( def ^ :private head-sig "First 3 bytes of Nippy header" ( .getBytes "NPY" StandardCharsets/UTF_8 ) )
2016-04-07 05:49:26 +00:00
( def ^ :private ^ :const head-version "Current Nippy header format version" 1 )
( def ^ :private ^ :const head-meta
"Final byte of 4-byte Nippy header stores version-dependent metadata"
2020-07-24 17:38:16 +00:00
;; Currently
;; - 5 compressors, #{nil :snappy :lz4 :lzma2 :else}
;; - 4 encryptors, #{nil :aes128-cbc-sha512 :aes128-gcm-sha512 :else}
2014-04-05 11:30:28 +00:00
{ ( byte 0 ) { :version 1 :compressor-id nil :encryptor-id nil }
2020-07-24 17:38:16 +00:00
( byte 2 ) { :version 1 :compressor-id nil :encryptor-id :aes128-cbc-sha512 }
( byte 14 ) { :version 1 :compressor-id nil :encryptor-id :aes128-gcm-sha512 }
2014-04-05 11:30:28 +00:00
( byte 4 ) { :version 1 :compressor-id nil :encryptor-id :else }
2020-07-24 17:38:16 +00:00
2014-04-05 11:30:28 +00:00
( byte 1 ) { :version 1 :compressor-id :snappy :encryptor-id nil }
2020-07-24 17:38:16 +00:00
( byte 3 ) { :version 1 :compressor-id :snappy :encryptor-id :aes128-cbc-sha512 }
( byte 15 ) { :version 1 :compressor-id :snappy :encryptor-id :aes128-gcm-sha512 }
2014-04-05 11:30:28 +00:00
( byte 7 ) { :version 1 :compressor-id :snappy :encryptor-id :else }
2020-07-24 17:38:16 +00:00
2015-04-19 03:48:01 +00:00
;;; :lz4 used for both lz4 and lz4hc compressor (the two are compatible)
2014-04-05 11:30:28 +00:00
( byte 8 ) { :version 1 :compressor-id :lz4 :encryptor-id nil }
2020-07-24 17:38:16 +00:00
( byte 9 ) { :version 1 :compressor-id :lz4 :encryptor-id :aes128-cbc-sha512 }
( byte 16 ) { :version 1 :compressor-id :lz4 :encryptor-id :aes128-gcm-sha512 }
2014-04-05 11:30:28 +00:00
( byte 10 ) { :version 1 :compressor-id :lz4 :encryptor-id :else }
2020-07-24 17:38:16 +00:00
2014-04-05 11:30:28 +00:00
( byte 11 ) { :version 1 :compressor-id :lzma2 :encryptor-id nil }
2020-07-24 17:38:16 +00:00
( byte 12 ) { :version 1 :compressor-id :lzma2 :encryptor-id :aes128-cbc-sha512 }
( byte 17 ) { :version 1 :compressor-id :lzma2 :encryptor-id :aes128-gcm-sha512 }
( byte 13 ) { :version 1 :compressor-id :lzma2 :encryptor-id :else }
( byte 5 ) { :version 1 :compressor-id :else :encryptor-id nil }
( byte 18 ) { :version 1 :compressor-id :else :encryptor-id :aes128-cbc-sha512 }
( byte 19 ) { :version 1 :compressor-id :else :encryptor-id :aes128-gcm-sha512 }
( byte 6 ) { :version 1 :compressor-id :else :encryptor-id :else } } )
( comment ( count ( sort ( keys head-meta ) ) ) )
2013-06-12 18:14:46 +00:00
2016-04-07 05:49:26 +00:00
( defmacro ^ :private when-debug [ & body ] ( when # _true false ` ( do ~@ body ) ) )
2022-10-13 16:52:19 +00:00
( def ^ :private types-spec
" Private representation of Nippy 's internal type schema ,
{ <type-id> [ <type-kw> ?<payload-info> ] } .
See ` public-types-spec ` for more info. "
{ 3 [ :nil [ ] ]
8 [ :true [ ] ]
9 [ :false [ ] ]
10 [ :char [ [ :bytes 2 ] ] ]
40 [ :byte [ [ :bytes 1 ] ] ]
41 [ :short [ [ :bytes 2 ] ] ]
42 [ :integer [ [ :bytes 4 ] ] ]
[new] [Storage efficiency] PoC: separate signed long types
Before:
Longs in [ -128, 127] use 1 byte
Longs in [-32768, 32767] use 2 bytes
etc.
After:
Longs in [ -255, 255] use 1 byte
Longs in [-65535, 65535] use 2 bytes
etc.
I.e. doubles the range of longs that can be stored by 1, 2, and 4 bytes.
This changes saves:
- 1 byte per long in [ 128, 255], or [ -129, -255]
- 2 bytes per long in [32768, 65535], or [-32769, -65535]
- 4 bytes per long ...
Is this advantage worth the extra complexity? Probably yes, given how
common longs (and colls of longs) are in Clojure.
2023-07-31 20:37:08 +00:00
0 [ :long-0 [ ] ]
87 [ :long-pos-sm [ [ :bytes 1 ] ] ]
88 [ :long-pos-md [ [ :bytes 2 ] ] ]
89 [ :long-pos-lg [ [ :bytes 4 ] ] ]
93 [ :long-neg-sm [ [ :bytes 1 ] ] ]
94 [ :long-neg-md [ [ :bytes 2 ] ] ]
95 [ :long-neg-lg [ [ :bytes 4 ] ] ]
43 [ :long-xl [ [ :bytes 8 ] ] ]
2022-10-13 16:52:19 +00:00
55 [ :double-0 [ ] ]
60 [ :float [ [ :bytes 4 ] ] ]
61 [ :double [ [ :bytes 8 ] ] ]
2023-07-27 12:56:34 +00:00
91 [ :uuid [ [ :bytes 16 ] ] ]
90 [ :util-date [ [ :bytes 8 ] ] ]
92 [ :sql-date [ [ :bytes 8 ] ] ]
2022-10-13 16:52:19 +00:00
;; JVM >=8
79 [ :time-instant [ [ :bytes 12 ] ] ]
83 [ :time-duration [ [ :bytes 12 ] ] ]
84 [ :time-period [ [ :bytes 12 ] ] ]
34 [ :str-0 [ ] ]
2023-07-31 21:13:19 +00:00
96 [ :str-sm* [ [ :bytes { :read 1 :unsigned? true } ] ] ]
2022-10-13 16:52:19 +00:00
16 [ :str-md [ [ :bytes { :read 2 } ] ] ]
13 [ :str-lg [ [ :bytes { :read 4 } ] ] ]
106 [ :kw-sm [ [ :bytes { :read 1 } ] ] ]
85 [ :kw-md [ [ :bytes { :read 2 } ] ] ]
56 [ :sym-sm [ [ :bytes { :read 1 } ] ] ]
86 [ :sym-md [ [ :bytes { :read 2 } ] ] ]
47 [ :reader-sm [ [ :bytes { :read 1 } ] ] ]
51 [ :reader-md [ [ :bytes { :read 2 } ] ] ]
52 [ :reader-lg [ [ :bytes { :read 4 } ] ] ]
53 [ :bytes-0 [ ] ]
7 [ :bytes-sm [ [ :bytes { :read 1 } ] ] ]
15 [ :bytes-md [ [ :bytes { :read 2 } ] ] ]
2 [ :bytes-lg [ [ :bytes { :read 4 } ] ] ]
17 [ :vec-0 [ ] ]
113 [ :vec-2 [ [ :elements 2 ] ] ]
114 [ :vec-3 [ [ :elements 3 ] ] ]
2023-07-31 21:13:19 +00:00
97 [ :vec-sm* [ [ :elements { :read 1 :unsigned? true } ] ] ]
2022-10-13 16:52:19 +00:00
69 [ :vec-md [ [ :elements { :read 2 } ] ] ]
21 [ :vec-lg [ [ :elements { :read 4 } ] ] ]
18 [ :set-0 [ ] ]
2023-07-31 21:13:19 +00:00
98 [ :set-sm* [ [ :elements { :read 1 :unsigned? true } ] ] ]
2022-10-13 16:52:19 +00:00
32 [ :set-md [ [ :elements { :read 2 } ] ] ]
23 [ :set-lg [ [ :elements { :read 4 } ] ] ]
19 [ :map-0 [ ] ]
2023-07-31 21:13:19 +00:00
99 [ :map-sm* [ [ :elements { :read 1 :multiplier 2 :unsigned? true } ] ] ]
2022-10-13 16:52:19 +00:00
33 [ :map-md [ [ :elements { :read 2 :multiplier 2 } ] ] ]
30 [ :map-lg [ [ :elements { :read 4 :multiplier 2 } ] ] ]
35 [ :list-0 [ ] ]
36 [ :list-sm [ [ :elements { :read 1 } ] ] ]
54 [ :list-md [ [ :elements { :read 2 } ] ] ]
20 [ :list-lg [ [ :elements { :read 4 } ] ] ]
37 [ :seq-0 [ ] ]
38 [ :seq-sm [ [ :elements { :read 1 } ] ] ]
39 [ :seq-md [ [ :elements { :read 2 } ] ] ]
24 [ :seq-lg [ [ :elements { :read 4 } ] ] ]
28 [ :sorted-set-lg [ [ :elements { :read 4 } ] ] ]
31 [ :sorted-map-lg [ [ :elements { :read 4 :multiplier 2 } ] ] ]
26 [ :queue-lg [ [ :elements { :read 4 } ] ] ]
115 [ :objects-lg [ [ :elements { :read 4 } ] ] ]
25 [ :meta [ [ :elements 1 ] ] ]
58 [ :regex [ [ :elements 1 ] ] ]
71 [ :uri [ [ :elements 1 ] ] ]
;; BigInteger based
44 [ :bigint [ [ :bytes { :read 4 } ] ] ]
45 [ :biginteger [ [ :bytes { :read 4 } ] ] ]
62 [ :bigdec [ [ :bytes 4 ]
[ :bytes { :read 4 } ] ] ]
70 [ :ratio [ [ :bytes { :read 4 } ]
[ :bytes { :read 4 } ] ] ]
;; Serializable
75 [ :sz-quarantined-sm [ [ :bytes { :read 1 } ] [ :elements 1 ] ] ]
76 [ :sz-quarantined-md [ [ :bytes { :read 2 } ] [ :elements 1 ] ] ]
48 [ :record-sm [ [ :bytes { :read 1 } ] [ :elements 1 ] ] ]
49 [ :record-md [ [ :bytes { :read 2 } ] [ :elements 1 ] ] ]
;; Necessarily without size information
81 [ :type nil ]
82 [ :prefixed-custom-md nil ]
2023-07-31 14:43:09 +00:00
59 [ :cached-0 nil ]
63 [ :cached-1 nil ]
64 [ :cached-2 nil ]
65 [ :cached-3 nil ]
66 [ :cached-4 nil ]
72 [ :cached-5 nil ]
73 [ :cached-6 nil ]
74 [ :cached-7 nil ]
67 [ :cached-sm nil ]
68 [ :cached-md nil ]
2020-10-24 10:27:53 +00:00
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
;;; DEPRECATED (only support thawing)
2022-10-13 16:52:19 +00:00
;; Desc-sorted by deprecation date
2023-07-31 21:13:19 +00:00
105 [ :str-sm_ [ [ :bytes { :read 1 } ] ] ] ; [2023-mm-dd v3.3.3] Switch to unsigned sm*
110 [ :vec-sm_ [ [ :elements { :read 1 } ] ] ] ; [2023-mm-dd v3.3.3] Switch to unsigned sm*
111 [ :set-sm_ [ [ :elements { :read 1 } ] ] ] ; [2023-mm-dd v3.3.3] Switch to unsigned sm*
112 [ :map-sm_ [ [ :elements { :read 1 :multiplier 2 } ] ] ] ; [2023-mm-dd v3.3.3] Switch to unsigned sm*
[new] [Storage efficiency] PoC: separate signed long types
Before:
Longs in [ -128, 127] use 1 byte
Longs in [-32768, 32767] use 2 bytes
etc.
After:
Longs in [ -255, 255] use 1 byte
Longs in [-65535, 65535] use 2 bytes
etc.
I.e. doubles the range of longs that can be stored by 1, 2, and 4 bytes.
This changes saves:
- 1 byte per long in [ 128, 255], or [ -129, -255]
- 2 bytes per long in [32768, 65535], or [-32769, -65535]
- 4 bytes per long ...
Is this advantage worth the extra complexity? Probably yes, given how
common longs (and colls of longs) are in Clojure.
2023-07-31 20:37:08 +00:00
100 [ :long-sm_ [ [ :bytes 1 ] ] ] ; [2023-mm-dd v3.3.3] Switch to 2x pos/neg ids
101 [ :long-md_ [ [ :bytes 2 ] ] ] ; [2023-mm-dd v3.3.3] Switch to 2x pos/neg ids
102 [ :long-lg_ [ [ :bytes 4 ] ] ] ; [2023-mm-dd v3.3.3] Switch to 2x pos/neg ids
2022-10-13 16:52:19 +00:00
78 [ :sym-md_ [ [ :bytes { :read 4 } ] ] ] ; [2020-11-18 v3.1.1] Buggy size field, Ref. #138
77 [ :kw-md_ [ [ :bytes { :read 4 } ] ] ] ; [2020-11-18 v3.1.1] Buggy size field, Ref. #138
6 [ :sz-unquarantined-lg_ nil ] ; [2020-07-24 v2.15.0] Unskippable, Ref. #130
50 [ :sz-unquarantined-md_ nil ] ; [2020-07-24 v2.15.0] Unskippable, Ref. #130
46 [ :sz-unquarantined-sm_ nil ] ; [2020-07-24 v2.15.0] Unskippable, Ref. #130
2022-07-19 07:19:37 +00:00
14 [ :kw-lg_ [ [ :bytes { :read 4 } ] ] ] ; [2020-09-20 v3.0.0] Unrealistic
57 [ :sym-lg_ [ [ :bytes { :read 4 } ] ] ] ; [2020-09-20 v3.0.0] Unrealistic
80 [ :record-lg_ [ [ :bytes { :read 4 } ] [ :elements 1 ] ] ] ; [2020-09-20 v3.0.0] Unrealistic
2022-10-13 16:52:19 +00:00
5 [ :reader-lg_ [ [ :bytes { :read 4 } ] ] ] ; [2016-07-24 v2.12.0] Identical to :reader-lg, historical accident
4 [ :boolean_ [ [ :bytes 1 ] ] ] ; [2016-07-24 v2.12.0] For switch to true/false ids
29 [ :sorted-map_ [ [ :elements { :read 4 } ] ] ] ; [2016-02-25 v2.11.0] For count/2
27 [ :map__ [ [ :elements { :read 4 } ] ] ] ; [2016-02-25 v2.11.0] For count/2
12 [ :kw_ [ [ :bytes { :read 2 } ] ] ] ; [2013-07-22 v2.0.0] For consistecy with str impln
1 [ :reader_ [ [ :bytes { :read 2 } ] ] ] ; [2012-07-20 v0.9.2] For >64k length support
11 [ :str_ [ [ :bytes { :read 2 } ] ] ] ; [2012-07-20 v0.9.2] For >64k length support
22 [ :map_ [ [ :elements { :read 4 :multiplier 2 } ] ] ] ; [2012-07-07 v0.9.0] For more efficient thaw impln
2016-04-07 05:49:26 +00:00
} )
2013-10-31 06:15:22 +00:00
2016-04-07 05:49:26 +00:00
( comment
2022-10-13 16:52:19 +00:00
( count ; Eval to check for unused type-ids
( enc/reduce-n ( fn [ acc in ] ( if-not ( types-spec in ) ( conj acc in ) acc ) )
[ ] Byte/MAX_VALUE ) ) )
2016-04-07 05:49:26 +00:00
( defmacro ^ :private defids [ ]
` ( do
~@ ( map
2022-10-13 16:52:19 +00:00
( fn [ [ id# [ kw# ] ] ]
( let [ kw# ( str "id-" ( name kw# ) )
sym# ( with-meta ( symbol kw# ) { :const true :private true } ) ]
2016-04-07 05:49:26 +00:00
` ( def ~ sym# ( byte ~ id# ) ) ) )
2022-10-13 16:52:19 +00:00
types-spec ) ) )
2016-04-07 05:49:26 +00:00
( comment ( macroexpand ' ( defids ) ) )
( defids )
2022-10-13 16:52:19 +00:00
( def public-types-spec
" Public representation of Nippy 's internal type schema.
For use by tooling and advanced users.
**HIGHLY EXPERIMENTAL!**
Subject to breaking change without notice.
Currently completely untested , may contain bugs.
Intended for use only by early adopters to give design feedback.
Format :
{ <type-id> { :keys [ type-id type-kw payload-spec deprecated? ] } } ,
- ` type-id ` : A +ive single-byte identifier like ` 110 ` .
-ive type ids are reserved for custom user-defined types.
2023-07-31 21:13:19 +00:00
- ` type-kw ` : A keyword like ` :kw-sm ` ,
2022-10-13 16:52:19 +00:00
suffixes used to differentiate subtypes of different sizes :
-0 ; Empty => 0 byte payload / element-count
-sm ; Small => 1 byte (byte) payload / element-count
-md ; Medium => 2 byte (short) payload / element-count
-lg ; Large => 4 byte (int) payload / element-count
-xl ; Extra large => 8 byte (long) payload / element-count
- ` payload-spec ` examples :
- nil ; No spec available (e.g. unpredictable payload)
- [ ] ; Type has no payload
- [ [ :bytes 4 ] ] ; Type has a payload of exactly 4 bytes
- [ [ :bytes 2 ] [ :elements 2 ] ] ; Type has a payload of exactly 2 bytes, then
; 2 elements
- [ [ :bytes { :read 2 } ]
2023-07-31 21:13:19 +00:00
[ :elements { :read 4 :multiplier 2 :unsigned? true } ] ]
2022-10-13 16:52:19 +00:00
; Type has payload of <short-count> bytes, then
2023-07-31 21:13:19 +00:00
; <unsigned-int-count>*2 (multiplier) elements
2022-10-13 16:52:19 +00:00
Note that ` payload-spec ` can be handy for skipping over items in
data stream without fully reading every item. "
;; TODO Add unit tests for size data once API is finalized
( reduce-kv
( fn [ m type-id [ type-kw ?payload-spec ] ]
( assoc m type-id
( enc/assoc-when
{ :type-id type-id
:type-kw type-kw }
:payload-spec ?payload-spec
:deprecated? ( enc/str-ends-with? ( name type-kw ) "_" ) ) ) )
types-spec
types-spec ) )
2023-07-31 21:13:19 +00:00
( comment ( get public-types-spec 96 ) )
2016-04-07 05:49:26 +00:00
;;;; Ns imports (for convenience of lib consumers)
2014-04-05 11:30:28 +00:00
2022-07-19 07:19:37 +00:00
( enc/defaliases
compression/compress
compression/decompress
compression/snappy-compressor
compression/lzma2-compressor
compression/lz4-compressor
compression/lz4hc-compressor
2014-04-05 11:30:28 +00:00
2022-07-19 07:19:37 +00:00
encryption/encrypt
encryption/decrypt
2020-07-24 17:38:16 +00:00
2022-07-19 07:19:37 +00:00
encryption/aes128-gcm-encryptor
encryption/aes128-cbc-encryptor
encryption/aes128-gcm-encryptor
{ :src encryption/aes128-gcm-encryptor , :alias aes128-encryptor }
2014-04-05 11:30:28 +00:00
2022-07-19 07:19:37 +00:00
utils/freezable? )
2014-04-05 11:30:28 +00:00
2016-04-07 05:49:26 +00:00
;;;; Dynamic config
;; See also `nippy.tools` ns for further dynamic config support
2020-07-24 17:38:16 +00:00
;; For back compatibility (nb Timbre's Carmine appender)
2023-08-16 09:56:19 +00:00
( enc/defonce ^ :dynamic ^ :no-doc ^ :deprecated *final-freeze-fallback* "DEPRECATED: prefer `*freeze-fallback`." nil )
( enc/defonce ^ :dynamic *freeze-fallback* "(fn [data-output x])->freeze, nil => default" nil )
2016-07-16 12:09:38 +00:00
2020-07-24 17:38:16 +00:00
( enc/defonce ^ :dynamic *custom-readers* "{<hash-or-byte-id> (fn [data-input])->read}" nil )
2016-07-24 08:41:31 +00:00
( enc/defonce ^ :dynamic *auto-freeze-compressor*
2016-04-07 05:49:26 +00:00
" ( fn [ byte-array ] ) ->compressor used by ` ( freeze <x> { :compressor :auto } ) ,
nil => default "
nil )
2020-07-25 08:15:27 +00:00
( enc/defonce ^ :dynamic *incl-metadata?* "Include metadata when freezing/thawing?" true )
2023-08-01 12:46:22 +00:00
( enc/defonce ^ :dynamic *thaw-xform*
" Experimental , subject to change. Feedback welcome.
Transducer to use when thawing standard Clojure collection types
( vectors , maps , sets , lists , etc. ) .
Allows fast+flexible inspection and manipulation of data being thawed.
Key-val style data structures like maps will provide ` MapEntry ` args
to reducing function. Use ` map-entry? ` , ` key ` , ` val ` utils for these.
Example transducers :
( map ( fn [ x ] ( println x ) x ) ) ; Print each coll item thawed
( comp
( map ( fn [ x ] ( if ( = x :secret ) :redacted x ) ) ) ; Replace secrets
( remove ( fn [ x ] ; Remove maps with a truthy :remove?
( or
( and ( map? x ) ( :remove? x ) )
( and ( map-entry? x ) ( = ( key x ) :remove? ) ( val y ) ) ) ) ) ) )
Note that while this is a very powerful feature , correctly writing
and debugging transducers and reducing fns can be tricky.
2023-08-16 09:56:19 +00:00
To help , if Nippy encounters an error while applying your xform , it
2023-08-01 12:46:22 +00:00
will throw a detailed ` ExceptionInfo ` with message
\" Error thrown via ` *thaw-xform* ` \" to help you debug. "
2023-08-02 12:44:07 +00:00
{ :added "v3.3.0-RC1 (2023-08-02)" }
2023-08-01 12:46:22 +00:00
nil )
( comment
( binding [ *thaw-xform*
( comp
( map ( fn [ x ] ( println x ) x ) )
( map ( fn [ x ] ( if ( = x 1 ) 0 x ) ) )
( map ( fn [ x ] ( / 1 0 ) ) ) ) ]
( thaw ( freeze [ 1 1 0 1 1 ] ) ) ) )
2020-09-10 16:42:25 +00:00
;;;; Java Serializable config
;; Unfortunately quite a bit of complexity to do this safely
( def default-freeze-serializable-allowlist
" Allows *any* class-name to be frozen using Java 's Serializable interface.
This is generally safe since RCE risk is present only when thawing.
See also ` *freeze-serializable-allowlist* ` . "
# { "*" } )
( def default-thaw-serializable-allowlist
" A set of common safe class-names to allow to be frozen using Java 's
Serializable interface. PRs welcome for additions.
See also ` *thaw-serializable-allowlist* ` . "
2020-07-25 09:07:12 +00:00
# { "[I" "[F" "[Z" "[B" "[C" "[D" "[S" "[J"
"java.lang.Throwable"
"java.lang.Exception"
"java.lang.RuntimeException"
"java.lang.ArithmeticException"
"java.lang.IllegalArgumentException"
"java.lang.NullPointerException"
"java.lang.IndexOutOfBoundsException"
2023-07-27 12:49:50 +00:00
"java.lang.ClassCastException"
2020-07-25 09:07:12 +00:00
"java.net.URI"
2023-07-27 12:56:34 +00:00
;; "java.util.UUID" ; Unnecessary (have native Nippy implementation)
;; "java.util.Date" ; ''
;; "java.sql.Date" ; ''
2020-11-06 14:39:37 +00:00
2020-07-25 09:07:12 +00:00
# _ "java.time.*" ; Safe?
2020-11-06 14:39:37 +00:00
"java.time.Clock"
"java.time.LocalDate"
"java.time.LocalDateTime"
"java.time.LocalTime"
"java.time.MonthDay"
"java.time.OffsetDateTime"
"java.time.OffsetTime"
"java.time.Year"
"java.time.YearMonth"
"java.time.ZonedDateTime"
"java.time.ZoneId"
"java.time.ZoneOffset"
"java.time.DateTimeException"
2021-10-26 11:32:49 +00:00
"org.joda.time.DateTime"
2020-11-06 14:39:37 +00:00
2020-07-25 09:07:12 +00:00
"clojure.lang.ExceptionInfo"
"clojure.lang.ArityException" } )
2020-07-25 07:38:26 +00:00
2020-09-12 07:11:02 +00:00
( defn- allow-and-record? [ s ] ( = s "allow-and-record" ) )
2020-08-25 20:07:33 +00:00
( defn- split-class-names>set [ s ] ( when ( string? s ) ( if ( = s "" ) # { } ( set ( mapv str/trim ( str/split s # "[,:]" ) ) ) ) ) )
( comment
( split-class-names>set "" )
( split-class-names>set "foo, bar:baz" ) )
2020-09-10 16:42:25 +00:00
( comment ( .getName ( .getSuperclass ( .getClass ( java.util.concurrent.TimeoutException. ) ) ) ) )
( let [ ids
{ :legacy { :base { :prop "taoensso.nippy.serializable-whitelist-base" :env "TAOENSSO_NIPPY_SERIALIZABLE_WHITELIST_BASE" }
:add { :prop "taoensso.nippy.serializable-whitelist-add" :env "TAOENSSO_NIPPY_SERIALIZABLE_WHITELIST_ADD" } }
:freeze { :base { :prop "taoensso.nippy.freeze-serializable-allowlist-base" :env "TAOENSSO_NIPPY_FREEZE_SERIALIZABLE_ALLOWLIST_BASE" }
:add { :prop "taoensso.nippy.freeze-serializable-allowlist-add" :env "TAOENSSO_NIPPY_FREEZE_SERIALIZABLE_ALLOWLIST_ADD" } }
:thaw { :base { :prop "taoensso.nippy.thaw-serializable-allowlist-base" :env "TAOENSSO_NIPPY_THAW_SERIALIZABLE_ALLOWLIST_BASE" }
:add { :prop "taoensso.nippy.thaw-serializable-allowlist-add" :env "TAOENSSO_NIPPY_THAW_SERIALIZABLE_ALLOWLIST_ADD" } } } ]
2020-09-15 07:34:45 +00:00
( defn- init-allowlist [ action default incl-legacy? ]
2020-09-10 16:42:25 +00:00
( let [ allowlist-base
( or
2022-07-19 07:19:37 +00:00
( when-let [ s
( or
2023-09-25 09:24:58 +00:00
( do ( enc/get-sys-val* ( get-in ids [ action :base :prop ] ) ( get-in ids [ action :base :env ] ) ) )
( when incl-legacy? ( enc/get-sys-val* ( get-in ids [ :legacy :base :prop ] ) ( get-in ids [ :legacy :base :env ] ) ) ) ) ]
2022-07-19 07:19:37 +00:00
2020-09-12 07:11:02 +00:00
( if ( allow-and-record? s ) s ( split-class-names>set s ) ) )
2020-09-10 16:42:25 +00:00
default )
allowlist-add
2022-07-19 07:19:37 +00:00
( when-let [ s
( or
2023-09-25 09:24:58 +00:00
( do ( enc/get-sys-val* ( get-in ids [ action :add :prop ] ) ( get-in ids [ action :add :env ] ) ) )
( when incl-legacy? ( enc/get-sys-val* ( get-in ids [ :legacy :add :prop ] ) ( get-in ids [ :legacy :add :env ] ) ) ) ) ]
2022-07-19 07:19:37 +00:00
2020-09-12 07:11:02 +00:00
( if ( allow-and-record? s ) s ( split-class-names>set s ) ) ) ]
2020-09-10 16:42:25 +00:00
( if ( and allowlist-base allowlist-add )
( into ( enc/have set? allowlist-base ) allowlist-add )
( do allowlist-base ) ) ) ) )
( let [ doc
" Used when attempting to <freeze/thaw> an object that :
2020-09-12 07:11:02 +00:00
- Does NOT implement Nippy 's Freezable protocol.
- DOES implement Java 's Serializable interface.
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-12 07:11:02 +00:00
In this case , the allowlist will be checked to see if Java 's
Serializable interface may be used.
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-10 16:42:25 +00:00
This is a security measure to prevent possible Remote Code Execution
( RCE ) when thawing malicious payloads. See [ 1 ] for details.
2020-08-25 20:07:33 +00:00
2020-09-10 16:42:25 +00:00
If ` freeze ` encounters a disallowed Serialized class , it will throw.
If ` thaw ` encounters a disallowed Serialized class , it will :
2020-08-25 20:07:33 +00:00
2020-09-10 16:42:25 +00:00
- Throw if it 's not possible to safely quarantine the object
( object was frozen with Nippy < v2.15.0-final ) .
2020-08-25 20:07:33 +00:00
2020-09-10 16:42:25 +00:00
- Otherwise it will return a safely quarantined object of form
` { :nippy/unthawable { :class-name <> :content <quarantined-ba> } } ` .
- Quarantined objects may be manually unquarantined with
` read-quarantined-serializable-object-unsafe! ` .
2020-07-25 07:38:26 +00:00
2020-09-11 11:40:25 +00:00
There are 2 x allowlists :
- ` *freeze-serializable-allowlist* ` ; Checked when freezing
- ` *thaw-serializable-allowlist* ` ; Checked when thawing
2020-08-25 21:26:19 +00:00
2020-09-12 07:11:02 +00:00
Example allowlist values :
2020-09-11 11:40:25 +00:00
- ` ( fn allow-class? [ class-name ] true ) ` ; Arbitrary predicate fn
- ` # { \" java.lang.Throwable \" , \" clojure.lang.* \" } ` ; Set of class-names
2020-09-12 07:11:02 +00:00
- ` \" allow-and-record \" ` ; Special value, see [2]
2020-08-25 21:26:19 +00:00
2020-09-10 16:42:25 +00:00
Note that class-names in sets may contain \" * \" wildcards.
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-10 16:42:25 +00:00
Default allowlist values are :
2020-09-11 11:40:25 +00:00
- default-freeze-serializable-allowlist ; `{\"*\"}` => allow any class
2020-09-10 16:42:25 +00:00
- default-thaw-serializable-allowlist ; A set of common safe classes
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-10 16:42:25 +00:00
Allowlist values may be overridden with ` binding ` , ` alter-var-root ` , or :
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-10 16:42:25 +00:00
- ` taoensso.nippy.<freeze/thaw>-serializable-allowlist-base ` JVM property
- ` taoensso.nippy.<freeze/thaw>-serializable-allowlist-add ` JVM property
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-10 16:42:25 +00:00
- ` TAOENSSO_NIPPY_<FREEZE/THAW>_SERIALIZABLE_ALLOWLIST_BASE ` env var
- ` TAOENSSO_NIPPY_<FREEZE/THAW>_SERIALIZABLE_ALLOWLIST_ADD ` env var
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-10 16:42:25 +00:00
If present , these will be read as comma-separated lists of class names
and formed into sets. Each initial allowlist value will then be :
( into ( or <?base> <default> ) <?additions> ) .
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-10 16:42:25 +00:00
I.e. you can use :
- The \" base \" property/var to replace Nippy 's default allowlists.
- The \" add \" property/var to add to Nippy 's default allowlists.
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-12 07:11:02 +00:00
The special ` \" allow-and-record \" ` value is also possible , see [ 2 ] .
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-10 16:42:25 +00:00
Upgrading from an older version of Nippy and unsure whether you 've been
2020-09-12 07:11:02 +00:00
using Nippy 's Serializable support , or which classes to allow? See [ 2 ] .
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2023-09-25 09:24:58 +00:00
See also ` taoensso.encore/name-filter ` for a util to help easily build
more advanced predicate functions.
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-10 16:42:25 +00:00
Thanks to Timo Mihaljov ( @ solita-timo-mihaljov ) for an excellent report
identifying this vulnerability.
2020-08-25 20:07:33 +00:00
2020-09-12 07:11:02 +00:00
[ 1 ] https ://github.com/ptaoussanis/nippy/issues/130
[ 2 ] See ` allow-and-record-any-serializable-class-unsafe ` . " ]
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-15 07:34:45 +00:00
( enc/defonce ^ { :dynamic true :doc doc } *freeze-serializable-allowlist* ( init-allowlist :freeze default-freeze-serializable-allowlist false ) )
( enc/defonce ^ { :dynamic true :doc doc } *thaw-serializable-allowlist* ( init-allowlist :thaw default-thaw-serializable-allowlist true ) ) )
2023-08-16 09:56:19 +00:00
( enc/defonce ^ :dynamic ^ :no-doc ^ :deprecated *serializable-whitelist*
2020-09-15 07:34:45 +00:00
;; Mostly retained for https://github.com/juxt/crux/releases/tag/20.09-1.11.0
"DEPRECATED, now called `*thaw-serializable-allowlist*`" nil )
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-12 07:11:02 +00:00
( let [ nmax 1000
2020-09-12 14:59:39 +00:00
ngc 16000
2020-09-12 07:11:02 +00:00
state_ ( atom { } ) ; {<class-name> <frequency>}
lock_ ( atom nil ) ; ?promise
trim ( fn [ nmax state ]
( persistent!
( enc/reduce-top nmax val enc/rcompare conj!
( transient { } ) state ) ) ) ]
;; Note: trim strategy isn't perfect: it can be tough for new
;; classes to break into the top set since frequencies are being
;; reset only for classes outside the top set.
;;
;; In practice this is probably good enough since the main objective
;; is to discard one-off anonymous classes to protect state from
;; endlessly growing. Also `gc-rate` allows state to temporarily grow
;; significantly beyond `nmax` size, which helps to give new classes
;; some chance to accumulate a competitive frequency before next GC.
2020-09-12 10:26:18 +00:00
( defn ^ { :-state_ state_ } ; Undocumented
2020-09-12 07:11:02 +00:00
allow-and-record-any-serializable-class-unsafe
" A predicate ( fn allow-class? [ class-name ] ) fn that can be assigned
to ` *freeze-serializable-allowlist* ` and/or
` *thaw-serializable-allowlist* ` that :
- Will allow ANY class to use Nippy 's Serializable support ( unsafe ) .
- And will record { <class-name> <frequency-allowed> } for the <=1000
classes that ~ most frequently made use of this support.
` get-recorded-serializable-classes ` returns the recorded state.
This predicate is provided as a convenience for users upgrading from
previous versions of Nippy that allowed the use of Serializable for all
classes by default.
While transitioning from an unsafe->safe configuration , you can use
this predicate ( unsafe ) to record information about which classes have
been using Nippy 's Serializable support in your environment.
Once some time has passed , you can check the recorded state. If you 're
satisfied that all recorded classes are safely Serializable , you can
then merge the recorded classes into Nippy 's default allowlist/s , e.g. :
2020-09-11 11:40:25 +00:00
2020-09-12 07:11:02 +00:00
( alter-var-root # 'thaw-serializable-allowlist*
( fn [ _ ] ( into default-thaw-serializable-allowlist
( keys ( get-recorded-serializable-classes ) ) ) ) ) "
2020-09-11 11:40:25 +00:00
2020-09-12 07:11:02 +00:00
[ class-name ]
( when-let [ p @ lock_ ] @ p )
2020-09-11 11:40:25 +00:00
2020-09-12 07:11:02 +00:00
( let [ n ( count
( swap! state_
( fn [ m ] ( assoc m class-name
( inc ( long ( or ( get m class-name ) 0 ) ) ) ) ) ) ) ]
;; Garbage collection (GC): may be serializing anonymous classes, etc.
;; so input domain could be infinite
2020-09-12 14:59:39 +00:00
( when ( > n ngc ) ; Too many classes recorded, uncommon
( let [ p ( promise ) ]
( when ( compare-and-set! lock_ nil p ) ; Acquired GC lock
( try
( do ( reset! state_ ( trim nmax @ state_ ) ) ) ; GC state
( finally ( reset! lock_ nil ) ( deliver p nil ) ) ) ) ) )
2020-09-12 07:11:02 +00:00
n ) )
( defn get-recorded-serializable-classes
" Returns { <class-name> <frequency> } of the <=1000 classes that ~ most
frequently made use of Nippy 's Serializable support via
` allow-and-record-any-serializable-class-unsafe ` .
See that function 's docstring for more info. "
[ ] ( trim nmax @ state_ ) ) )
( comment
( count ( get-recorded-serializable-classes ) )
( enc/reduce-n
( fn [ _ n ] ( allow-and-record-any-serializable-class-unsafe ( str n ) ) )
nil 0 1 e5 ) )
( let [ fn? fn?
compile
( enc/fmemoize
( fn [ x ]
( if ( allow-and-record? x )
allow-and-record-any-serializable-class-unsafe
2023-09-25 09:24:58 +00:00
( enc/name-filter x ) ) ) )
2020-09-11 11:40:25 +00:00
2020-09-10 16:42:25 +00:00
conform?* ( fn [ x cn ] ( ( compile x ) cn ) ) ; Uncached because input domain possibly infinite
2020-08-28 12:03:47 +00:00
conform?
2020-09-10 16:42:25 +00:00
( fn [ x cn ]
2020-08-28 12:03:47 +00:00
( if ( fn? x )
2020-09-10 16:42:25 +00:00
( x cn ) ; Intentionally uncached, can be handy
( conform?* x cn ) ) ) ]
2020-08-28 12:03:47 +00:00
2020-09-10 16:42:25 +00:00
( defn- freeze-serializable-allowed? [ class-name ] ( conform? *freeze-serializable-allowlist* class-name ) )
2020-09-15 07:34:45 +00:00
( defn- thaw-serializable-allowed? [ class-name ]
( conform? ( or *serializable-whitelist* *thaw-serializable-allowlist* )
class-name ) ) )
2020-08-25 21:26:19 +00:00
( comment
2020-09-10 16:42:25 +00:00
( enc/qb 1 e6 ( freeze-serializable-allowed? "foo" ) ) ; 119.92
( binding [ *freeze-serializable-allowlist* # { "foo.*" "bar" } ]
( freeze-serializable-allowed? "foo.bar" ) ) )
2016-04-07 05:49:26 +00:00
2013-06-16 05:51:30 +00:00
;;;; Freezing
2013-10-23 18:28:58 +00:00
[new] [Storage efficiency] PoC: separate signed long types
Before:
Longs in [ -128, 127] use 1 byte
Longs in [-32768, 32767] use 2 bytes
etc.
After:
Longs in [ -255, 255] use 1 byte
Longs in [-65535, 65535] use 2 bytes
etc.
I.e. doubles the range of longs that can be stored by 1, 2, and 4 bytes.
This changes saves:
- 1 byte per long in [ 128, 255], or [ -129, -255]
- 2 bytes per long in [32768, 65535], or [-32769, -65535]
- 4 bytes per long ...
Is this advantage worth the extra complexity? Probably yes, given how
common longs (and colls of longs) are in Clojure.
2023-07-31 20:37:08 +00:00
( do
( def ^ :private ^ :const range-ubyte ( - Byte/MAX_VALUE Byte/MIN_VALUE ) )
( def ^ :private ^ :const range-ushort ( - Short/MAX_VALUE Short/MIN_VALUE ) )
( def ^ :private ^ :const range-uint ( - Integer/MAX_VALUE Integer/MIN_VALUE ) ) )
2016-04-13 04:57:50 +00:00
( do
( defmacro write-id [ out id ] ` ( .writeByte ~ out ~ id ) )
2023-07-31 21:13:19 +00:00
( defmacro ^ :private sm-count?* [ n ] ` ( <= ~ n range-ubyte ) ) ; Unsigned
( defmacro ^ :private sm-count? [ n ] ` ( <= ~ n Byte/MAX_VALUE ) )
( defmacro ^ :private md-count? [ n ] ` ( <= ~ n Short/MAX_VALUE ) )
2016-04-13 04:57:50 +00:00
2023-07-31 21:13:19 +00:00
( defmacro ^ :private write-sm-count* [ out n ] ` ( .writeByte ~ out ( + ~ n Byte/MIN_VALUE ) ) )
( defmacro ^ :private write-sm-count [ out n ] ` ( .writeByte ~ out ~ n ) )
( defmacro ^ :private write-md-count [ out n ] ` ( .writeShort ~ out ~ n ) )
( defmacro ^ :private write-lg-count [ out n ] ` ( .writeInt ~ out ~ n ) )
2016-04-13 04:57:50 +00:00
2023-07-31 21:13:19 +00:00
( defmacro ^ :private read-sm-count* [ in ] ` ( - ( .readByte ~ in ) Byte/MIN_VALUE ) )
( defmacro ^ :private read-sm-count [ in ] ` ( .readByte ~ in ) )
( defmacro ^ :private read-md-count [ in ] ` ( .readShort ~ in ) )
( defmacro ^ :private read-lg-count [ in ] ` ( .readInt ~ in ) ) )
2014-01-23 07:30:56 +00:00
2016-07-16 12:09:38 +00:00
; We extend `IFreezable1` to supported types:
( defprotocol IFreezable1 ( -freeze-without-meta! [ x data-output ] ) )
( defprotocol IFreezable2 ( -freeze-with-meta! [ x data-output ] ) )
( extend-protocol IFreezable2 ; Must be a separate protocol
2023-07-27 14:02:43 +00:00
clojure.lang.IObj ; IMeta => `meta` will work, IObj => `with-meta` will work
2016-07-16 12:09:38 +00:00
( -freeze-with-meta! [ x ^ DataOutput data-output ]
2023-07-27 14:02:43 +00:00
( let [ m ( when *incl-metadata?* ( meta x ) ) ]
2016-07-16 12:09:38 +00:00
( when m
( write-id data-output id-meta )
( -freeze-without-meta! m data-output ) ) )
( -freeze-without-meta! x data-output ) )
2020-07-24 20:50:05 +00:00
nil ( -freeze-with-meta! [ x data-output ] ( -freeze-without-meta! x data-output ) )
Object ( -freeze-with-meta! [ x data-output ] ( -freeze-without-meta! x data-output ) ) )
2016-07-16 12:09:38 +00:00
2023-07-31 21:13:19 +00:00
( defn- write-bytes-sm* [ ^ DataOutput out ^ bytes ba ] ( let [ len ( alength ba ) ] ( write-sm-count* out len ) ( .write out ba 0 len ) ) )
( defn- write-bytes-sm [ ^ DataOutput out ^ bytes ba ] ( let [ len ( alength ba ) ] ( write-sm-count out len ) ( .write out ba 0 len ) ) )
( defn- write-bytes-md [ ^ DataOutput out ^ bytes ba ] ( let [ len ( alength ba ) ] ( write-md-count out len ) ( .write out ba 0 len ) ) )
( defn- write-bytes-lg [ ^ DataOutput out ^ bytes ba ] ( let [ len ( alength ba ) ] ( write-lg-count out len ) ( .write out ba 0 len ) ) )
2022-07-19 07:19:37 +00:00
( defn- write-bytes [ ^ DataOutput out ^ bytes ba ]
2016-04-07 05:49:26 +00:00
( let [ len ( alength ba ) ]
( if ( zero? len )
2016-04-13 04:57:50 +00:00
( write-id out id-bytes-0 )
2016-04-07 05:49:26 +00:00
( do
2020-07-24 20:50:05 +00:00
( enc/cond
2022-07-19 07:19:37 +00:00
( sm-count? len ) ( do ( write-id out id-bytes-sm ) ( write-sm-count out len ) )
( md-count? len ) ( do ( write-id out id-bytes-md ) ( write-md-count out len ) )
:else ( do ( write-id out id-bytes-lg ) ( write-lg-count out len ) ) )
2016-04-07 05:49:26 +00:00
( .write out ba 0 len ) ) ) ) )
( defn- write-biginteger [ out ^ BigInteger n ] ( write-bytes-lg out ( .toByteArray n ) ) )
2023-07-31 21:13:19 +00:00
( defn- write-str-sm* [ ^ DataOutput out ^ String s ] ( write-bytes-sm* out ( .getBytes s StandardCharsets/UTF_8 ) ) )
2023-08-03 13:44:33 +00:00
( defn- write-str-sm [ ^ DataOutput out ^ String s ] ( write-bytes-sm out ( .getBytes s StandardCharsets/UTF_8 ) ) )
2023-07-31 21:13:19 +00:00
( defn- write-str-md [ ^ DataOutput out ^ String s ] ( write-bytes-md out ( .getBytes s StandardCharsets/UTF_8 ) ) )
( defn- write-str-lg [ ^ DataOutput out ^ String s ] ( write-bytes-lg out ( .getBytes s StandardCharsets/UTF_8 ) ) )
( defn- write-str [ ^ DataOutput out ^ String s ]
2016-04-07 05:49:26 +00:00
( if ( identical? s "" )
2016-04-13 04:57:50 +00:00
( write-id out id-str-0 )
2023-01-25 14:24:50 +00:00
( let [ ba ( .getBytes s StandardCharsets/UTF_8 )
2016-04-07 05:49:26 +00:00
len ( alength ba ) ]
2020-07-24 20:50:05 +00:00
( enc/cond
2023-07-31 21:13:19 +00:00
( sm-count?* len ) ( do ( write-id out id-str-sm* ) ( write-sm-count* out len ) )
( md-count? len ) ( do ( write-id out id-str-md ) ( write-md-count out len ) )
:else ( do ( write-id out id-str-lg ) ( write-lg-count out len ) ) )
2016-04-07 05:49:26 +00:00
( .write out ba 0 len ) ) ) )
( defn- write-kw [ ^ DataOutput out kw ]
( let [ s ( if-let [ ns ( namespace kw ) ] ( str ns "/" ( name kw ) ) ( name kw ) )
2023-01-25 14:24:50 +00:00
ba ( .getBytes s StandardCharsets/UTF_8 )
2016-04-07 05:49:26 +00:00
len ( alength ba ) ]
2020-07-24 20:50:05 +00:00
( enc/cond
2022-07-19 07:19:37 +00:00
( sm-count? len ) ( do ( write-id out id-kw-sm ) ( write-sm-count out len ) )
( md-count? len ) ( do ( write-id out id-kw-md ) ( write-md-count out len ) )
;; :else (do (write-id out id-kw-lg) (write-lg-count out len)) ; Unrealistic
2020-07-24 20:50:05 +00:00
:else ( throw ( ex-info "Keyword too long" { :full-name s } ) ) )
2015-09-29 09:02:46 +00:00
2016-04-07 05:49:26 +00:00
( .write out ba 0 len ) ) )
2015-09-29 09:02:46 +00:00
2016-04-07 05:49:26 +00:00
( defn- write-sym [ ^ DataOutput out s ]
( let [ s ( if-let [ ns ( namespace s ) ] ( str ns "/" ( name s ) ) ( name s ) )
2023-01-25 14:24:50 +00:00
ba ( .getBytes s StandardCharsets/UTF_8 )
2016-04-07 05:49:26 +00:00
len ( alength ba ) ]
2020-07-24 20:50:05 +00:00
( enc/cond
2022-07-19 07:19:37 +00:00
( sm-count? len ) ( do ( write-id out id-sym-sm ) ( write-sm-count out len ) )
( md-count? len ) ( do ( write-id out id-sym-md ) ( write-md-count out len ) )
;; :else (do (write-id out id-sym-lg) (write-lg-count out len)) ; Unrealistic
2020-07-24 20:50:05 +00:00
:else ( throw ( ex-info "Symbol too long" { :full-name s } ) ) )
2016-04-07 05:49:26 +00:00
( .write out ba 0 len ) ) )
2015-09-29 09:02:46 +00:00
2016-04-07 05:49:26 +00:00
( defn- write-long [ ^ DataOutput out ^ long n ]
2020-07-24 20:50:05 +00:00
( enc/cond
2022-07-19 07:19:37 +00:00
( zero? n ) ( write-id out id-long-0 )
( pos? n )
2020-07-24 20:50:05 +00:00
( enc/cond
[new] [Storage efficiency] PoC: separate signed long types
Before:
Longs in [ -128, 127] use 1 byte
Longs in [-32768, 32767] use 2 bytes
etc.
After:
Longs in [ -255, 255] use 1 byte
Longs in [-65535, 65535] use 2 bytes
etc.
I.e. doubles the range of longs that can be stored by 1, 2, and 4 bytes.
This changes saves:
- 1 byte per long in [ 128, 255], or [ -129, -255]
- 2 bytes per long in [32768, 65535], or [-32769, -65535]
- 4 bytes per long ...
Is this advantage worth the extra complexity? Probably yes, given how
common longs (and colls of longs) are in Clojure.
2023-07-31 20:37:08 +00:00
( <= n range-ubyte ) ( do ( write-id out id-long-pos-sm ) ( .writeByte out ( + n Byte/MIN_VALUE ) ) )
( <= n range-ushort ) ( do ( write-id out id-long-pos-md ) ( .writeShort out ( + n Short/MIN_VALUE ) ) )
( <= n range-uint ) ( do ( write-id out id-long-pos-lg ) ( .writeInt out ( + n Integer/MIN_VALUE ) ) )
:else ( do ( write-id out id-long-xl ) ( .writeLong out n ) ) )
2015-09-29 09:02:46 +00:00
:else
[new] [Storage efficiency] PoC: separate signed long types
Before:
Longs in [ -128, 127] use 1 byte
Longs in [-32768, 32767] use 2 bytes
etc.
After:
Longs in [ -255, 255] use 1 byte
Longs in [-65535, 65535] use 2 bytes
etc.
I.e. doubles the range of longs that can be stored by 1, 2, and 4 bytes.
This changes saves:
- 1 byte per long in [ 128, 255], or [ -129, -255]
- 2 bytes per long in [32768, 65535], or [-32769, -65535]
- 4 bytes per long ...
Is this advantage worth the extra complexity? Probably yes, given how
common longs (and colls of longs) are in Clojure.
2023-07-31 20:37:08 +00:00
( let [ y ( - n ) ]
( enc/cond
( <= y range-ubyte ) ( do ( write-id out id-long-neg-sm ) ( .writeByte out ( + y Byte/MIN_VALUE ) ) )
( <= y range-ushort ) ( do ( write-id out id-long-neg-md ) ( .writeShort out ( + y Short/MIN_VALUE ) ) )
( <= y range-uint ) ( do ( write-id out id-long-neg-lg ) ( .writeInt out ( + y Integer/MIN_VALUE ) ) )
:else ( do ( write-id out id-long-xl ) ( .writeLong out n ) ) ) ) ) )
2016-04-07 05:49:26 +00:00
( defmacro ^ :private -run! [ proc coll ] ` ( do ( reduce # ( ~ proc %2 ) nil ~ coll ) nil ) )
( defmacro ^ :private -run-kv! [ proc m ] ` ( do ( reduce-kv # ( ~ proc %2 %3 ) nil ~ m ) nil ) )
( defn- write-vec [ ^ DataOutput out v ]
2016-03-08 04:13:46 +00:00
( let [ cnt ( count v ) ]
2016-04-07 05:49:26 +00:00
( if ( zero? cnt )
2016-04-13 04:57:50 +00:00
( write-id out id-vec-0 )
2016-04-07 05:49:26 +00:00
( do
2020-07-24 20:50:05 +00:00
( enc/cond
2023-07-31 21:13:19 +00:00
( sm-count?* cnt )
2020-07-24 20:50:05 +00:00
( enc/cond
2016-04-13 04:57:50 +00:00
( == cnt 2 ) ( write-id out id-vec-2 )
( == cnt 3 ) ( write-id out id-vec-3 )
2023-07-31 21:13:19 +00:00
:else ( do ( write-id out id-vec-sm* ) ( write-sm-count* out cnt ) ) )
2016-04-07 05:49:26 +00:00
2022-07-19 07:19:37 +00:00
( md-count? cnt ) ( do ( write-id out id-vec-md ) ( write-md-count out cnt ) )
:else ( do ( write-id out id-vec-lg ) ( write-lg-count out cnt ) ) )
2016-04-07 05:49:26 +00:00
2016-07-16 12:09:38 +00:00
( -run! ( fn [ in ] ( -freeze-with-meta! in out ) ) v ) ) ) ) )
2016-04-07 05:49:26 +00:00
( defn- write-kvs
( [ ^ DataOutput out id-lg coll ]
( let [ cnt ( count coll ) ]
2016-04-13 04:57:50 +00:00
( write-id out id-lg )
( write-lg-count out cnt )
2016-04-07 05:49:26 +00:00
( -run-kv!
( fn [ k v ]
2016-07-16 12:09:38 +00:00
( -freeze-with-meta! k out )
( -freeze-with-meta! v out ) )
2016-04-07 05:49:26 +00:00
coll ) ) )
( [ ^ DataOutput out id-empty id-sm id-md id-lg coll ]
( let [ cnt ( count coll ) ]
( if ( zero? cnt )
2016-04-13 04:57:50 +00:00
( write-id out id-empty )
2016-04-07 05:49:26 +00:00
( do
2020-07-24 20:50:05 +00:00
( enc/cond
2022-07-19 07:19:37 +00:00
( sm-count? cnt ) ( do ( write-id out id-sm ) ( write-sm-count out cnt ) )
( md-count? cnt ) ( do ( write-id out id-md ) ( write-md-count out cnt ) )
:else ( do ( write-id out id-lg ) ( write-lg-count out cnt ) ) )
2016-04-07 05:49:26 +00:00
( -run-kv!
( fn [ k v ]
2016-07-16 12:09:38 +00:00
( -freeze-with-meta! k out )
( -freeze-with-meta! v out ) )
2016-04-07 05:49:26 +00:00
coll ) ) ) ) ) )
( defn- write-counted-coll
( [ ^ DataOutput out id-lg coll ]
( let [ cnt ( count coll ) ]
;; (assert (counted? coll))
2016-04-13 04:57:50 +00:00
( write-id out id-lg )
( write-lg-count out cnt )
2016-07-16 12:09:38 +00:00
( -run! ( fn [ in ] ( -freeze-with-meta! in out ) ) coll ) ) )
2016-04-07 05:49:26 +00:00
( [ ^ DataOutput out id-empty id-sm id-md id-lg coll ]
( let [ cnt ( count coll ) ]
;; (assert (counted? coll))
( if ( zero? cnt )
2016-04-13 04:57:50 +00:00
( write-id out id-empty )
2016-04-07 05:49:26 +00:00
( do
2020-07-24 20:50:05 +00:00
( enc/cond
2022-07-19 07:19:37 +00:00
( sm-count? cnt ) ( do ( write-id out id-sm ) ( write-sm-count out cnt ) )
( md-count? cnt ) ( do ( write-id out id-md ) ( write-md-count out cnt ) )
:else ( do ( write-id out id-lg ) ( write-lg-count out cnt ) ) )
2016-04-07 05:49:26 +00:00
2016-07-16 12:09:38 +00:00
( -run! ( fn [ in ] ( -freeze-with-meta! in out ) ) coll ) ) ) ) ) )
2016-04-07 05:49:26 +00:00
( defn- write-uncounted-coll
( [ ^ DataOutput out id-lg coll ]
;; (assert (not (counted? coll)))
( let [ bas ( ByteArrayOutputStream. 32 )
sout ( DataOutputStream. bas )
2016-07-16 12:09:38 +00:00
^ long cnt ( reduce ( fn [ ^ long cnt in ] ( -freeze-with-meta! in sout ) ( unchecked-inc cnt ) ) 0 coll )
2016-04-07 05:49:26 +00:00
ba ( .toByteArray bas ) ]
2016-04-13 04:57:50 +00:00
( write-id out id-lg )
( write-lg-count out cnt )
( .write out ba ) ) )
2016-04-07 05:49:26 +00:00
( [ ^ DataOutput out id-empty id-sm id-md id-lg coll ]
( let [ bas ( ByteArrayOutputStream. 32 )
sout ( DataOutputStream. bas )
2016-07-16 12:09:38 +00:00
^ long cnt ( reduce ( fn [ ^ long cnt in ] ( -freeze-with-meta! in sout ) ( unchecked-inc cnt ) ) 0 coll )
2016-04-07 05:49:26 +00:00
ba ( .toByteArray bas ) ]
( if ( zero? cnt )
2016-04-13 04:57:50 +00:00
( write-id out id-empty )
2016-04-07 05:49:26 +00:00
( do
2020-07-24 20:50:05 +00:00
( enc/cond
2022-07-19 07:19:37 +00:00
( sm-count? cnt ) ( do ( write-id out id-sm ) ( write-sm-count out cnt ) )
( md-count? cnt ) ( do ( write-id out id-md ) ( write-md-count out cnt ) )
:else ( do ( write-id out id-lg ) ( write-lg-count out cnt ) ) )
2016-04-07 05:49:26 +00:00
( .write out ba ) ) ) ) ) )
( defn- write-coll
( [ out id-lg coll ]
( if ( counted? coll )
( write-counted-coll out id-lg coll )
( write-uncounted-coll out id-lg coll ) ) )
( [ out id-empty id-sm id-md id-lg coll ]
( if ( counted? coll )
( write-counted-coll out id-empty id-sm id-md id-lg coll )
( write-uncounted-coll out id-empty id-sm id-md id-lg coll ) ) ) )
;; Micro-optimization:
;; As (write-kvs out id-map-0 id-map-sm id-map-md id-map-lg x)
( defn- write-map [ ^ DataOutput out m ]
( let [ cnt ( count m ) ]
( if ( zero? cnt )
2016-04-13 04:57:50 +00:00
( write-id out id-map-0 )
2016-04-07 05:49:26 +00:00
( do
2020-07-24 20:50:05 +00:00
( enc/cond
2023-07-31 21:13:19 +00:00
( sm-count?* cnt ) ( do ( write-id out id-map-sm* ) ( write-sm-count* out cnt ) )
( md-count? cnt ) ( do ( write-id out id-map-md ) ( write-md-count out cnt ) )
:else ( do ( write-id out id-map-lg ) ( write-lg-count out cnt ) ) )
2016-04-07 05:49:26 +00:00
( -run-kv!
( fn [ k v ]
2016-07-16 12:09:38 +00:00
( -freeze-with-meta! k out )
( -freeze-with-meta! v out ) )
2016-04-07 05:49:26 +00:00
m ) ) ) ) )
;; Micro-optimization:
;; As (write-counted-coll out id-set-0 id-set-sm id-set-md id-set-lg x)
( defn- write-set [ ^ DataOutput out s ]
( let [ cnt ( count s ) ]
( if ( zero? cnt )
2016-04-13 04:57:50 +00:00
( write-id out id-set-0 )
2016-04-07 05:49:26 +00:00
( do
2020-07-24 20:50:05 +00:00
( enc/cond
2023-07-31 21:13:19 +00:00
( sm-count?* cnt ) ( do ( write-id out id-set-sm* ) ( write-sm-count* out cnt ) )
( md-count? cnt ) ( do ( write-id out id-set-md ) ( write-md-count out cnt ) )
:else ( do ( write-id out id-set-lg ) ( write-lg-count out cnt ) ) )
2016-04-07 05:49:26 +00:00
2016-07-16 12:09:38 +00:00
( -run! ( fn [ in ] ( -freeze-with-meta! in out ) ) s ) ) ) ) )
2016-04-07 05:49:26 +00:00
2020-07-24 17:38:16 +00:00
( defn- write-objects [ ^ DataOutput out ^ objects ary ]
( let [ len ( alength ary ) ]
( write-id out id-objects-lg )
( write-lg-count out len )
( -run! ( fn [ in ] ( -freeze-with-meta! in out ) ) ary ) ) )
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
( defn- write-serializable [ ^ DataOutput out x ^ String class-name ]
2016-04-07 05:49:26 +00:00
( when-debug ( println ( str "write-serializable: " ( type x ) ) ) )
2023-01-25 14:24:50 +00:00
( let [ class-name-ba ( .getBytes class-name StandardCharsets/UTF_8 )
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
len ( alength class-name-ba ) ]
2020-07-24 20:50:05 +00:00
( enc/cond
2022-07-19 07:19:37 +00:00
( sm-count? len ) ( do ( write-id out id-sz-quarantined-sm ) ( write-bytes-sm out class-name-ba ) )
( md-count? len ) ( do ( write-id out id-sz-quarantined-md ) ( write-bytes-md out class-name-ba ) )
;; :else (do (write-id out id-sz-quarantined-lg) (write-bytes-md out class-name-ba)) ; Unrealistic
2016-03-08 04:13:46 +00:00
:else
2020-07-24 20:50:05 +00:00
( throw
( ex-info "Serializable class name too long"
{ :class-name class-name } ) ) )
2016-03-08 04:13:46 +00:00
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
;; Legacy: write object directly to out.
;; (.writeObject (ObjectOutputStream. out) x)
;; Quarantined: write object to ba, then ba to out.
;; We'll have object length during thaw, allowing us to skip readObject.
2020-09-10 16:42:25 +00:00
( let [ quarantined-ba ( ByteArrayOutputStream. ) ]
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
( .writeObject ( ObjectOutputStream. ( DataOutputStream. quarantined-ba ) ) x )
( write-bytes out ( .toByteArray quarantined-ba ) ) ) ) )
2015-09-29 13:10:09 +00:00
2016-04-07 05:49:26 +00:00
( defn- write-readable [ ^ DataOutput out x ]
( when-debug ( println ( str "write-readable: " ( type x ) ) ) )
( let [ edn ( enc/pr-edn x )
2023-01-25 14:24:50 +00:00
edn-ba ( .getBytes ^ String edn StandardCharsets/UTF_8 )
2016-04-07 05:49:26 +00:00
len ( alength edn-ba ) ]
2020-07-24 20:50:05 +00:00
( enc/cond
2022-07-19 07:19:37 +00:00
( sm-count? len ) ( do ( write-id out id-reader-sm ) ( write-bytes-sm out edn-ba ) )
( md-count? len ) ( do ( write-id out id-reader-md ) ( write-bytes-md out edn-ba ) )
:else ( do ( write-id out id-reader-lg ) ( write-bytes-lg out edn-ba ) ) ) ) )
2016-04-07 05:49:26 +00:00
( defn try-write-serializable [ out x ]
2020-09-10 16:42:25 +00:00
( when ( and ( instance? Serializable x ) ( not ( fn? x ) ) )
2016-06-17 05:03:07 +00:00
( try
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
( let [ class-name ( .getName ( class x ) ) ] ; Reflect
2020-09-10 16:42:25 +00:00
( when ( freeze-serializable-allowed? class-name )
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
( write-serializable out x class-name )
true ) )
2016-06-17 05:03:07 +00:00
( catch Throwable _ nil ) ) ) )
2016-04-07 05:49:26 +00:00
( defn try-write-readable [ out x ]
( when ( utils/readable? x )
2016-06-17 05:03:07 +00:00
( try
( write-readable out x )
true
( catch Throwable _ nil ) ) ) )
2016-04-07 05:49:26 +00:00
( defn- try-pr-edn [ x ]
( try
( enc/pr-edn x )
( catch Throwable _
( try
( str x )
2020-09-10 10:08:28 +00:00
( catch Throwable _
:nippy/unprintable ) ) ) ) )
2016-04-07 05:49:26 +00:00
( defn write-unfreezable [ out x ]
2016-07-16 12:09:38 +00:00
( -freeze-without-meta!
2020-09-10 10:08:28 +00:00
{ :nippy/unfreezable
{ :type ( type x )
:content ( try-pr-edn x ) } }
2016-04-07 05:49:26 +00:00
out ) )
( defn throw-unfreezable [ x ]
2020-09-10 10:08:28 +00:00
( let [ t ( type x ) ]
( throw
( ex-info ( str "Unfreezable type: " t )
{ :type t
:as-str ( try-pr-edn x ) } ) ) ) )
2013-04-14 07:44:06 +00:00
2016-07-16 12:09:38 +00:00
;; Public `-freeze-with-meta!` with different arg order
2016-04-07 05:49:26 +00:00
( defn freeze-to-out!
2016-07-18 04:30:38 +00:00
" Serializes arg ( any Clojure data type ) to a DataOutput.
This is a low-level util : in most cases you 'll want ` freeze ` instead. "
2016-07-16 12:09:38 +00:00
[ ^ DataOutput data-output x ] ( -freeze-with-meta! x data-output ) )
2016-04-07 05:49:26 +00:00
( defmacro ^ :private freezer [ type & body ]
2016-07-16 12:09:38 +00:00
` ( extend-type ~ type IFreezable1
( ~ '-freeze-without-meta! [ ~ 'x ~ ( with-meta 'out { :tag 'DataOutput } ) ]
2015-09-28 09:38:48 +00:00
~@ body ) ) )
2012-07-06 19:12:59 +00:00
2016-04-07 05:49:26 +00:00
( defmacro ^ :private id-freezer [ type id & body ]
2016-07-16 12:09:38 +00:00
` ( extend-type ~ type IFreezable1
( ~ '-freeze-without-meta! [ ~ 'x ~ ( with-meta 'out { :tag 'DataOutput } ) ]
2016-04-13 04:57:50 +00:00
( write-id ~ 'out ~ id )
2016-04-07 05:49:26 +00:00
~@ body ) ) )
2023-07-31 14:43:09 +00:00
;;;; Caching ; Experimental
;; Nb: don't use an auto initialValue; can cause thread-local state to
;; accidentally hang around with the use of `freeze-to-out!`, etc.
;; Safer to require explicit activation through `with-cache`.
( def ^ ThreadLocal -cache-proxy
"{[<x> <meta>] <idx>} for freezing, {<idx> <x-with-meta>} for thawing."
( proxy [ ThreadLocal ] [ ] ) )
2016-07-16 12:09:38 +00:00
( defmacro ^ :private with-cache
2023-07-31 14:43:09 +00:00
" Executes body with support for freezing/thawing cached values.
This is a low-level util : you won 't need to use this yourself unless
you 're using ` freeze-to-out! ` or ` thaw-from-in! ` ( also low-level utils ) .
See also ` cache ` . "
2016-07-16 12:09:38 +00:00
[ & body ]
` ( try
2020-09-10 10:03:33 +00:00
( .set -cache-proxy ( volatile! nil ) )
2016-07-16 12:09:38 +00:00
( do ~@ body )
( finally ( .remove -cache-proxy ) ) ) )
2023-07-31 14:43:09 +00:00
( deftype Cached [ val ] )
( defn cache
" Experimental , subject to change. Feedback welcome.
Wraps value so that future writes of the same wrapped value with same
metadata will be efficiently encoded as references to this one.
( freeze [ ( cache \" foo \" ) ( cache \" foo \" ) ( cache \" foo \" ) ] )
will incl. a single \" foo \" , plus 2 x single-byte references to \" foo \" . "
[ x ]
( if ( instance? Cached x ) x ( Cached. x ) ) )
( comment ( cache "foo" ) )
( freezer Cached
( let [ x-val ( .-val x ) ]
( if-let [ cache_ ( .get -cache-proxy ) ]
( let [ cache @ cache_
k # _x-val [ x-val ( meta x-val ) ] ; Also check meta for equality
?idx ( get cache k )
^ int idx ( or ?idx
( let [ idx ( count cache ) ]
( vswap! cache_ assoc k idx )
idx ) )
first-occurance? ( nil? ?idx ) ]
( enc/cond
( sm-count? idx )
( case ( int idx )
0 ( do ( write-id out id-cached-0 ) ( when first-occurance? ( -freeze-with-meta! x-val out ) ) )
1 ( do ( write-id out id-cached-1 ) ( when first-occurance? ( -freeze-with-meta! x-val out ) ) )
2 ( do ( write-id out id-cached-2 ) ( when first-occurance? ( -freeze-with-meta! x-val out ) ) )
3 ( do ( write-id out id-cached-3 ) ( when first-occurance? ( -freeze-with-meta! x-val out ) ) )
4 ( do ( write-id out id-cached-4 ) ( when first-occurance? ( -freeze-with-meta! x-val out ) ) )
5 ( do ( write-id out id-cached-5 ) ( when first-occurance? ( -freeze-with-meta! x-val out ) ) )
6 ( do ( write-id out id-cached-6 ) ( when first-occurance? ( -freeze-with-meta! x-val out ) ) )
7 ( do ( write-id out id-cached-7 ) ( when first-occurance? ( -freeze-with-meta! x-val out ) ) )
( do
( write-id out id-cached-sm )
( write-sm-count out idx )
( when first-occurance? ( -freeze-with-meta! x-val out ) ) ) )
( md-count? idx )
( do
( write-id out id-cached-md )
( write-md-count out idx )
( when first-occurance? ( -freeze-with-meta! x-val out ) ) )
:else
;; (throw (ex-info "Max cache size exceeded" {:idx idx}))
( -freeze-with-meta! x-val out ) ; Just freeze uncached
) )
( -freeze-with-meta! x-val out ) ) ) )
2016-04-12 17:52:15 +00:00
( declare thaw-from-in! )
2016-07-16 12:09:38 +00:00
( def ^ :private thaw-cached
( let [ not-found ( Object. ) ]
( fn [ idx in ]
( if-let [ cache_ ( .get -cache-proxy ) ]
( let [ v ( get @ cache_ idx not-found ) ]
( if ( identical? v not-found )
( let [ x ( thaw-from-in! in ) ]
2020-09-10 10:03:33 +00:00
( vswap! cache_ assoc idx x )
2016-07-16 12:09:38 +00:00
x )
v ) )
( throw ( ex-info "No cache_ established, can't thaw. See `with-cache`."
{ } ) ) ) ) ) )
2016-04-12 17:52:15 +00:00
2023-07-31 14:43:09 +00:00
( comment
( thaw ( freeze [ ( cache "foo" ) ( cache "foo" ) ( cache "foo" ) ] ) )
( let [ v1 ( with-meta [ ] { :id :v1 } )
v2 ( with-meta [ ] { :id :v2 } ) ]
( mapv meta
( thaw ( freeze [ ( cache v1 ) ( cache v2 ) ( cache v1 ) ( cache v2 ) ] ) ) ) ) )
2016-04-12 17:52:15 +00:00
;;;;
2016-04-07 05:49:26 +00:00
( id-freezer nil id-nil )
( id-freezer ( type ' ( ) ) id-list-0 )
( id-freezer Character id-char ( .writeChar out ( int x ) ) )
( id-freezer Byte id-byte ( .writeByte out x ) )
( id-freezer Short id-short ( .writeShort out x ) )
( id-freezer Integer id-integer ( .writeInt out x ) )
( id-freezer BigInt id-bigint ( write-biginteger out ( .toBigInteger x ) ) )
( id-freezer BigInteger id-biginteger ( write-biginteger out x ) )
( id-freezer Pattern id-regex ( write-str out ( str x ) ) )
( id-freezer Float id-float ( .writeFloat out x ) )
( id-freezer BigDecimal id-bigdec
2015-09-29 09:02:46 +00:00
( write-biginteger out ( .unscaledValue x ) )
2022-07-19 07:19:37 +00:00
( .writeInt out ( .scale x ) ) )
2012-07-06 19:12:59 +00:00
2016-04-07 05:49:26 +00:00
( id-freezer Ratio id-ratio
2015-09-29 09:02:46 +00:00
( write-biginteger out ( .numerator x ) )
( write-biginteger out ( .denominator x ) ) )
2012-07-06 19:12:59 +00:00
2023-07-27 12:56:34 +00:00
( id-freezer java.util.Date id-util-date ( .writeLong out ( .getTime x ) ) )
( id-freezer java.sql.Date id-sql-date ( .writeLong out ( .getTime x ) ) )
2020-07-24 17:38:16 +00:00
( id-freezer URI id-uri
( write-str out ( .toString x ) ) )
2016-04-07 05:49:26 +00:00
( id-freezer UUID id-uuid
2015-09-29 09:02:46 +00:00
( .writeLong out ( .getMostSignificantBits x ) )
( .writeLong out ( .getLeastSignificantBits x ) ) )
2013-08-06 20:55:27 +00:00
2022-06-23 10:23:49 +00:00
( freezer Boolean ( if ( boolean x ) ( write-id out id-true ) ( write-id out id-false ) ) )
2020-07-24 17:38:16 +00:00
( freezer ( Class/forName "[B" ) ( write-bytes out x ) )
( freezer ( Class/forName "[Ljava.lang.Object;" ) ( write-objects out x ) )
2016-04-07 05:49:26 +00:00
( freezer String ( write-str out x ) )
( freezer Keyword ( write-kw out x ) )
( freezer Symbol ( write-sym out x ) )
( freezer Long ( write-long out x ) )
( freezer Double
2016-04-13 04:57:50 +00:00
( if ( zero? ^ double x )
2022-07-19 07:19:37 +00:00
( do ( write-id out id-double-0 ) )
( do ( write-id out id-double ) ( .writeDouble out x ) ) ) )
2016-04-07 05:49:26 +00:00
2022-10-13 16:52:19 +00:00
( freezer PersistentQueue ( write-counted-coll out id-queue-lg x ) )
( freezer PersistentTreeSet ( write-counted-coll out id-sorted-set-lg x ) )
( freezer PersistentTreeMap ( write-kvs out id-sorted-map-lg x ) )
( freezer APersistentVector ( write-vec out x ) )
( freezer APersistentSet ( write-set out x ) )
( freezer APersistentMap ( write-map out x ) )
2016-04-07 05:49:26 +00:00
( freezer PersistentList ( write-counted-coll out id-list-0 id-list-sm id-list-md id-list-lg x ) )
( freezer LazySeq ( write-uncounted-coll out id-seq-0 id-seq-sm id-seq-md id-seq-lg x ) )
( freezer ISeq ( write-coll out id-seq-0 id-seq-sm id-seq-md id-seq-lg x ) )
( freezer IRecord
2020-07-24 20:50:05 +00:00
( let [ class-name ( .getName ( class x ) ) ; Reflect
2023-01-25 14:24:50 +00:00
class-name-ba ( .getBytes class-name StandardCharsets/UTF_8 )
2020-07-24 20:50:05 +00:00
len ( alength class-name-ba ) ]
( enc/cond
2022-07-19 07:19:37 +00:00
( sm-count? len ) ( do ( write-id out id-record-sm ) ( write-bytes-sm out class-name-ba ) )
( md-count? len ) ( do ( write-id out id-record-md ) ( write-bytes-md out class-name-ba ) )
;; :else (do (write-id out id-record-lg) (write-bytes-md out class-name-ba)) ; Unrealistic
2016-04-07 05:49:26 +00:00
:else
2020-07-24 20:50:05 +00:00
( throw
( ex-info "Record class name too long"
{ :class-name class-name } ) ) )
2016-04-07 05:49:26 +00:00
2016-07-16 12:09:38 +00:00
( -freeze-without-meta! ( into { } x ) out ) ) )
2018-10-06 07:54:28 +00:00
2022-06-27 07:51:47 +00:00
( let [ munge-cached ( enc/fmemoize munge ) ]
( freezer IType
( let [ aclass ( class x )
class-name ( .getName aclass ) ]
( write-id out id-type )
( write-str out class-name )
;; Could cache basis generation for given class-name with generalized
;; `-cache-proxy` or something like it, but probably not worth the extra complexity.
( let [ basis-method ( .getMethod aclass "getBasis" nil )
basis ( .invoke basis-method nil nil ) ]
( -run!
( fn [ b ]
( let [ ^ Field cfield ( .getField aclass ( munge-cached ( name b ) ) ) ]
( let [ fvalue ( .get cfield x ) ]
( -freeze-without-meta! fvalue out ) ) ) )
basis ) ) ) ) )
2020-07-24 17:38:16 +00:00
2020-10-24 10:27:53 +00:00
( enc/compile-if java.time.Instant
( id-freezer java.time.Instant id-time-instant
( .writeLong out ( .getEpochSecond x ) )
( .writeInt out ( .getNano x ) ) )
nil )
2020-11-06 13:06:15 +00:00
( enc/compile-if java.time.Duration
( id-freezer java.time.Duration id-time-duration
( .writeLong out ( .getSeconds x ) )
( .writeInt out ( .getNano x ) ) )
nil )
2020-11-06 14:32:36 +00:00
( enc/compile-if java.time.Period
( id-freezer java.time.Period id-time-period
( .writeInt out ( .getYears x ) )
( .writeInt out ( .getMonths x ) )
( .writeInt out ( .getDays x ) ) )
nil )
2016-04-07 05:49:26 +00:00
( freezer Object
( when-debug ( println ( str "freeze-fallback: " ( type x ) ) ) )
( if-let [ ff *freeze-fallback* ]
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
( if-not ( identical? ff :write-unfreezable )
( ff out x ) ; Modern approach with ff
( or ; Legacy approach with ff
2016-04-07 05:49:26 +00:00
( try-write-serializable out x )
( try-write-readable out x )
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
( write-unfreezable out x ) ) )
2016-04-16 04:24:57 +00:00
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
;; Without ff
2016-04-07 05:49:26 +00:00
( or
( try-write-serializable out x )
( try-write-readable out x )
2016-04-16 04:24:57 +00:00
2020-07-24 17:38:16 +00:00
( when-let [ fff *final-freeze-fallback* ] ( fff out x ) true ) ; Deprecated
2016-04-16 04:24:57 +00:00
2016-06-17 05:25:31 +00:00
( throw-unfreezable x ) ) ) )
2016-04-07 05:49:26 +00:00
;;;;
2013-08-07 09:19:11 +00:00
2013-06-13 15:40:44 +00:00
( def ^ :private head-meta-id ( reduce-kv # ( assoc %1 %3 %2 ) { } head-meta ) )
2014-04-05 11:30:28 +00:00
( def ^ :private get-head-ba
2022-06-23 08:57:03 +00:00
( enc/fmemoize
2016-04-07 05:49:26 +00:00
( fn [ head-meta ]
( when-let [ meta-id ( get head-meta-id ( assoc head-meta :version head-version ) ) ]
( enc/ba-concat head-sig ( byte-array [ meta-id ] ) ) ) ) ) )
2013-06-13 15:40:44 +00:00
2014-04-05 11:30:28 +00:00
( defn- wrap-header [ data-ba head-meta ]
( if-let [ head-ba ( get-head-ba head-meta ) ]
2015-09-29 07:30:25 +00:00
( enc/ba-concat head-ba data-ba )
2016-04-07 05:49:26 +00:00
( throw ( ex-info ( str "Unrecognized header meta: " head-meta )
2014-04-05 11:30:28 +00:00
{ :head-meta head-meta } ) ) ) )
2013-06-13 15:40:44 +00:00
2014-04-05 11:30:28 +00:00
( comment ( wrap-header ( .getBytes "foo" ) { :compressor-id :lz4
:encryptor-id nil } ) )
2013-06-13 05:12:28 +00:00
2020-07-25 08:19:00 +00:00
( defn- call-with-bindings
"Allow opts to override config bindings."
2020-09-10 16:42:25 +00:00
[ action opts f ]
2020-07-25 08:19:00 +00:00
( if ( empty? opts )
( f )
( let [ opt->bindings
( fn [ bindings id var ]
( let [ v ( get opts id :default ) ]
( if ( identical? v :default )
( do bindings )
( assoc bindings var v ) ) ) )
bindings
( -> nil
( opt->bindings :freeze-fallback # '*freeze-fallback* )
( opt->bindings :auto-freeze-compressor # '*auto-freeze-compressor* )
2020-07-25 08:15:27 +00:00
( opt->bindings :custom-readers # '*custom-readers* )
2020-09-10 16:42:25 +00:00
( opt->bindings :incl-metadata? # '*incl-metadata?* )
2023-08-01 12:46:22 +00:00
( opt->bindings :thaw-xform # '*thaw-xform* )
2020-09-10 16:42:25 +00:00
( opt->bindings :serializable-allowlist
( case action
:freeze # '*freeze-serializable-allowlist*
:thaw # '*thaw-serializable-allowlist* ) ) ) ]
2020-07-25 08:19:00 +00:00
( if-not bindings
( f ) ; Common case
( try
( push-thread-bindings bindings )
( f )
( finally
( pop-thread-bindings ) ) ) ) ) ) )
( comment
( enc/qb 1 e4
2020-09-10 16:42:25 +00:00
( call-with-bindings :freeze { } ( fn [ ] *freeze-fallback* ) )
( call-with-bindings :freeze { :freeze-fallback "foo" } ( fn [ ] *freeze-fallback* ) ) ) )
2020-07-25 08:19:00 +00:00
2016-04-14 03:43:09 +00:00
( defn fast-freeze
" Like ` freeze ` but :
- Writes data without a Nippy header
- Drops all support for compression and encryption
- Must be thawed with ` fast-thaw `
2020-07-25 08:19:00 +00:00
Equivalent to ( but a little faster than ) ` freeze ` with opts :
- :compressor nil
- :encryptor nil
- :no-header? true "
2016-04-14 05:02:27 +00:00
[ x ]
2020-09-10 16:42:25 +00:00
( let [ baos ( ByteArrayOutputStream. 64 )
dos ( DataOutputStream. baos ) ]
2023-07-31 14:43:09 +00:00
( with-cache ( -freeze-with-meta! x dos ) )
2020-09-10 16:42:25 +00:00
( .toByteArray baos ) ) )
2016-04-14 03:43:09 +00:00
2013-06-13 05:12:28 +00:00
( defn freeze
2022-07-19 07:19:37 +00:00
" Serializes arg ( any Clojure data type ) to a byte array.
To freeze custom types , extend the Clojure reader or see ` extend-freeze ` . "
2020-09-10 11:35:55 +00:00
2016-04-14 05:02:27 +00:00
( [ x ] ( freeze x nil ) )
2020-07-24 17:38:16 +00:00
( [ x { :as opts
2020-09-10 16:42:25 +00:00
:keys [ compressor encryptor password serializable-allowlist incl-metadata? ]
2016-04-14 05:02:27 +00:00
:or { compressor :auto
2020-09-10 16:42:25 +00:00
encryptor aes128-gcm-encryptor } } ]
2020-07-24 17:38:16 +00:00
2020-09-10 16:42:25 +00:00
( call-with-bindings :freeze opts
2020-07-24 17:38:16 +00:00
( fn [ ]
( let [ ;; Intentionally undocumented:
no-header? ( or ( get opts :no-header? )
2020-07-24 20:50:05 +00:00
( get opts :skip-header? ) )
2020-07-24 17:38:16 +00:00
encryptor ( when password encryptor )
baos ( ByteArrayOutputStream. 64 )
dos ( DataOutputStream. baos ) ]
( if ( and ( nil? compressor ) ( nil? encryptor ) )
2020-07-24 20:50:05 +00:00
( do ; Optimized case
( when-not no-header? ; Avoid `wrap-header`'s array copy:
2020-07-24 17:38:16 +00:00
( let [ head-ba ( get-head-ba { :compressor-id nil :encryptor-id nil } ) ]
( .write dos head-ba 0 4 ) ) )
2023-07-31 14:43:09 +00:00
( with-cache ( -freeze-with-meta! x dos ) )
2020-07-24 17:38:16 +00:00
( .toByteArray baos ) )
( do
2023-07-31 14:43:09 +00:00
( with-cache ( -freeze-with-meta! x dos ) )
2020-07-24 17:38:16 +00:00
( let [ ba ( .toByteArray baos )
compressor
( if ( identical? compressor :auto )
( if no-header?
lz4-compressor
( if-let [ fc *auto-freeze-compressor* ]
( fc ba )
;; Intelligently enable compression only if benefit
;; is likely to outweigh cost:
( when ( > ( alength ba ) 8192 ) lz4-compressor ) ) )
( if ( fn? compressor )
2020-07-24 20:50:05 +00:00
( compressor ba ) ; Assume compressor selector fn
compressor ; Assume compressor
2020-07-24 17:38:16 +00:00
) )
ba ( if compressor ( compress compressor ba ) ba )
ba ( if encryptor ( encrypt encryptor password ba ) ba ) ]
( if no-header?
ba
( wrap-header ba
2022-07-19 07:19:37 +00:00
{ :compressor-id ( when-let [ c compressor ] ( or ( compression/standard-header-ids ( compression/header-id c ) ) :else ) )
:encryptor-id ( when-let [ e encryptor ] ( or ( encryption/standard-header-ids ( encryption/header-id e ) ) :else ) ) } ) ) ) ) ) ) ) ) ) )
2012-07-06 19:12:59 +00:00
;;;; Thawing
2020-07-24 20:46:30 +00:00
( declare ^ :private read-bytes )
2023-07-31 21:13:19 +00:00
( defn- read-bytes-sm* [ ^ DataInput in ] ( read-bytes in ( read-sm-count* in ) ) )
( defn- read-bytes-sm [ ^ DataInput in ] ( read-bytes in ( read-sm-count in ) ) )
( defn- read-bytes-md [ ^ DataInput in ] ( read-bytes in ( read-md-count in ) ) )
( defn- read-bytes-lg [ ^ DataInput in ] ( read-bytes in ( read-lg-count in ) ) )
2020-07-24 12:20:11 +00:00
( defn- read-bytes
2020-07-24 20:46:30 +00:00
( [ ^ DataInput in len ] ( let [ ba ( byte-array len ) ] ( .readFully in ba 0 len ) ba ) )
( [ ^ DataInput in ]
2020-07-24 12:20:11 +00:00
( enc/case-eval ( .readByte in )
2022-07-19 07:19:37 +00:00
id-bytes-0 ( byte-array 0 )
id-bytes-sm ( read-bytes in ( read-sm-count in ) )
id-bytes-md ( read-bytes in ( read-md-count in ) )
id-bytes-lg ( read-bytes in ( read-lg-count in ) ) ) ) )
2015-09-29 07:36:23 +00:00
2023-07-31 21:13:19 +00:00
( defn- read-str-sm* [ ^ DataInput in ] ( String. ^ bytes ( read-bytes in ( read-sm-count* in ) ) StandardCharsets/UTF_8 ) )
( defn- read-str-sm [ ^ DataInput in ] ( String. ^ bytes ( read-bytes in ( read-sm-count in ) ) StandardCharsets/UTF_8 ) )
( defn- read-str-md [ ^ DataInput in ] ( String. ^ bytes ( read-bytes in ( read-md-count in ) ) StandardCharsets/UTF_8 ) )
( defn- read-str-lg [ ^ DataInput in ] ( String. ^ bytes ( read-bytes in ( read-lg-count in ) ) StandardCharsets/UTF_8 ) )
2020-07-24 20:50:05 +00:00
( defn- read-str
2023-01-25 14:24:50 +00:00
( [ ^ DataInput in len ] ( String. ^ bytes ( read-bytes in len ) StandardCharsets/UTF_8 ) )
2020-07-24 20:46:30 +00:00
( [ ^ DataInput in ]
( enc/case-eval ( .readByte in )
id-str-0 ""
2023-07-31 21:13:19 +00:00
id-str-sm* ( String. ^ bytes ( read-bytes in ( read-sm-count* in ) ) StandardCharsets/UTF_8 )
id-str-sm_ ( String. ^ bytes ( read-bytes in ( read-sm-count in ) ) StandardCharsets/UTF_8 )
id-str-md ( String. ^ bytes ( read-bytes in ( read-md-count in ) ) StandardCharsets/UTF_8 )
id-str-lg ( String. ^ bytes ( read-bytes in ( read-lg-count in ) ) StandardCharsets/UTF_8 ) ) ) )
2016-04-07 05:49:26 +00:00
2016-04-14 05:02:27 +00:00
( defn- read-biginteger [ ^ DataInput in ] ( BigInteger. ^ bytes ( read-bytes in ( .readInt in ) ) ) )
2015-09-29 07:36:23 +00:00
2016-04-07 05:49:26 +00:00
( defmacro ^ :private editable? [ coll ] ` ( instance? clojure.lang.IEditableCollection ~ coll ) )
2015-09-29 07:36:23 +00:00
2023-08-01 12:46:22 +00:00
( defn- xform* [ xform ] ( enc/catching-xform { :error/msg "Error thrown via `*thaw-xform*`" } xform ) )
( defn- transduce-thaw1 [ ^ DataInput in xform n init rf ]
( let [ rf ( if xform ( ( xform* xform ) rf ) rf ) ]
( rf ( enc/reduce-n ( fn [ acc _ ] ( rf acc ( thaw-from-in! in ) ) ) init n ) ) ) )
( defn- transduce-thaw2 [ ^ DataInput in xform n init rf2 rf1 ]
( if xform
( let [ rf ( ( xform* xform ) rf1 ) ] ( rf ( enc/reduce-n ( fn [ acc _ ] ( rf acc ( clojure.lang.MapEntry/create ( thaw-from-in! in ) ( thaw-from-in! in ) ) ) ) init n ) ) )
( let [ rf rf2 ] ( rf ( enc/reduce-n ( fn [ acc _ ] ( rf acc ( thaw-from-in! in ) ( thaw-from-in! in ) ) ) init n ) ) ) ) )
2016-04-07 05:49:26 +00:00
( defn- read-into [ to ^ DataInput in ^ long n ]
( if ( and ( editable? to ) ( > n 10 ) )
2023-08-01 12:46:22 +00:00
( transduce-thaw1 in *thaw-xform* n ( transient to ) ( fn rf ( [ x ] ( persistent! x ) ) ( [ acc x ] ( conj! acc x ) ) ) )
( transduce-thaw1 in *thaw-xform* n to ( fn rf ( [ x ] x ) ( [ acc x ] ( conj acc x ) ) ) ) ) )
( declare ^ :private read-kvs-into )
( defn- read-kvs-depr [ to ^ DataInput in ] ( read-kvs-into to in ( quot ( .readInt in ) 2 ) ) )
( defn- read-kvs-into [ to ^ DataInput in ^ long n ]
( if ( and ( editable? to ) ( > n 10 ) )
2015-09-29 07:36:23 +00:00
2023-08-01 12:46:22 +00:00
( transduce-thaw2 in *thaw-xform* n ( transient to )
( fn rf2 ( [ x ] ( persistent! x ) ) ( [ acc k v ] ( assoc! acc k v ) ) )
( fn rf1 ( [ x ] ( persistent! x ) ) ( [ acc kv ] ( assoc! acc ( key kv ) ( val kv ) ) ) ) )
( transduce-thaw2 in *thaw-xform* n to
( fn rf2 ( [ x ] x ) ( [ acc k v ] ( assoc acc k v ) ) )
( fn rf1 ( [ x ] x ) ( [ acc kv ] ( assoc acc ( key kv ) ( val kv ) ) ) ) ) ) )
2015-09-29 07:36:23 +00:00
2020-07-24 17:38:16 +00:00
( defn- read-objects [ ^ objects ary ^ DataInput in ]
( enc/reduce-n
( fn [ ^ objects ary i ]
( aset ary i ( thaw-from-in! in ) )
ary )
ary ( alength ary ) ) )
2015-05-29 07:13:35 +00:00
( def ^ :private class-method-sig ( into-array Class [ IPersistentMap ] ) )
2013-08-02 08:20:14 +00:00
2016-04-13 17:13:33 +00:00
( defn- read-custom! [ in prefixed? type-id ]
2015-06-01 04:07:50 +00:00
( if-let [ custom-reader ( get *custom-readers* type-id ) ]
2014-07-04 13:05:18 +00:00
( try
( custom-reader in )
( catch Exception e
( throw
( ex-info
2016-04-13 17:13:33 +00:00
( str "Reader exception for custom type id: " type-id )
2022-07-19 07:19:37 +00:00
{ :type-id type-id , :prefixed? prefixed? } e ) ) ) )
2014-07-04 13:05:18 +00:00
( throw
( ex-info
2016-04-13 17:13:33 +00:00
( str "No reader provided for custom type id: " type-id )
2022-07-19 07:19:37 +00:00
{ :type-id type-id , :prefixed? prefixed? } ) ) ) )
2012-12-04 06:16:29 +00:00
2016-04-07 05:49:26 +00:00
( defn- read-edn [ edn ]
( try
( enc/read-edn { :readers *data-readers* } edn )
( catch Exception e
2020-09-10 10:08:28 +00:00
{ :nippy/unthawable
{ :type :reader
:cause :exception
:content edn
:exception e } } ) ) )
2016-04-07 05:49:26 +00:00
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
( defn- read-object [ ^ DataInput in class-name ]
2016-04-07 05:49:26 +00:00
( try
( let [ content ( .readObject ( ObjectInputStream. in ) ) ]
( try
( let [ class ( Class/forName class-name ) ] ( cast class content ) )
( catch Exception e
2020-09-10 10:08:28 +00:00
{ :nippy/unthawable
{ :type :serializable
:cause :exception
:class-name class-name
:content content
:exception e } } ) ) )
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2016-04-07 05:49:26 +00:00
( catch Exception e
2020-09-10 10:08:28 +00:00
{ :nippy/unthawable
{ :type :serializable
:cause :exception
:class-name class-name
:content nil
:exception e } } ) ) )
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2020-09-10 11:47:23 +00:00
( defn read-quarantined-serializable-object-unsafe!
" Given a quarantined Serializable object like
{ :nippy/unthawable { :class-name <> :content <quarantined-ba> } } , reads and
2020-09-10 16:42:25 +00:00
returns the object WITHOUT regard for ` *thaw-serializable-allowlist* ` .
2020-09-10 11:47:23 +00:00
**MAY BE UNSAFE!** Don 't call this unless you absolutely trust the payload
to not contain any malicious code.
2020-09-10 16:42:25 +00:00
See ` *thaw-serializable-allowlist* ` for more info. "
2020-09-10 11:47:23 +00:00
[ m ]
( when-let [ m ( get m :nippy/unthawable ) ]
( let [ { :keys [ class-name content ] } m ]
( when ( and class-name content )
( read-object
( DataInputStream. ( ByteArrayInputStream. content ) )
class-name ) ) ) ) )
( comment
( read-quarantined-serializable-object-unsafe!
( thaw ( freeze ( java.util.concurrent.Semaphore. 1 ) ) ) ) )
2022-10-13 16:52:19 +00:00
( defn- read-sz-quarantined
2020-07-24 20:50:05 +00:00
" Quarantined => object serialized to ba , then ba written to output stream.
2020-09-10 16:42:25 +00:00
Has length prefix => can skip ` readObject ` in event of allowlist failure. "
2020-07-24 20:50:05 +00:00
[ ^ DataInput in class-name ]
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
( let [ quarantined-ba ( read-bytes in ) ]
2020-09-10 16:42:25 +00:00
( if ( thaw-serializable-allowed? class-name )
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
( read-object ( DataInputStream. ( ByteArrayInputStream. quarantined-ba ) ) class-name )
2020-09-10 10:08:28 +00:00
{ :nippy/unthawable
{ :type :serializable
:cause :quarantined
:class-name class-name
:content quarantined-ba } } ) ) )
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
2022-10-13 16:52:19 +00:00
( defn- read-sz-unquarantined
2020-07-24 20:50:05 +00:00
" Unquarantined => object serialized directly to output stream.
2020-09-10 16:42:25 +00:00
No length prefix => cannot skip ` readObject ` in event of allowlist failure. "
2020-07-24 20:50:05 +00:00
[ ^ DataInput in class-name ]
2020-09-10 16:42:25 +00:00
( if ( thaw-serializable-allowed? class-name )
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
( read-object in class-name )
( throw ; No way to skip bytes, so best we can do is throw
2020-09-15 07:19:10 +00:00
( ex-info "Cannot thaw object: `taoensso.nippy/*thaw-serializable-allowlist*` check failed. This is a security feature. See `*thaw-serializable-allowlist*` docstring or https://github.com/ptaoussanis/nippy/issues/130 for details!"
[BREAKING] [Security] Fix RCE vulnerability
Fix a Remote Code Execution (RCE) vulnerability identified in an
excellent report by Timo Mihaljov (@solita-timo-mihaljov).
You are vulnerable iff both:
1. You are using Nippy to serialize and deserialize data from an
UNTRUSTED SOURCE.
2. You have a vulnerable ("gadget") class on your classpath.
Notably Clojure <= 1.8 includes such a class [1].
Many other libraries do too, some examples at [2].
To prevent this risk, a Serialization whitelist has been added.
Any classes not *explicitly* authorized by the whitelist to use
Serialization will NOT be permitted to.
The default whitelist is EMPTY, meaning this is a BREAKING
change iff you make use of Nippy's Serialization support. In
this case, you'll need to update the whitelist for your needs.
For more info see the `*serializable-whitelist*` docstring.
[1] https://clojure.atlassian.net/browse/CLJ-2204
[2] https://github.com/frohoff/ysoserial
Further info below provided by Timo:
------------------------------------
Deserialization vulnerabilities are exploited by constructing objects of classes
whose constructors perform some action that's useful to the attacker. A class like
this is called a gadget, and a collection of such classes that can be combined to
reach the attacker's goal is called a gadget chain.
There are three prerequisites for exploiting a deserialization vulnerability:
1) The attacker must be able to control the deserialized data, for example,
by gaining write access to the data store where trusted parties serialize
data or by exploiting some other vulnerability on the other end of a
communications channel.
2) The deserializer must construct objects of classes specified in the
serialized data. In other words, the attacker must have full control over
which classes get instantiated.
3) The classpath must contain gadgets that can be combined into a gadget chain.
The vulnerable code is in Nippy's function `read-serializable`, which calls the
`readObject` method of `ObjectInputStream`.
I have only tested the PoC with the latest stable version, 2.14.0, but looking at
Nippy's Git history, I believe all versions starting with the following commit
are vulnerable:
commit 9448d2b3cec5c28a1c7594dec00c01d66bb61a8b
[Thu Oct 24 13:47:25 2013 +0700]
For a user to be affected, they must:
1) use Nippy to serialize untrusted input, and
2) have a gadget chain on their classpath.
I suspect (but haven't verified) that using Nippy's encryption feature prevents
exploitation in some cases, but if it's used to encrypt the communications between
two systems, one compromised endpoint could send encrypted but
attacker-controlled data to the other.
Ysoserial [4] contains a list of some Java libraries with known gadget chains.
If any of those libraries can be found on the user's classpath, they are known
to be vulnerable. (Ysoserial's list is not exhaustive, so even if a user doesn't
have these particular libraries on their classpath, they may still have some
other gadget chains loaded.)
Unfortunately Clojure versions before 1.9 contained a gadget chain in the
standard library [5][6], so all Nippy users running Clojure 1.8 or earlier
are vulnerable. (Note that users of later Clojure versions may or may not
be vulnerable, depending on whether they have gadget chains from other
libraries on their classpath.)
[4] https://github.com/frohoff/ysoserial
[5] https://groups.google.com/forum/#!msg/clojure/WaL3hHzsevI/7zHU-L7LBQAJ
[6] https://clojure.atlassian.net/browse/CLJ-2204
2020-07-23 10:33:05 +00:00
{ :class-name class-name } ) ) ) )
2016-04-07 05:49:26 +00:00
( defn- read-record [ in class-name ]
( let [ content ( thaw-from-in! in ) ]
( try
2017-03-07 15:08:16 +00:00
( let [ class ( clojure.lang.RT/classForName class-name )
2016-04-07 05:49:26 +00:00
method ( .getMethod class "create" class-method-sig ) ]
( .invoke method class ( into-array Object [ content ] ) ) )
( catch Exception e
2020-09-10 10:08:28 +00:00
{ :nippy/unthawable
{ :type :record
:cause :exception
:class-name class-name
:content content
:exception e } } ) ) ) )
2016-04-07 05:49:26 +00:00
2020-07-24 17:38:16 +00:00
( defn- read-type [ in class-name ]
( try
( let [ aclass ( clojure.lang.RT/classForName class-name )
nbasis
( let [ basis-method ( .getMethod aclass "getBasis" nil )
basis ( .invoke basis-method nil nil ) ]
( count basis ) )
cvalues ( object-array nbasis ) ]
( enc/reduce-n
( fn [ _ i ] ( aset cvalues i ( thaw-from-in! in ) ) )
nil nbasis )
( let [ ctors ( .getConstructors aclass )
^ Constructor ctor ( aget ctors 0 ) ; Impl. detail? Ref. https://goo.gl/XWmckR
]
( .newInstance ctor cvalues ) ) )
( catch Exception e
2020-09-10 10:08:28 +00:00
{ :nippy/unthawable
{ :type :type
:cause :exception
:class-name class-name
:exception e } } ) ) )
2020-07-24 17:38:16 +00:00
2015-09-29 07:36:23 +00:00
( defn thaw-from-in!
" Deserializes a frozen object from given DataInput to its original Clojure
2016-07-18 04:30:38 +00:00
data type.
This is a low-level util : in most cases you 'll want ` thaw ` instead. "
2015-09-29 07:36:23 +00:00
[ ^ DataInput data-input ]
( let [ in data-input
type-id ( .readByte in ) ]
2022-07-19 07:19:37 +00:00
2016-04-07 05:49:26 +00:00
( when-debug ( println ( str "thaw-from-in!: " type-id ) ) )
2013-12-06 11:40:13 +00:00
( try
2015-09-29 07:30:25 +00:00
( enc/case-eval type-id
2013-12-06 11:40:13 +00:00
2020-07-24 20:50:05 +00:00
id-reader-sm ( read-edn ( read-str in ( read-sm-count in ) ) )
id-reader-md ( read-edn ( read-str in ( read-md-count in ) ) )
id-reader-lg ( read-edn ( read-str in ( read-lg-count in ) ) )
2022-10-13 16:52:19 +00:00
id-reader-lg_ ( read-edn ( read-str in ( read-lg-count in ) ) )
2020-07-24 20:50:05 +00:00
id-record-sm ( read-record in ( read-str in ( read-sm-count in ) ) )
id-record-md ( read-record in ( read-str in ( read-md-count in ) ) )
2022-07-19 07:19:37 +00:00
id-record-lg_ ( read-record in ( read-str in ( read-lg-count in ) ) )
2020-07-24 20:50:05 +00:00
2022-10-13 16:52:19 +00:00
id-sz-quarantined-sm ( read-sz-quarantined in ( read-str in ( read-sm-count in ) ) )
id-sz-quarantined-md ( read-sz-quarantined in ( read-str in ( read-md-count in ) ) )
2020-07-24 20:50:05 +00:00
2022-10-13 16:52:19 +00:00
id-sz-unquarantined-sm_ ( read-sz-unquarantined in ( read-str in ( read-sm-count in ) ) )
id-sz-unquarantined-md_ ( read-sz-unquarantined in ( read-str in ( read-md-count in ) ) )
id-sz-unquarantined-lg_ ( read-sz-unquarantined in ( read-str in ( read-lg-count in ) ) )
2016-04-07 05:49:26 +00:00
2020-07-24 17:38:16 +00:00
id-type ( read-type in ( thaw-from-in! in ) )
2016-04-07 05:49:26 +00:00
id-nil nil
id-true true
id-false false
id-char ( .readChar in )
id-meta ( let [ m ( thaw-from-in! in ) ]
2020-07-25 08:15:27 +00:00
( if *incl-metadata?*
( with-meta ( thaw-from-in! in ) m )
( do ( thaw-from-in! in ) ) ) )
2016-04-07 05:49:26 +00:00
2023-07-31 14:43:09 +00:00
id-cached-0 ( thaw-cached 0 in )
id-cached-1 ( thaw-cached 1 in )
id-cached-2 ( thaw-cached 2 in )
id-cached-3 ( thaw-cached 3 in )
id-cached-4 ( thaw-cached 4 in )
id-cached-5 ( thaw-cached 5 in )
id-cached-6 ( thaw-cached 6 in )
id-cached-7 ( thaw-cached 7 in )
id-cached-sm ( thaw-cached ( read-sm-count in ) in )
id-cached-md ( thaw-cached ( read-md-count in ) in )
2016-04-12 17:52:15 +00:00
2016-04-07 05:49:26 +00:00
id-bytes-0 ( byte-array 0 )
2016-04-13 04:57:50 +00:00
id-bytes-sm ( read-bytes in ( read-sm-count in ) )
id-bytes-md ( read-bytes in ( read-md-count in ) )
id-bytes-lg ( read-bytes in ( read-lg-count in ) )
2016-04-07 05:49:26 +00:00
2020-07-24 17:38:16 +00:00
id-objects-lg ( read-objects ( object-array ( read-lg-count in ) ) in )
2016-04-07 05:49:26 +00:00
id-str-0 ""
2023-07-31 21:13:19 +00:00
id-str-sm* ( read-str in ( read-sm-count* in ) )
id-str-sm_ ( read-str in ( read-sm-count in ) )
id-str-md ( read-str in ( read-md-count in ) )
id-str-lg ( read-str in ( read-lg-count in ) )
2020-07-24 20:50:05 +00:00
id-kw-sm ( keyword ( read-str in ( read-sm-count in ) ) )
id-kw-md ( keyword ( read-str in ( read-md-count in ) ) )
2022-10-13 16:52:19 +00:00
id-kw-md_ ( keyword ( read-str in ( read-lg-count in ) ) )
2022-07-19 07:19:37 +00:00
id-kw-lg_ ( keyword ( read-str in ( read-lg-count in ) ) )
2020-07-24 20:50:05 +00:00
2022-10-13 16:52:19 +00:00
id-sym-sm ( symbol ( read-str in ( read-sm-count in ) ) )
id-sym-md ( symbol ( read-str in ( read-md-count in ) ) )
id-sym-md_ ( symbol ( read-str in ( read-lg-count in ) ) )
2022-07-19 07:19:37 +00:00
id-sym-lg_ ( symbol ( read-str in ( read-lg-count in ) ) )
2022-10-13 16:52:19 +00:00
id-regex ( re-pattern ( thaw-from-in! in ) )
2016-04-07 05:49:26 +00:00
id-vec-0 [ ]
2023-08-01 12:46:22 +00:00
id-vec-2 ( read-into [ ] in 2 )
id-vec-3 ( read-into [ ] in 3 )
2023-07-31 21:13:19 +00:00
id-vec-sm* ( read-into [ ] in ( read-sm-count* in ) )
id-vec-sm_ ( read-into [ ] in ( read-sm-count in ) )
id-vec-md ( read-into [ ] in ( read-md-count in ) )
id-vec-lg ( read-into [ ] in ( read-lg-count in ) )
2016-04-07 05:49:26 +00:00
id-set-0 # { }
2023-07-31 21:13:19 +00:00
id-set-sm* ( read-into # { } in ( read-sm-count* in ) )
id-set-sm_ ( read-into # { } in ( read-sm-count in ) )
id-set-md ( read-into # { } in ( read-md-count in ) )
id-set-lg ( read-into # { } in ( read-lg-count in ) )
2016-04-07 05:49:26 +00:00
id-map-0 { }
2023-07-31 21:13:19 +00:00
id-map-sm* ( read-kvs-into { } in ( read-sm-count* in ) )
id-map-sm_ ( read-kvs-into { } in ( read-sm-count in ) )
id-map-md ( read-kvs-into { } in ( read-md-count in ) )
id-map-lg ( read-kvs-into { } in ( read-lg-count in ) )
2016-04-07 05:49:26 +00:00
2022-10-13 16:52:19 +00:00
id-queue-lg ( read-into ( PersistentQueue/EMPTY ) in ( read-lg-count in ) )
id-sorted-set-lg ( read-into ( sorted-set ) in ( read-lg-count in ) )
id-sorted-map-lg ( read-kvs-into ( sorted-map ) in ( read-lg-count in ) )
2016-04-07 05:49:26 +00:00
2022-07-19 07:19:37 +00:00
id-list-0 ' ( )
2016-04-13 04:57:50 +00:00
id-list-sm ( into ' ( ) ( rseq ( read-into [ ] in ( read-sm-count in ) ) ) )
id-list-md ( into ' ( ) ( rseq ( read-into [ ] in ( read-md-count in ) ) ) )
id-list-lg ( into ' ( ) ( rseq ( read-into [ ] in ( read-lg-count in ) ) ) )
2016-04-07 05:49:26 +00:00
id-seq-0 ( lazy-seq nil )
2016-04-13 04:57:50 +00:00
id-seq-sm ( or ( seq ( read-into [ ] in ( read-sm-count in ) ) ) ( lazy-seq nil ) )
id-seq-md ( or ( seq ( read-into [ ] in ( read-md-count in ) ) ) ( lazy-seq nil ) )
id-seq-lg ( or ( seq ( read-into [ ] in ( read-lg-count in ) ) ) ( lazy-seq nil ) )
2016-04-07 05:49:26 +00:00
id-byte ( .readByte in )
id-short ( .readShort in )
id-integer ( .readInt in )
2022-10-13 16:52:19 +00:00
id-long-0 0
[new] [Storage efficiency] PoC: separate signed long types
Before:
Longs in [ -128, 127] use 1 byte
Longs in [-32768, 32767] use 2 bytes
etc.
After:
Longs in [ -255, 255] use 1 byte
Longs in [-65535, 65535] use 2 bytes
etc.
I.e. doubles the range of longs that can be stored by 1, 2, and 4 bytes.
This changes saves:
- 1 byte per long in [ 128, 255], or [ -129, -255]
- 2 bytes per long in [32768, 65535], or [-32769, -65535]
- 4 bytes per long ...
Is this advantage worth the extra complexity? Probably yes, given how
common longs (and colls of longs) are in Clojure.
2023-07-31 20:37:08 +00:00
id-long-sm_ ( long ( .readByte in ) )
id-long-md_ ( long ( .readShort in ) )
id-long-lg_ ( long ( .readInt in ) )
2016-04-07 05:49:26 +00:00
id-long-xl ( .readLong in )
[new] [Storage efficiency] PoC: separate signed long types
Before:
Longs in [ -128, 127] use 1 byte
Longs in [-32768, 32767] use 2 bytes
etc.
After:
Longs in [ -255, 255] use 1 byte
Longs in [-65535, 65535] use 2 bytes
etc.
I.e. doubles the range of longs that can be stored by 1, 2, and 4 bytes.
This changes saves:
- 1 byte per long in [ 128, 255], or [ -129, -255]
- 2 bytes per long in [32768, 65535], or [-32769, -65535]
- 4 bytes per long ...
Is this advantage worth the extra complexity? Probably yes, given how
common longs (and colls of longs) are in Clojure.
2023-07-31 20:37:08 +00:00
id-long-pos-sm ( - ( long ( .readByte in ) ) Byte/MIN_VALUE )
id-long-pos-md ( - ( long ( .readShort in ) ) Short/MIN_VALUE )
id-long-pos-lg ( - ( long ( .readInt in ) ) Integer/MIN_VALUE )
id-long-neg-sm ( - ( - ( long ( .readByte in ) ) Byte/MIN_VALUE ) )
id-long-neg-md ( - ( - ( long ( .readShort in ) ) Short/MIN_VALUE ) )
id-long-neg-lg ( - ( - ( long ( .readInt in ) ) Integer/MIN_VALUE ) )
2016-04-07 05:49:26 +00:00
id-bigint ( bigint ( read-biginteger in ) )
id-biginteger ( read-biginteger in )
id-float ( .readFloat in )
2022-10-13 16:52:19 +00:00
id-double-0 0.0
2016-04-07 05:49:26 +00:00
id-double ( .readDouble in )
2016-04-14 05:02:27 +00:00
id-bigdec ( BigDecimal. ^ BigInteger ( read-biginteger in ) ( .readInt in ) )
2016-04-07 05:49:26 +00:00
id-ratio ( clojure.lang.Ratio.
( read-biginteger in )
( read-biginteger in ) )
2023-07-27 12:56:34 +00:00
id-util-date ( java.util.Date. ( .readLong in ) )
id-sql-date ( java.sql.Date. ( .readLong in ) )
2016-04-07 05:49:26 +00:00
id-uuid ( UUID. ( .readLong in ) ( .readLong in ) )
2022-10-13 16:52:19 +00:00
id-uri ( URI. ( thaw-from-in! in ) )
2016-04-07 05:49:26 +00:00
2020-10-24 10:27:53 +00:00
id-time-instant
2020-11-06 13:06:15 +00:00
( let [ secs ( .readLong in )
nanos ( .readInt in ) ]
( enc/compile-if java.time.Instant
( java.time.Instant/ofEpochSecond secs nanos )
{ :nippy/unthawable
{ :type :class
:cause :class-not-found
:class-name "java.time.Instant"
:content { :epoch-second secs :nano nanos } } } ) )
2020-10-24 14:53:02 +00:00
2020-11-06 13:06:15 +00:00
id-time-duration
( let [ secs ( .readLong in )
nanos ( .readInt in ) ]
( enc/compile-if java.time.Duration
( java.time.Duration/ofSeconds secs nanos )
{ :nippy/unthawable
{ :type :class
:cause :class-not-found
:class-name "java.time.Duration"
:content { :seconds secs :nanos nanos } } } ) )
2020-10-24 10:27:53 +00:00
2020-11-06 14:32:36 +00:00
id-time-period
( let [ years ( .readInt in )
months ( .readInt in )
days ( .readInt in ) ]
( enc/compile-if java.time.Period
( java.time.Period/of years months days )
{ :nippy/unthawable
{ :type :class
:cause :class-not-found
:class-name "java.time.Period"
:content { :years years :months months :days days } } } ) )
2016-04-07 05:49:26 +00:00
;; Deprecated ------------------------------------------------------
2022-10-13 16:52:19 +00:00
id-boolean_ ( .readBoolean in )
id-sorted-map_ ( read-kvs-depr ( sorted-map ) in )
id-map__ ( read-kvs-depr { } in )
id-reader_ ( read-edn ( .readUTF in ) )
id-str_ ( .readUTF in )
id-kw_ ( keyword ( .readUTF in ) )
id-map_ ( apply hash-map
( enc/repeatedly-into [ ] ( * 2 ( .readInt in ) )
( fn [ ] ( thaw-from-in! in ) ) ) )
2016-04-07 05:49:26 +00:00
;; -----------------------------------------------------------------
2022-10-13 16:52:19 +00:00
id-prefixed-custom-md ( read-custom! in :prefixed ( .readShort in ) )
2016-04-13 17:13:33 +00:00
( if ( neg? type-id )
( read-custom! in nil type-id ) ; Unprefixed custom type
( throw
( ex-info
( str "Unrecognized type id (" type-id "). Data frozen with newer Nippy version?" )
{ :type-id type-id } ) ) ) )
2013-12-06 11:40:13 +00:00
2022-07-19 07:19:37 +00:00
( catch Throwable t
( throw
( ex-info ( str "Thaw failed against type-id: " type-id )
{ :type-id type-id } t ) ) ) ) ) )
2012-07-06 19:12:59 +00:00
2016-07-16 12:09:38 +00:00
( let [ head-sig head-sig ] ; Not ^:const
( defn- try-parse-header [ ^ bytes ba ]
( let [ len ( alength ba ) ]
( when ( > len 4 )
( let [ -head-sig ( java.util.Arrays/copyOf ba 3 ) ]
( when ( java.util.Arrays/equals -head-sig ^ bytes head-sig )
;; Header appears to be well-formed
( let [ meta-id ( aget ba 3 )
data-ba ( java.util.Arrays/copyOfRange ba 4 len ) ]
[ data-ba ( get head-meta meta-id { :unrecognized-meta? true } ) ] ) ) ) ) ) ) )
2014-04-05 11:30:28 +00:00
( defn- get-auto-compressor [ compressor-id ]
( case compressor-id
nil nil
:snappy snappy-compressor
:lzma2 lzma2-compressor
:lz4 lz4-compressor
:no-header ( throw ( ex-info ":auto not supported on headerless data." { } ) )
2020-07-24 17:38:16 +00:00
:else ( throw ( ex-info ":auto not supported for non-standard compressors." { } ) )
( do ( throw ( ex-info ( str "Unrecognized :auto compressor id: " compressor-id )
{ :compressor-id compressor-id } ) ) ) ) )
2014-04-05 11:30:28 +00:00
( defn- get-auto-encryptor [ encryptor-id ]
( case encryptor-id
2020-07-24 17:38:16 +00:00
nil nil
:aes128-gcm-sha512 aes128-gcm-encryptor
:aes128-cbc-sha512 aes128-cbc-encryptor
:no-header ( throw ( ex-info ":auto not supported on headerless data." { } ) )
:else ( throw ( ex-info ":auto not supported for non-standard encryptors." { } ) )
( do ( throw ( ex-info ( str "Unrecognized :auto encryptor id: " encryptor-id )
{ :encryptor-id encryptor-id } ) ) ) ) )
2013-06-13 15:40:44 +00:00
2023-08-01 12:46:22 +00:00
( def ^ :private err-msg-unknown-thaw-failure "Possible decryption/decompression error, unfrozen/damaged data, etc." )
2022-07-19 07:19:37 +00:00
( def ^ :private err-msg-unrecognized-header "Unrecognized (but apparently well-formed) header. Data frozen with newer Nippy version?" )
2015-10-06 06:32:43 +00:00
2016-04-14 03:43:09 +00:00
( defn fast-thaw
" Like ` thaw ` but :
- Drops all support for compression and encryption
- Supports only data frozen with ` fast-freeze `
2020-07-25 08:19:00 +00:00
Equivalent to ( but a little faster than ) ` thaw ` with opts :
- :compressor nil
- :encryptor nil
- :no-header? true "
2016-04-14 03:43:09 +00:00
[ ^ bytes ba ]
( let [ dis ( DataInputStream. ( ByteArrayInputStream. ba ) ) ]
2016-04-14 04:23:31 +00:00
( with-cache ( thaw-from-in! dis ) ) ) )
2016-04-14 03:43:09 +00:00
2013-06-13 05:12:28 +00:00
( defn thaw
2016-03-10 13:55:51 +00:00
" Deserializes a frozen Nippy byte array to its original Clojure data type.
To thaw custom types , extend the Clojure reader or see ` extend-thaw ` .
2015-10-06 06:12:25 +00:00
** By default , supports data frozen with Nippy v2+ ONLY **
Add ` { :v1-compatibility? true } ` option to support thawing of data frozen with
legacy versions of Nippy.
2014-04-05 11:30:28 +00:00
Options include :
2015-10-06 06:12:25 +00:00
:v1-compatibility? - support data frozen by legacy versions of Nippy?
2015-10-06 08:04:19 +00:00
:compressor - :auto ( checks header , default ) an ICompressor , or nil
:encryptor - :auto ( checks header , default ) , an IEncryptor , or nil "
2015-09-26 04:31:49 +00:00
( [ ba ] ( thaw ba nil ) )
( [ ^ bytes ba
2020-07-24 17:38:16 +00:00
{ :as opts
2020-07-25 08:19:00 +00:00
:keys [ v1-compatibility? compressor encryptor password
2023-08-01 12:46:22 +00:00
serializable-allowlist incl-metadata? thaw-xform ]
2015-10-06 06:12:25 +00:00
:or { compressor :auto
2020-07-24 17:38:16 +00:00
encryptor :auto } } ]
2015-09-26 04:31:49 +00:00
2016-07-16 12:09:38 +00:00
( assert ( not ( get opts :headerless-meta ) )
2015-09-26 04:31:49 +00:00
":headerless-meta `thaw` opt removed in Nippy v2.7+" )
2020-09-10 16:42:25 +00:00
( call-with-bindings :thaw opts
2020-07-24 17:38:16 +00:00
( fn [ ]
( let [ v2+? ( not v1-compatibility? )
no-header? ( get opts :no-header? ) ; Intentionally undocumented
2022-07-19 07:19:37 +00:00
ex
( fn ex
( [ msg ] ( ex nil msg ) )
( [ e msg ]
( throw
( ex-info ( str "Thaw failed. " msg )
2023-08-02 10:20:17 +00:00
{ :opts
( assoc opts
:compressor compressor
:encryptor encryptor )
:bindings
( enc/assoc-some { }
'*freeze-fallback* *freeze-fallback*
'*auto-freeze-compressor* *auto-freeze-compressor*
'*custom-readers* *custom-readers*
'*incl-metadata?* *incl-metadata?*
2023-08-01 12:46:22 +00:00
'*thaw-serializable-allowlist* *thaw-serializable-allowlist*
'*thaw-xform* *thaw-xform* ) }
2023-08-02 10:20:17 +00:00
2022-07-19 07:19:37 +00:00
e ) ) ) )
2020-07-24 17:38:16 +00:00
thaw-data
( fn [ data-ba compressor-id encryptor-id ex-fn ]
2022-07-19 07:19:37 +00:00
( let [ compressor ( if ( identical? compressor :auto ) ( get-auto-compressor compressor-id ) compressor )
encryptor ( if ( identical? encryptor :auto ) ( get-auto-encryptor encryptor-id ) encryptor ) ]
2020-07-24 17:38:16 +00:00
( when ( and encryptor ( not password ) )
( ex "Password required for decryption." ) )
( try
( let [ ba data-ba
ba ( if encryptor ( decrypt encryptor password ba ) ba )
ba ( if compressor ( decompress compressor ba ) ba )
dis ( DataInputStream. ( ByteArrayInputStream. ba ) ) ]
( with-cache ( thaw-from-in! dis ) ) )
( catch Exception e ( ex-fn e ) ) ) ) )
;; Hackish + can actually segfault JVM due to Snappy bug,
;; Ref. http://goo.gl/mh7Rpy - no better alternatives, unfortunately
thaw-v1-data
( fn [ data-ba ex-fn ]
( thaw-data data-ba :snappy nil
( fn [ _ ] ( thaw-data data-ba nil nil ( fn [ _ ] ( ex-fn nil ) ) ) ) ) ) ]
( if no-header?
( if v2+?
2022-07-19 07:19:37 +00:00
( thaw-data ba :no-header :no-header ( fn [ e ] ( ex e err-msg-unknown-thaw-failure ) ) )
( thaw-data ba :no-header :no-header ( fn [ e ] ( thaw-v1-data ba ( fn [ _ ] ( ex e err-msg-unknown-thaw-failure ) ) ) ) ) )
2020-07-24 17:38:16 +00:00
;; At this point we assume that we have a header iff we have v2+ data
( if-let [ [ data-ba { :keys [ compressor-id encryptor-id unrecognized-meta? ]
:as head-meta } ] ( try-parse-header ba ) ]
;; A well-formed header _appears_ to be present (it's possible though
;; unlikely that this is a fluke and data is actually headerless):
( if v2+?
( if unrecognized-meta?
( ex err-msg-unrecognized-header )
( thaw-data data-ba compressor-id encryptor-id
( fn [ e ] ( ex e err-msg-unknown-thaw-failure ) ) ) )
( if unrecognized-meta?
2022-07-19 07:19:37 +00:00
( thaw-v1-data ba ( fn [ _ ] ( ex err-msg-unrecognized-header ) ) )
( thaw-data data-ba compressor-id encryptor-id ( fn [ e ] ( thaw-v1-data ba ( fn [ _ ] ( ex e err-msg-unknown-thaw-failure ) ) ) ) ) ) )
2020-07-24 17:38:16 +00:00
;; Well-formed header definitely not present
( if v2+?
( ex err-msg-unknown-thaw-failure )
( thaw-v1-data ba ( fn [ _ ] ( ex err-msg-unknown-thaw-failure ) ) ) ) ) ) ) ) ) ) )
2015-10-06 06:32:43 +00:00
( comment
( thaw ( freeze "hello" ) )
2022-07-19 07:19:37 +00:00
( thaw ( freeze "hello" { :compressor nil } ) )
( thaw ( freeze "hello" { :password [ :salted "p" ] } ) ) ; ex: no pwd
2015-10-06 06:32:43 +00:00
( thaw ( freeze "hello" ) { :password [ :salted "p" ] } ) )
2012-07-06 19:12:59 +00:00
2013-08-02 08:20:14 +00:00
;;;; Custom types
2014-07-04 13:05:18 +00:00
( defn- assert-custom-type-id [ custom-type-id ]
( assert ( or ( keyword? custom-type-id )
( and ( integer? custom-type-id ) ( <= 1 custom-type-id 128 ) ) ) ) )
2014-07-06 06:47:38 +00:00
( defn- coerce-custom-type-id
2020-07-25 10:54:50 +00:00
" * +ive byte id -> -ive byte id ( for unprefixed custom types )
2022-07-19 07:19:37 +00:00
* Keyword id -> Short hash id ( for prefixed custom types ) "
2015-04-19 03:48:01 +00:00
[ custom-type-id ]
2014-07-04 13:05:18 +00:00
( assert-custom-type-id custom-type-id )
( if-not ( keyword? custom-type-id )
2015-04-19 03:48:01 +00:00
( int ( - ^ long custom-type-id ) )
2015-10-06 08:04:19 +00:00
( let [ ^ int hash-id ( hash custom-type-id )
2014-07-04 13:05:18 +00:00
short-hash-id ( if ( pos? hash-id )
( mod hash-id Short/MAX_VALUE )
( mod hash-id Short/MIN_VALUE ) ) ]
2022-07-19 07:19:37 +00:00
2014-07-04 13:05:18 +00:00
;; Make sure hash ids can't collide with byte ids (unlikely anyway):
2022-07-19 07:19:37 +00:00
( assert ( not ( <= Byte/MIN_VALUE short-hash-id -1 ) )
2014-07-04 13:05:18 +00:00
"Custom type id hash collision; please choose a different id" )
2020-07-25 10:54:50 +00:00
2014-07-04 13:05:18 +00:00
( int short-hash-id ) ) ) )
( comment ( coerce-custom-type-id 77 )
( coerce-custom-type-id :foo/bar ) )
2013-08-02 08:20:14 +00:00
( defmacro extend-freeze
2014-04-05 11:30:28 +00:00
" Extends Nippy to support freezing of a custom type ( ideally concrete ) with
2014-07-04 13:05:18 +00:00
given id of form :
2020-07-25 10:54:50 +00:00
* Keyword - 2 byte overhead , keywords hashed to 16 bit id
* ℕ ∈ [ 1 , 128 ] - 0 byte overhead
2014-07-04 13:05:18 +00:00
2016-07-16 12:09:38 +00:00
NB : be careful about extending to interfaces , Ref. http ://goo.gl/6gGRlU.
( defrecord MyRec [ data ] )
( extend-freeze MyRec :foo/my-type [ x data-output ] ; Keyword id
2014-07-04 13:05:18 +00:00
( .writeUTF [ data-output ] ( :data x ) ) )
;; or
2016-07-16 12:09:38 +00:00
( extend-freeze MyRec 1 [ x data-output ] ; Byte id
2014-01-22 07:14:26 +00:00
( .writeUTF [ data-output ] ( :data x ) ) ) "
2020-07-25 10:54:50 +00:00
2014-01-22 07:14:26 +00:00
[ type custom-type-id [ x out ] & body ]
2014-07-04 13:05:18 +00:00
( assert-custom-type-id custom-type-id )
2017-12-21 09:08:24 +00:00
( let [ write-id-form
( if ( keyword? custom-type-id )
;; Prefixed [const byte id][cust hash id][payload]:
2022-10-13 16:52:19 +00:00
` ( do ( write-id ~ out ~ id-prefixed-custom-md )
2017-12-21 09:08:24 +00:00
( .writeShort ~ out ~ ( coerce-custom-type-id custom-type-id ) ) )
;; Unprefixed [cust byte id][payload]:
` ( write-id ~ out ~ ( coerce-custom-type-id custom-type-id ) ) ) ]
` ( extend-type ~ type IFreezable1
( ~ '-freeze-without-meta! [ ~ x ~ ( with-meta out { :tag 'java.io.DataOutput } ) ]
~ write-id-form
~@ body ) ) ) )
2013-08-02 08:20:14 +00:00
( defmacro extend-thaw
2014-07-04 13:05:18 +00:00
" Extends Nippy to support thawing of a custom type with given id :
( extend-thaw :foo/my-type [ data-input ] ; Keyword id
2016-07-16 12:09:38 +00:00
( MyRec. ( .readUTF data-input ) ) )
2014-07-04 13:05:18 +00:00
;; or
( extend-thaw 1 [ data-input ] ; Byte id
2016-07-16 12:09:38 +00:00
( MyRec. ( .readUTF data-input ) ) ) "
2014-01-22 07:14:26 +00:00
[ custom-type-id [ in ] & body ]
2014-07-04 13:05:18 +00:00
( assert-custom-type-id custom-type-id )
2015-06-01 04:09:59 +00:00
` ( do
( when ( contains? *custom-readers* ~ ( coerce-custom-type-id custom-type-id ) )
2016-04-07 05:49:26 +00:00
( println ( str "Warning: resetting Nippy thaw for custom type with id: "
2015-06-01 04:09:59 +00:00
~ custom-type-id ) ) )
2020-09-11 08:22:31 +00:00
( alter-var-root # '*custom-readers*
2015-06-01 04:09:59 +00:00
( fn [ m# ]
( assoc m#
~ ( coerce-custom-type-id custom-type-id )
( fn [ ~ ( with-meta in { :tag 'java.io.DataInput } ) ]
~@ body ) ) ) ) ) )
( comment
2015-06-01 04:53:55 +00:00
*custom-readers*
2016-07-16 12:09:38 +00:00
( defrecord MyRec [ data ] )
( extend-freeze MyRec 1 [ x out ] ( .writeUTF out ( :data x ) ) )
( extend-thaw 1 [ in ] ( MyRec. ( .readUTF in ) ) )
( thaw ( freeze ( MyRec. "Joe" ) ) ) )
2013-08-02 08:20:14 +00:00
2013-06-12 18:14:46 +00:00
;;;; Stress data
2012-07-06 19:12:59 +00:00
2022-07-01 08:10:54 +00:00
( defrecord StressRecord [ my-data ] )
( deftype StressType [ my-data ]
Object ( equals [ a b ] ( = ( .-my-data a ) ( .-my-data ^ StressType b ) ) ) )
2020-09-11 08:22:31 +00:00
2015-09-28 09:25:43 +00:00
( def stress-data "Reference data used for tests & benchmarks"
2022-06-25 10:40:52 +00:00
{ :nil nil
:true true
:false false
:boxed-false ( Boolean. false )
2016-04-07 05:49:26 +00:00
:char \ಬ
:str-short "ಬಾ ಇಲ್ಲಿ ಸಂಭವಿಸ"
:str-long ( apply str ( range 1000 ) )
:kw :keyword
:kw-ns ::keyword
2020-11-18 09:46:37 +00:00
:kw-long ( keyword
( apply str "kw" ( range 1000 ) )
( apply str "kw" ( range 1000 ) ) )
2016-04-07 05:49:26 +00:00
:sym 'foo
:sym-ns 'foo/bar
2020-11-18 09:46:37 +00:00
:sym-long ( symbol
( apply str "sym" ( range 1000 ) )
( apply str "sym" ( range 1000 ) ) )
2016-04-07 05:49:26 +00:00
:regex # "^(https?:)?//(www\?|\?)?"
2015-09-29 09:02:46 +00:00
;;; Try reflect real-world data:
2022-07-01 08:10:54 +00:00
:many-small-numbers ( vec ( range 200 ) )
:many-small-keywords ( ->> ( java.util.Locale/getISOLanguages )
( mapv keyword ) )
:many-small-strings ( ->> ( java.util.Locale/getISOCountries )
( mapv # ( .getDisplayCountry ( java.util.Locale. "en" % ) ) ) )
2015-09-29 09:02:46 +00:00
2016-04-07 05:49:26 +00:00
:queue ( enc/queue [ :a :b :c :d :e :f :g ] )
:queue-empty ( enc/queue )
2015-09-29 09:02:46 +00:00
:sorted-set ( sorted-set 1 2 3 4 5 )
:sorted-map ( sorted-map :b 2 :a 1 :d 4 :c 3 )
2022-07-01 08:10:54 +00:00
:list ( list 1 2 3 4 5 ( list 6 7 8 ( list 9 10 ' ( ( ) ) ) ) )
:vector [ 1 2 3 4 5 [ 6 7 8 [ 9 10 [ [ ] ] ] ] ]
:map { :a 1 :b 2 :c 3 :d { :e 4 :f { :g 5 :h 6 :i 7 :j { { } { } } } } }
:set # { 1 2 3 4 5 # { 6 7 8 # { 9 10 # { # { } } } } }
2015-09-29 09:02:46 +00:00
:meta ( with-meta { :a :A } { :metakey :metaval } )
2022-07-01 08:10:54 +00:00
:nested [ # { { 1 [ :a :b ] 2 [ :c :d ] 3 [ :e :f ] } [ # { { } } ] # { :a :b } }
# { { 1 [ :a :b ] 2 [ :c :d ] 3 [ :e :f ] } [ # { { } } ] # { :a :b } }
2016-04-07 05:49:26 +00:00
[ 1 [ 1 2 [ 1 2 3 [ 1 2 3 4 [ 1 2 3 4 5 ] ] ] ] ] ]
2015-09-29 09:02:46 +00:00
2016-04-07 05:49:26 +00:00
:lazy-seq ( repeatedly 1000 rand )
2015-09-29 09:02:46 +00:00
:lazy-seq-empty ( map identity ' ( ) )
2020-10-24 12:08:56 +00:00
:byte ( byte 16 )
:short ( short 42 )
:integer ( int 3 )
:long ( long 3 )
2015-09-29 09:02:46 +00:00
:bigint ( bigint 31415926535897932384626433832795 )
2020-10-24 12:08:56 +00:00
:float ( float 3.14 )
2015-09-29 09:02:46 +00:00
:double ( double 3.14 )
:bigdec ( bigdec 3.1415926535897932384626433832795 )
:ratio 22 /7
2022-07-19 07:19:37 +00:00
:uri ( java.net.URI. "https://clojure.org/reference/data_structures" )
2015-09-29 09:02:46 +00:00
:uuid ( java.util.UUID/randomUUID )
2023-07-27 12:56:34 +00:00
:util-date ( java.util.Date. )
:sql-date ( java.sql.Date/valueOf "2023-06-21" )
2020-11-06 13:06:15 +00:00
2020-11-06 14:32:36 +00:00
;;; JVM 8+
:time-instant ( enc/compile-if java.time.Instant ( java.time.Instant/now ) nil )
:time-duration ( enc/compile-if java.time.Duration ( java.time.Duration/ofSeconds 100 100 ) nil )
:time-period ( enc/compile-if java.time.Period ( java.time.Period/of 1 1 1 ) nil )
2020-11-06 13:06:15 +00:00
2022-06-25 10:40:52 +00:00
:bytes ( byte-array [ ( byte 1 ) ( byte 2 ) ( byte 3 ) ] )
2020-10-24 10:27:53 +00:00
:objects ( object-array [ 1 "two" { :data "data" } ] )
2015-09-29 09:02:46 +00:00
2016-06-10 04:18:55 +00:00
:stress-record ( StressRecord. "data" )
2020-10-24 12:08:56 +00:00
:stress-type ( StressType. "data" )
2015-09-29 09:02:46 +00:00
;; Serializable
:throwable ( Throwable. "Yolo" )
:exception ( try ( / 1 0 ) ( catch Exception e e ) )
:ex-info ( ex-info "ExInfo" { :data "data" } ) } )
2013-06-12 18:14:46 +00:00
2014-01-21 07:33:35 +00:00
( def stress-data-comparable
2023-08-01 10:19:25 +00:00
"Reference data with stuff removed that breaks roundtrip equality."
( dissoc stress-data :bytes :objects :throwable :exception :ex-info :regex ) )
( comment ( let [ data stress-data-comparable ] ( = ( thaw ( freeze data ) ) data ) ) )
2014-01-21 07:33:35 +00:00
( def stress-data-benchable
" Reference data with stuff removed that breaks reader or other utils we 'll
2023-08-01 10:19:25 +00:00
be benching with. "
( dissoc stress-data-comparable
:queue :queue-empty
:stress-record :stress-type
:time-instant :time-duration :time-period
:byte :uri ) )
2014-01-21 07:33:35 +00:00
2014-02-14 16:06:53 +00:00
;;;; Tools
2022-07-19 07:19:37 +00:00
( defn inspect-ba
"Experimental, subject to change. Feedback welcome."
2015-09-29 16:06:33 +00:00
( [ ba ] ( inspect-ba ba nil ) )
( [ ba thaw-opts ]
( when ( enc/bytes? ba )
( let [ [ first2bytes nextbytes ] ( enc/ba-split ba 2 )
?known-wrapper
2020-07-24 20:50:05 +00:00
( enc/cond
2023-01-25 14:24:50 +00:00
( enc/ba= first2bytes ( .getBytes "\u0000<" StandardCharsets/UTF_8 ) ) :carmine/bin
( enc/ba= first2bytes ( .getBytes "\u0000>" StandardCharsets/UTF_8 ) ) :carmine/clj )
2015-09-29 16:06:33 +00:00
unwrapped-ba ( if ?known-wrapper nextbytes ba )
[ data-ba ?nippy-header ] ( or ( try-parse-header unwrapped-ba )
[ unwrapped-ba :no-header ] ) ]
{ :?known-wrapper ?known-wrapper
:?header ?nippy-header
:thawable? ( try ( thaw unwrapped-ba thaw-opts ) true
( catch Exception _ false ) )
:unwrapped-ba unwrapped-ba
:data-ba data-ba
:unwrapped-len ( alength ^ bytes unwrapped-ba )
:ba-len ( alength ^ bytes ba )
:data-len ( alength ^ bytes data-ba ) } ) ) ) )
( comment
2022-07-19 07:19:37 +00:00
( do ( inspect-ba ( freeze "hello" ) ) )
2015-09-29 16:06:33 +00:00
( seq ( :data-ba ( inspect-ba ( freeze "hello" ) ) ) ) )
2016-04-16 04:24:57 +00:00
2020-07-24 17:38:16 +00:00
( defn freeze-to-string
" Convenience util : like ` freeze ` , but returns a Base64-encoded string.
See also ` thaw-from-string ` . "
( [ x ] ( freeze-to-string x nil ) )
( [ x freeze-opts ]
( let [ ba ( freeze x freeze-opts ) ]
( .encodeToString ( java.util.Base64/getEncoder )
ba ) ) ) )
( defn thaw-from-string
" Convenience util : like ` thaw ` , but takes a Base64-encoded string.
See also ` freeze-to-string ` . "
( [ s ] ( thaw-from-string s nil ) )
( [ ^ String s thaw-opts ]
( let [ ba ( .decode ( java.util.Base64/getDecoder ) s ) ]
( thaw ba thaw-opts ) ) ) )
2017-02-13 16:35:18 +00:00
2020-07-24 17:38:16 +00:00
( comment ( thaw-from-string ( freeze-to-string { :a :A :b [ :B1 :B2 ] } ) ) )
2017-02-13 16:35:18 +00:00
2020-07-24 17:38:16 +00:00
( defn freeze-to-file
2020-11-06 13:51:19 +00:00
"Convenience util: like `freeze`, but writes to `(clojure.java.io/file <file>)`."
2017-02-13 16:35:18 +00:00
( [ file x ] ( freeze-to-file file x nil ) )
( [ file x freeze-opts ]
( let [ ^ bytes ba ( freeze x freeze-opts ) ]
( with-open [ out ( jio/output-stream ( jio/file file ) ) ]
( .write out ba ) )
ba ) ) )
( defn thaw-from-file
2020-11-06 13:51:19 +00:00
"Convenience util: like `thaw`, but reads from `(clojure.java.io/file <file>)`."
2017-02-13 16:35:18 +00:00
( [ file ] ( thaw-from-file file nil ) )
( [ file thaw-opts ]
2020-11-06 13:51:19 +00:00
( let [ file ( jio/file file )
frozen-ba
( let [ ba ( byte-array ( .length file ) ) ]
( with-open [ in ( DataInputStream. ( jio/input-stream file ) ) ]
( .readFully in ba )
( do ba ) ) ) ]
( thaw frozen-ba thaw-opts ) ) ) )
( defn thaw-from-resource
"Convenience util: like `thaw`, but reads from `(clojure.java.io/resource <res>)`."
( [ res ] ( thaw-from-resource res nil ) )
( [ res thaw-opts ]
( let [ res ( jio/resource res )
frozen-ba
( with-open [ in ( jio/input-stream res )
out ( ByteArrayOutputStream. ) ]
( jio/copy in out )
( .toByteArray out ) ) ]
( thaw frozen-ba thaw-opts ) ) ) )
2017-02-13 16:35:18 +00:00
( comment
2020-11-06 13:51:19 +00:00
( freeze-to-file "resources/foo.npy" "hello, world!" )
( thaw-from-file "resources/foo.npy" )
( thaw-from-resource "foo.npy" ) )
2017-02-13 16:35:18 +00:00
2016-04-16 04:24:57 +00:00
;;;; Deprecated
2020-09-11 08:22:31 +00:00
( enc/deprecated
2022-07-19 07:19:37 +00:00
( def ^ :deprecated freeze-fallback-as-str "DEPRECATED, use `write-unfreezable`" write-unfreezable )
( defn ^ :deprecated set-freeze-fallback! "DEPRECATED, just use `alter-var-root`" [ x ] ( alter-var-root # '*freeze-fallback* ( constantly x ) ) )
( defn ^ :deprecated set-auto-freeze-compressor! "DEPRECATED, just use `alter-var-root`" [ x ] ( alter-var-root # '*auto-freeze-compressor* ( constantly x ) ) )
( defn ^ :deprecated swap-custom-readers! "DEPRECATED, just use `alter-var-root`" [ f ] ( alter-var-root # '*custom-readers* f ) )
( defn ^ :deprecated swap-serializable-whitelist!
2020-11-06 14:24:47 +00:00
" DEPRECATED , just use
( alter-var-root *thaw-serializable-allowlist* f ) and/or
( alter-var-root *freeze-serializable-allow-list* f ) instead. "
[ f ]
( alter-var-root *freeze-serializable-allowlist* ( fn [ old ] ( f ( enc/have set? old ) ) ) )
( alter-var-root *thaw-serializable-allowlist* ( fn [ old ] ( f ( enc/have set? old ) ) ) ) ) )
