1import builtins
2import collections
3import copyreg
4import dbm
5import io
6import functools
7import os
8import math
9import pickle
10import pickletools
11import shutil
12import struct
13import sys
14import threading
15import types
16import unittest
17import weakref
18from textwrap import dedent
19from http.cookies import SimpleCookie
20
21try:
22    import _testbuffer
23except ImportError:
24    _testbuffer = None
25
26from test import support
27from test.support import os_helper
28from test.support import (
29    TestFailed, run_with_locale, no_tracing,
30    _2G, _4G, bigmemtest
31    )
32from test.support.import_helper import forget
33from test.support.os_helper import TESTFN
34from test.support import threading_helper
35from test.support.warnings_helper import save_restore_warnings_filters
36
37from pickle import bytes_types
38
39
40# bpo-41003: Save/restore warnings filters to leave them unchanged.
41# Ignore filters installed by numpy.
42try:
43    with save_restore_warnings_filters():
44        import numpy as np
45except ImportError:
46    np = None
47
48
49requires_32b = unittest.skipUnless(sys.maxsize < 2**32,
50                                   "test is only meaningful on 32-bit builds")
51
52# Tests that try a number of pickle protocols should have a
53#     for proto in protocols:
54# kind of outer loop.
55protocols = range(pickle.HIGHEST_PROTOCOL + 1)
56
57
58# Return True if opcode code appears in the pickle, else False.
59def opcode_in_pickle(code, pickle):
60    for op, dummy, dummy in pickletools.genops(pickle):
61        if op.code == code.decode("latin-1"):
62            return True
63    return False
64
65# Return the number of times opcode code appears in pickle.
66def count_opcode(code, pickle):
67    n = 0
68    for op, dummy, dummy in pickletools.genops(pickle):
69        if op.code == code.decode("latin-1"):
70            n += 1
71    return n
72
73
74def identity(x):
75    return x
76
77
78class UnseekableIO(io.BytesIO):
79    def peek(self, *args):
80        raise NotImplementedError
81
82    def seekable(self):
83        return False
84
85    def seek(self, *args):
86        raise io.UnsupportedOperation
87
88    def tell(self):
89        raise io.UnsupportedOperation
90
91
92class MinimalIO(object):
93    """
94    A file-like object that doesn't support readinto().
95    """
96    def __init__(self, *args):
97        self._bio = io.BytesIO(*args)
98        self.getvalue = self._bio.getvalue
99        self.read = self._bio.read
100        self.readline = self._bio.readline
101        self.write = self._bio.write
102
103
104# We can't very well test the extension registry without putting known stuff
105# in it, but we have to be careful to restore its original state.  Code
106# should do this:
107#
108#     e = ExtensionSaver(extension_code)
109#     try:
110#         fiddle w/ the extension registry's stuff for extension_code
111#     finally:
112#         e.restore()
113
114class ExtensionSaver:
115    # Remember current registration for code (if any), and remove it (if
116    # there is one).
117    def __init__(self, code):
118        self.code = code
119        if code in copyreg._inverted_registry:
120            self.pair = copyreg._inverted_registry[code]
121            copyreg.remove_extension(self.pair[0], self.pair[1], code)
122        else:
123            self.pair = None
124
125    # Restore previous registration for code.
126    def restore(self):
127        code = self.code
128        curpair = copyreg._inverted_registry.get(code)
129        if curpair is not None:
130            copyreg.remove_extension(curpair[0], curpair[1], code)
131        pair = self.pair
132        if pair is not None:
133            copyreg.add_extension(pair[0], pair[1], code)
134
135class C:
136    def __eq__(self, other):
137        return self.__dict__ == other.__dict__
138
139class D(C):
140    def __init__(self, arg):
141        pass
142
143class E(C):
144    def __getinitargs__(self):
145        return ()
146
147# Simple mutable object.
148class Object:
149    pass
150
151# Hashable immutable key object containing unheshable mutable data.
152class K:
153    def __init__(self, value):
154        self.value = value
155
156    def __reduce__(self):
157        # Shouldn't support the recursion itself
158        return K, (self.value,)
159
160import __main__
161__main__.C = C
162C.__module__ = "__main__"
163__main__.D = D
164D.__module__ = "__main__"
165__main__.E = E
166E.__module__ = "__main__"
167
168class myint(int):
169    def __init__(self, x):
170        self.str = str(x)
171
172class initarg(C):
173
174    def __init__(self, a, b):
175        self.a = a
176        self.b = b
177
178    def __getinitargs__(self):
179        return self.a, self.b
180
181class metaclass(type):
182    pass
183
184class use_metaclass(object, metaclass=metaclass):
185    pass
186
187class pickling_metaclass(type):
188    def __eq__(self, other):
189        return (type(self) == type(other) and
190                self.reduce_args == other.reduce_args)
191
192    def __reduce__(self):
193        return (create_dynamic_class, self.reduce_args)
194
195def create_dynamic_class(name, bases):
196    result = pickling_metaclass(name, bases, dict())
197    result.reduce_args = (name, bases)
198    return result
199
200
201class ZeroCopyBytes(bytes):
202    readonly = True
203    c_contiguous = True
204    f_contiguous = True
205    zero_copy_reconstruct = True
206
207    def __reduce_ex__(self, protocol):
208        if protocol >= 5:
209            return type(self)._reconstruct, (pickle.PickleBuffer(self),), None
210        else:
211            return type(self)._reconstruct, (bytes(self),)
212
213    def __repr__(self):
214        return "{}({!r})".format(self.__class__.__name__, bytes(self))
215
216    __str__ = __repr__
217
218    @classmethod
219    def _reconstruct(cls, obj):
220        with memoryview(obj) as m:
221            obj = m.obj
222            if type(obj) is cls:
223                # Zero-copy
224                return obj
225            else:
226                return cls(obj)
227
228
229class ZeroCopyBytearray(bytearray):
230    readonly = False
231    c_contiguous = True
232    f_contiguous = True
233    zero_copy_reconstruct = True
234
235    def __reduce_ex__(self, protocol):
236        if protocol >= 5:
237            return type(self)._reconstruct, (pickle.PickleBuffer(self),), None
238        else:
239            return type(self)._reconstruct, (bytes(self),)
240
241    def __repr__(self):
242        return "{}({!r})".format(self.__class__.__name__, bytes(self))
243
244    __str__ = __repr__
245
246    @classmethod
247    def _reconstruct(cls, obj):
248        with memoryview(obj) as m:
249            obj = m.obj
250            if type(obj) is cls:
251                # Zero-copy
252                return obj
253            else:
254                return cls(obj)
255
256
257if _testbuffer is not None:
258
259    class PicklableNDArray:
260        # A not-really-zero-copy picklable ndarray, as the ndarray()
261        # constructor doesn't allow for it
262
263        zero_copy_reconstruct = False
264
265        def __init__(self, *args, **kwargs):
266            self.array = _testbuffer.ndarray(*args, **kwargs)
267
268        def __getitem__(self, idx):
269            cls = type(self)
270            new = cls.__new__(cls)
271            new.array = self.array[idx]
272            return new
273
274        @property
275        def readonly(self):
276            return self.array.readonly
277
278        @property
279        def c_contiguous(self):
280            return self.array.c_contiguous
281
282        @property
283        def f_contiguous(self):
284            return self.array.f_contiguous
285
286        def __eq__(self, other):
287            if not isinstance(other, PicklableNDArray):
288                return NotImplemented
289            return (other.array.format == self.array.format and
290                    other.array.shape == self.array.shape and
291                    other.array.strides == self.array.strides and
292                    other.array.readonly == self.array.readonly and
293                    other.array.tobytes() == self.array.tobytes())
294
295        def __ne__(self, other):
296            if not isinstance(other, PicklableNDArray):
297                return NotImplemented
298            return not (self == other)
299
300        def __repr__(self):
301            return (f"{type(self)}(shape={self.array.shape},"
302                    f"strides={self.array.strides}, "
303                    f"bytes={self.array.tobytes()})")
304
305        def __reduce_ex__(self, protocol):
306            if not self.array.contiguous:
307                raise NotImplementedError("Reconstructing a non-contiguous "
308                                          "ndarray does not seem possible")
309            ndarray_kwargs = {"shape": self.array.shape,
310                              "strides": self.array.strides,
311                              "format": self.array.format,
312                              "flags": (0 if self.readonly
313                                        else _testbuffer.ND_WRITABLE)}
314            pb = pickle.PickleBuffer(self.array)
315            if protocol >= 5:
316                return (type(self)._reconstruct,
317                        (pb, ndarray_kwargs))
318            else:
319                # Need to serialize the bytes in physical order
320                with pb.raw() as m:
321                    return (type(self)._reconstruct,
322                            (m.tobytes(), ndarray_kwargs))
323
324        @classmethod
325        def _reconstruct(cls, obj, kwargs):
326            with memoryview(obj) as m:
327                # For some reason, ndarray() wants a list of integers...
328                # XXX This only works if format == 'B'
329                items = list(m.tobytes())
330            return cls(items, **kwargs)
331
332
333# DATA0 .. DATA4 are the pickles we expect under the various protocols, for
334# the object returned by create_data().
335
336DATA0 = (
337    b'(lp0\nL0L\naL1L\naF2.0\n'
338    b'ac__builtin__\ncomple'
339    b'x\np1\n(F3.0\nF0.0\ntp2\n'
340    b'Rp3\naL1L\naL-1L\naL255'
341    b'L\naL-255L\naL-256L\naL'
342    b'65535L\naL-65535L\naL-'
343    b'65536L\naL2147483647L'
344    b'\naL-2147483647L\naL-2'
345    b'147483648L\na(Vabc\np4'
346    b'\ng4\nccopy_reg\n_recon'
347    b'structor\np5\n(c__main'
348    b'__\nC\np6\nc__builtin__'
349    b'\nobject\np7\nNtp8\nRp9\n'
350    b'(dp10\nVfoo\np11\nL1L\ns'
351    b'Vbar\np12\nL2L\nsbg9\ntp'
352    b'13\nag13\naL5L\na.'
353)
354
355# Disassembly of DATA0
356DATA0_DIS = """\
357    0: (    MARK
358    1: l        LIST       (MARK at 0)
359    2: p    PUT        0
360    5: L    LONG       0
361    9: a    APPEND
362   10: L    LONG       1
363   14: a    APPEND
364   15: F    FLOAT      2.0
365   20: a    APPEND
366   21: c    GLOBAL     '__builtin__ complex'
367   42: p    PUT        1
368   45: (    MARK
369   46: F        FLOAT      3.0
370   51: F        FLOAT      0.0
371   56: t        TUPLE      (MARK at 45)
372   57: p    PUT        2
373   60: R    REDUCE
374   61: p    PUT        3
375   64: a    APPEND
376   65: L    LONG       1
377   69: a    APPEND
378   70: L    LONG       -1
379   75: a    APPEND
380   76: L    LONG       255
381   82: a    APPEND
382   83: L    LONG       -255
383   90: a    APPEND
384   91: L    LONG       -256
385   98: a    APPEND
386   99: L    LONG       65535
387  107: a    APPEND
388  108: L    LONG       -65535
389  117: a    APPEND
390  118: L    LONG       -65536
391  127: a    APPEND
392  128: L    LONG       2147483647
393  141: a    APPEND
394  142: L    LONG       -2147483647
395  156: a    APPEND
396  157: L    LONG       -2147483648
397  171: a    APPEND
398  172: (    MARK
399  173: V        UNICODE    'abc'
400  178: p        PUT        4
401  181: g        GET        4
402  184: c        GLOBAL     'copy_reg _reconstructor'
403  209: p        PUT        5
404  212: (        MARK
405  213: c            GLOBAL     '__main__ C'
406  225: p            PUT        6
407  228: c            GLOBAL     '__builtin__ object'
408  248: p            PUT        7
409  251: N            NONE
410  252: t            TUPLE      (MARK at 212)
411  253: p        PUT        8
412  256: R        REDUCE
413  257: p        PUT        9
414  260: (        MARK
415  261: d            DICT       (MARK at 260)
416  262: p        PUT        10
417  266: V        UNICODE    'foo'
418  271: p        PUT        11
419  275: L        LONG       1
420  279: s        SETITEM
421  280: V        UNICODE    'bar'
422  285: p        PUT        12
423  289: L        LONG       2
424  293: s        SETITEM
425  294: b        BUILD
426  295: g        GET        9
427  298: t        TUPLE      (MARK at 172)
428  299: p    PUT        13
429  303: a    APPEND
430  304: g    GET        13
431  308: a    APPEND
432  309: L    LONG       5
433  313: a    APPEND
434  314: .    STOP
435highest protocol among opcodes = 0
436"""
437
438DATA1 = (
439    b']q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c__'
440    b'builtin__\ncomplex\nq\x01'
441    b'(G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00t'
442    b'q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ'
443    b'\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff'
444    b'\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00ab'
445    b'cq\x04h\x04ccopy_reg\n_reco'
446    b'nstructor\nq\x05(c__main'
447    b'__\nC\nq\x06c__builtin__\n'
448    b'object\nq\x07Ntq\x08Rq\t}q\n('
449    b'X\x03\x00\x00\x00fooq\x0bK\x01X\x03\x00\x00\x00bar'
450    b'q\x0cK\x02ubh\ttq\rh\rK\x05e.'
451)
452
453# Disassembly of DATA1
454DATA1_DIS = """\
455    0: ]    EMPTY_LIST
456    1: q    BINPUT     0
457    3: (    MARK
458    4: K        BININT1    0
459    6: K        BININT1    1
460    8: G        BINFLOAT   2.0
461   17: c        GLOBAL     '__builtin__ complex'
462   38: q        BINPUT     1
463   40: (        MARK
464   41: G            BINFLOAT   3.0
465   50: G            BINFLOAT   0.0
466   59: t            TUPLE      (MARK at 40)
467   60: q        BINPUT     2
468   62: R        REDUCE
469   63: q        BINPUT     3
470   65: K        BININT1    1
471   67: J        BININT     -1
472   72: K        BININT1    255
473   74: J        BININT     -255
474   79: J        BININT     -256
475   84: M        BININT2    65535
476   87: J        BININT     -65535
477   92: J        BININT     -65536
478   97: J        BININT     2147483647
479  102: J        BININT     -2147483647
480  107: J        BININT     -2147483648
481  112: (        MARK
482  113: X            BINUNICODE 'abc'
483  121: q            BINPUT     4
484  123: h            BINGET     4
485  125: c            GLOBAL     'copy_reg _reconstructor'
486  150: q            BINPUT     5
487  152: (            MARK
488  153: c                GLOBAL     '__main__ C'
489  165: q                BINPUT     6
490  167: c                GLOBAL     '__builtin__ object'
491  187: q                BINPUT     7
492  189: N                NONE
493  190: t                TUPLE      (MARK at 152)
494  191: q            BINPUT     8
495  193: R            REDUCE
496  194: q            BINPUT     9
497  196: }            EMPTY_DICT
498  197: q            BINPUT     10
499  199: (            MARK
500  200: X                BINUNICODE 'foo'
501  208: q                BINPUT     11
502  210: K                BININT1    1
503  212: X                BINUNICODE 'bar'
504  220: q                BINPUT     12
505  222: K                BININT1    2
506  224: u                SETITEMS   (MARK at 199)
507  225: b            BUILD
508  226: h            BINGET     9
509  228: t            TUPLE      (MARK at 112)
510  229: q        BINPUT     13
511  231: h        BINGET     13
512  233: K        BININT1    5
513  235: e        APPENDS    (MARK at 3)
514  236: .    STOP
515highest protocol among opcodes = 1
516"""
517
518DATA2 = (
519    b'\x80\x02]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
520    b'__builtin__\ncomplex\n'
521    b'q\x01G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00'
522    b'\x86q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xff'
523    b'J\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff'
524    b'\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00a'
525    b'bcq\x04h\x04c__main__\nC\nq\x05'
526    b')\x81q\x06}q\x07(X\x03\x00\x00\x00fooq\x08K\x01'
527    b'X\x03\x00\x00\x00barq\tK\x02ubh\x06tq\nh'
528    b'\nK\x05e.'
529)
530
531# Disassembly of DATA2
532DATA2_DIS = """\
533    0: \x80 PROTO      2
534    2: ]    EMPTY_LIST
535    3: q    BINPUT     0
536    5: (    MARK
537    6: K        BININT1    0
538    8: K        BININT1    1
539   10: G        BINFLOAT   2.0
540   19: c        GLOBAL     '__builtin__ complex'
541   40: q        BINPUT     1
542   42: G        BINFLOAT   3.0
543   51: G        BINFLOAT   0.0
544   60: \x86     TUPLE2
545   61: q        BINPUT     2
546   63: R        REDUCE
547   64: q        BINPUT     3
548   66: K        BININT1    1
549   68: J        BININT     -1
550   73: K        BININT1    255
551   75: J        BININT     -255
552   80: J        BININT     -256
553   85: M        BININT2    65535
554   88: J        BININT     -65535
555   93: J        BININT     -65536
556   98: J        BININT     2147483647
557  103: J        BININT     -2147483647
558  108: J        BININT     -2147483648
559  113: (        MARK
560  114: X            BINUNICODE 'abc'
561  122: q            BINPUT     4
562  124: h            BINGET     4
563  126: c            GLOBAL     '__main__ C'
564  138: q            BINPUT     5
565  140: )            EMPTY_TUPLE
566  141: \x81         NEWOBJ
567  142: q            BINPUT     6
568  144: }            EMPTY_DICT
569  145: q            BINPUT     7
570  147: (            MARK
571  148: X                BINUNICODE 'foo'
572  156: q                BINPUT     8
573  158: K                BININT1    1
574  160: X                BINUNICODE 'bar'
575  168: q                BINPUT     9
576  170: K                BININT1    2
577  172: u                SETITEMS   (MARK at 147)
578  173: b            BUILD
579  174: h            BINGET     6
580  176: t            TUPLE      (MARK at 113)
581  177: q        BINPUT     10
582  179: h        BINGET     10
583  181: K        BININT1    5
584  183: e        APPENDS    (MARK at 5)
585  184: .    STOP
586highest protocol among opcodes = 2
587"""
588
589DATA3 = (
590    b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
591    b'builtins\ncomplex\nq\x01G'
592    b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02'
593    b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff'
594    b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f'
595    b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq'
596    b'\x04h\x04c__main__\nC\nq\x05)\x81q'
597    b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00'
598    b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05'
599    b'e.'
600)
601
602# Disassembly of DATA3
603DATA3_DIS = """\
604    0: \x80 PROTO      3
605    2: ]    EMPTY_LIST
606    3: q    BINPUT     0
607    5: (    MARK
608    6: K        BININT1    0
609    8: K        BININT1    1
610   10: G        BINFLOAT   2.0
611   19: c        GLOBAL     'builtins complex'
612   37: q        BINPUT     1
613   39: G        BINFLOAT   3.0
614   48: G        BINFLOAT   0.0
615   57: \x86     TUPLE2
616   58: q        BINPUT     2
617   60: R        REDUCE
618   61: q        BINPUT     3
619   63: K        BININT1    1
620   65: J        BININT     -1
621   70: K        BININT1    255
622   72: J        BININT     -255
623   77: J        BININT     -256
624   82: M        BININT2    65535
625   85: J        BININT     -65535
626   90: J        BININT     -65536
627   95: J        BININT     2147483647
628  100: J        BININT     -2147483647
629  105: J        BININT     -2147483648
630  110: (        MARK
631  111: X            BINUNICODE 'abc'
632  119: q            BINPUT     4
633  121: h            BINGET     4
634  123: c            GLOBAL     '__main__ C'
635  135: q            BINPUT     5
636  137: )            EMPTY_TUPLE
637  138: \x81         NEWOBJ
638  139: q            BINPUT     6
639  141: }            EMPTY_DICT
640  142: q            BINPUT     7
641  144: (            MARK
642  145: X                BINUNICODE 'bar'
643  153: q                BINPUT     8
644  155: K                BININT1    2
645  157: X                BINUNICODE 'foo'
646  165: q                BINPUT     9
647  167: K                BININT1    1
648  169: u                SETITEMS   (MARK at 144)
649  170: b            BUILD
650  171: h            BINGET     6
651  173: t            TUPLE      (MARK at 110)
652  174: q        BINPUT     10
653  176: h        BINGET     10
654  178: K        BININT1    5
655  180: e        APPENDS    (MARK at 5)
656  181: .    STOP
657highest protocol among opcodes = 2
658"""
659
660DATA4 = (
661    b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@'
662    b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07'
663    b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G'
664    b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK'
665    b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ'
666    b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80('
667    b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c'
668    b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c'
669    b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.'
670)
671
672# Disassembly of DATA4
673DATA4_DIS = """\
674    0: \x80 PROTO      4
675    2: \x95 FRAME      168
676   11: ]    EMPTY_LIST
677   12: \x94 MEMOIZE
678   13: (    MARK
679   14: K        BININT1    0
680   16: K        BININT1    1
681   18: G        BINFLOAT   2.0
682   27: \x8c     SHORT_BINUNICODE 'builtins'
683   37: \x94     MEMOIZE
684   38: \x8c     SHORT_BINUNICODE 'complex'
685   47: \x94     MEMOIZE
686   48: \x93     STACK_GLOBAL
687   49: \x94     MEMOIZE
688   50: G        BINFLOAT   3.0
689   59: G        BINFLOAT   0.0
690   68: \x86     TUPLE2
691   69: \x94     MEMOIZE
692   70: R        REDUCE
693   71: \x94     MEMOIZE
694   72: K        BININT1    1
695   74: J        BININT     -1
696   79: K        BININT1    255
697   81: J        BININT     -255
698   86: J        BININT     -256
699   91: M        BININT2    65535
700   94: J        BININT     -65535
701   99: J        BININT     -65536
702  104: J        BININT     2147483647
703  109: J        BININT     -2147483647
704  114: J        BININT     -2147483648
705  119: (        MARK
706  120: \x8c         SHORT_BINUNICODE 'abc'
707  125: \x94         MEMOIZE
708  126: h            BINGET     6
709  128: \x8c         SHORT_BINUNICODE '__main__'
710  138: \x94         MEMOIZE
711  139: \x8c         SHORT_BINUNICODE 'C'
712  142: \x94         MEMOIZE
713  143: \x93         STACK_GLOBAL
714  144: \x94         MEMOIZE
715  145: )            EMPTY_TUPLE
716  146: \x81         NEWOBJ
717  147: \x94         MEMOIZE
718  148: }            EMPTY_DICT
719  149: \x94         MEMOIZE
720  150: (            MARK
721  151: \x8c             SHORT_BINUNICODE 'bar'
722  156: \x94             MEMOIZE
723  157: K                BININT1    2
724  159: \x8c             SHORT_BINUNICODE 'foo'
725  164: \x94             MEMOIZE
726  165: K                BININT1    1
727  167: u                SETITEMS   (MARK at 150)
728  168: b            BUILD
729  169: h            BINGET     10
730  171: t            TUPLE      (MARK at 119)
731  172: \x94     MEMOIZE
732  173: h        BINGET     14
733  175: K        BININT1    5
734  177: e        APPENDS    (MARK at 13)
735  178: .    STOP
736highest protocol among opcodes = 4
737"""
738
739# set([1,2]) pickled from 2.x with protocol 2
740DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.'
741
742# xrange(5) pickled from 2.x with protocol 2
743DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.'
744
745# a SimpleCookie() object pickled from 2.x with protocol 2
746DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
747               b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U'
748               b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07'
749               b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U'
750               b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b'
751               b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.')
752
753# set([3]) pickled from 2.x with protocol 2
754DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
755
756python2_exceptions_without_args = (
757    ArithmeticError,
758    AssertionError,
759    AttributeError,
760    BaseException,
761    BufferError,
762    BytesWarning,
763    DeprecationWarning,
764    EOFError,
765    EnvironmentError,
766    Exception,
767    FloatingPointError,
768    FutureWarning,
769    GeneratorExit,
770    IOError,
771    ImportError,
772    ImportWarning,
773    IndentationError,
774    IndexError,
775    KeyError,
776    KeyboardInterrupt,
777    LookupError,
778    MemoryError,
779    NameError,
780    NotImplementedError,
781    OSError,
782    OverflowError,
783    PendingDeprecationWarning,
784    ReferenceError,
785    RuntimeError,
786    RuntimeWarning,
787    # StandardError is gone in Python 3, we map it to Exception
788    StopIteration,
789    SyntaxError,
790    SyntaxWarning,
791    SystemError,
792    SystemExit,
793    TabError,
794    TypeError,
795    UnboundLocalError,
796    UnicodeError,
797    UnicodeWarning,
798    UserWarning,
799    ValueError,
800    Warning,
801    ZeroDivisionError,
802)
803
804exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.'
805
806# UnicodeEncodeError object pickled from 2.x with protocol 2
807DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n'
808              b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01'
809              b'U\x03badq\x03tq\x04Rq\x05.')
810
811
812def create_data():
813    c = C()
814    c.foo = 1
815    c.bar = 2
816    x = [0, 1, 2.0, 3.0+0j]
817    # Append some integer test cases at cPickle.c's internal size
818    # cutoffs.
819    uint1max = 0xff
820    uint2max = 0xffff
821    int4max = 0x7fffffff
822    x.extend([1, -1,
823              uint1max, -uint1max, -uint1max-1,
824              uint2max, -uint2max, -uint2max-1,
825               int4max,  -int4max,  -int4max-1])
826    y = ('abc', 'abc', c, c)
827    x.append(y)
828    x.append(y)
829    x.append(5)
830    return x
831
832
833class AbstractUnpickleTests:
834    # Subclass must define self.loads.
835
836    _testdata = create_data()
837
838    def assert_is_copy(self, obj, objcopy, msg=None):
839        """Utility method to verify if two objects are copies of each others.
840        """
841        if msg is None:
842            msg = "{!r} is not a copy of {!r}".format(obj, objcopy)
843        self.assertEqual(obj, objcopy, msg=msg)
844        self.assertIs(type(obj), type(objcopy), msg=msg)
845        if hasattr(obj, '__dict__'):
846            self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg)
847            self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg)
848        if hasattr(obj, '__slots__'):
849            self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg)
850            for slot in obj.__slots__:
851                self.assertEqual(
852                    hasattr(obj, slot), hasattr(objcopy, slot), msg=msg)
853                self.assertEqual(getattr(obj, slot, None),
854                                 getattr(objcopy, slot, None), msg=msg)
855
856    def check_unpickling_error(self, errors, data):
857        with self.subTest(data=data), \
858             self.assertRaises(errors):
859            try:
860                self.loads(data)
861            except BaseException as exc:
862                if support.verbose > 1:
863                    print('%-32r - %s: %s' %
864                          (data, exc.__class__.__name__, exc))
865                raise
866
867    def test_load_from_data0(self):
868        self.assert_is_copy(self._testdata, self.loads(DATA0))
869
870    def test_load_from_data1(self):
871        self.assert_is_copy(self._testdata, self.loads(DATA1))
872
873    def test_load_from_data2(self):
874        self.assert_is_copy(self._testdata, self.loads(DATA2))
875
876    def test_load_from_data3(self):
877        self.assert_is_copy(self._testdata, self.loads(DATA3))
878
879    def test_load_from_data4(self):
880        self.assert_is_copy(self._testdata, self.loads(DATA4))
881
882    def test_load_classic_instance(self):
883        # See issue5180.  Test loading 2.x pickles that
884        # contain an instance of old style class.
885        for X, args in [(C, ()), (D, ('x',)), (E, ())]:
886            xname = X.__name__.encode('ascii')
887            # Protocol 0 (text mode pickle):
888            """
889             0: (    MARK
890             1: i        INST       '__main__ X' (MARK at 0)
891            13: p    PUT        0
892            16: (    MARK
893            17: d        DICT       (MARK at 16)
894            18: p    PUT        1
895            21: b    BUILD
896            22: .    STOP
897            """
898            pickle0 = (b"(i__main__\n"
899                       b"X\n"
900                       b"p0\n"
901                       b"(dp1\nb.").replace(b'X', xname)
902            self.assert_is_copy(X(*args), self.loads(pickle0))
903
904            # Protocol 1 (binary mode pickle)
905            """
906             0: (    MARK
907             1: c        GLOBAL     '__main__ X'
908            13: q        BINPUT     0
909            15: o        OBJ        (MARK at 0)
910            16: q    BINPUT     1
911            18: }    EMPTY_DICT
912            19: q    BINPUT     2
913            21: b    BUILD
914            22: .    STOP
915            """
916            pickle1 = (b'(c__main__\n'
917                       b'X\n'
918                       b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
919            self.assert_is_copy(X(*args), self.loads(pickle1))
920
921            # Protocol 2 (pickle2 = b'\x80\x02' + pickle1)
922            """
923             0: \x80 PROTO      2
924             2: (    MARK
925             3: c        GLOBAL     '__main__ X'
926            15: q        BINPUT     0
927            17: o        OBJ        (MARK at 2)
928            18: q    BINPUT     1
929            20: }    EMPTY_DICT
930            21: q    BINPUT     2
931            23: b    BUILD
932            24: .    STOP
933            """
934            pickle2 = (b'\x80\x02(c__main__\n'
935                       b'X\n'
936                       b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
937            self.assert_is_copy(X(*args), self.loads(pickle2))
938
939    def test_maxint64(self):
940        maxint64 = (1 << 63) - 1
941        data = b'I' + str(maxint64).encode("ascii") + b'\n.'
942        got = self.loads(data)
943        self.assert_is_copy(maxint64, got)
944
945        # Try too with a bogus literal.
946        data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.'
947        self.check_unpickling_error(ValueError, data)
948
949    def test_unpickle_from_2x(self):
950        # Unpickle non-trivial data from Python 2.x.
951        loaded = self.loads(DATA_SET)
952        self.assertEqual(loaded, set([1, 2]))
953        loaded = self.loads(DATA_XRANGE)
954        self.assertEqual(type(loaded), type(range(0)))
955        self.assertEqual(list(loaded), list(range(5)))
956        loaded = self.loads(DATA_COOKIE)
957        self.assertEqual(type(loaded), SimpleCookie)
958        self.assertEqual(list(loaded.keys()), ["key"])
959        self.assertEqual(loaded["key"].value, "value")
960
961        # Exception objects without arguments pickled from 2.x with protocol 2
962        for exc in python2_exceptions_without_args:
963            data = exception_pickle.replace(b'?', exc.__name__.encode("ascii"))
964            loaded = self.loads(data)
965            self.assertIs(type(loaded), exc)
966
967        # StandardError is mapped to Exception, test that separately
968        loaded = self.loads(exception_pickle.replace(b'?', b'StandardError'))
969        self.assertIs(type(loaded), Exception)
970
971        loaded = self.loads(DATA_UEERR)
972        self.assertIs(type(loaded), UnicodeEncodeError)
973        self.assertEqual(loaded.object, "foo")
974        self.assertEqual(loaded.encoding, "ascii")
975        self.assertEqual(loaded.start, 0)
976        self.assertEqual(loaded.end, 1)
977        self.assertEqual(loaded.reason, "bad")
978
979    def test_load_python2_str_as_bytes(self):
980        # From Python 2: pickle.dumps('a\x00\xa0', protocol=0)
981        self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.",
982                                    encoding="bytes"), b'a\x00\xa0')
983        # From Python 2: pickle.dumps('a\x00\xa0', protocol=1)
984        self.assertEqual(self.loads(b'U\x03a\x00\xa0.',
985                                    encoding="bytes"), b'a\x00\xa0')
986        # From Python 2: pickle.dumps('a\x00\xa0', protocol=2)
987        self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.',
988                                    encoding="bytes"), b'a\x00\xa0')
989
990    def test_load_python2_unicode_as_str(self):
991        # From Python 2: pickle.dumps(u'π', protocol=0)
992        self.assertEqual(self.loads(b'V\\u03c0\n.',
993                                    encoding='bytes'), 'π')
994        # From Python 2: pickle.dumps(u'π', protocol=1)
995        self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.',
996                                    encoding="bytes"), 'π')
997        # From Python 2: pickle.dumps(u'π', protocol=2)
998        self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.',
999                                    encoding="bytes"), 'π')
1000
1001    def test_load_long_python2_str_as_bytes(self):
1002        # From Python 2: pickle.dumps('x' * 300, protocol=1)
1003        self.assertEqual(self.loads(pickle.BINSTRING +
1004                                    struct.pack("<I", 300) +
1005                                    b'x' * 300 + pickle.STOP,
1006                                    encoding='bytes'), b'x' * 300)
1007
1008    def test_constants(self):
1009        self.assertIsNone(self.loads(b'N.'))
1010        self.assertIs(self.loads(b'\x88.'), True)
1011        self.assertIs(self.loads(b'\x89.'), False)
1012        self.assertIs(self.loads(b'I01\n.'), True)
1013        self.assertIs(self.loads(b'I00\n.'), False)
1014
1015    def test_empty_bytestring(self):
1016        # issue 11286
1017        empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r')
1018        self.assertEqual(empty, '')
1019
1020    def test_short_binbytes(self):
1021        dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.'
1022        self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
1023
1024    def test_binbytes(self):
1025        dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.'
1026        self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
1027
1028    @requires_32b
1029    def test_negative_32b_binbytes(self):
1030        # On 32-bit builds, a BINBYTES of 2**31 or more is refused
1031        dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.'
1032        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1033                                    dumped)
1034
1035    @requires_32b
1036    def test_negative_32b_binunicode(self):
1037        # On 32-bit builds, a BINUNICODE of 2**31 or more is refused
1038        dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.'
1039        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1040                                    dumped)
1041
1042    def test_short_binunicode(self):
1043        dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.'
1044        self.assertEqual(self.loads(dumped), '\u20ac\x00')
1045
1046    def test_misc_get(self):
1047        self.check_unpickling_error(pickle.UnpicklingError, b'g0\np0')
1048        self.check_unpickling_error(pickle.UnpicklingError, b'jens:')
1049        self.check_unpickling_error(pickle.UnpicklingError, b'hens:')
1050        self.assert_is_copy([(100,), (100,)],
1051                            self.loads(b'((Kdtp0\nh\x00l.))'))
1052
1053    def test_binbytes8(self):
1054        dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
1055        self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
1056
1057    def test_binunicode8(self):
1058        dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
1059        self.assertEqual(self.loads(dumped), '\u20ac\x00')
1060
1061    def test_bytearray8(self):
1062        dumped = b'\x80\x05\x96\x03\x00\x00\x00\x00\x00\x00\x00xxx.'
1063        self.assertEqual(self.loads(dumped), bytearray(b'xxx'))
1064
1065    @requires_32b
1066    def test_large_32b_binbytes8(self):
1067        dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
1068        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1069                                    dumped)
1070
1071    @requires_32b
1072    def test_large_32b_bytearray8(self):
1073        dumped = b'\x80\x05\x96\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
1074        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1075                                    dumped)
1076
1077    @requires_32b
1078    def test_large_32b_binunicode8(self):
1079        dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
1080        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1081                                    dumped)
1082
1083    def test_get(self):
1084        pickled = b'((lp100000\ng100000\nt.'
1085        unpickled = self.loads(pickled)
1086        self.assertEqual(unpickled, ([],)*2)
1087        self.assertIs(unpickled[0], unpickled[1])
1088
1089    def test_binget(self):
1090        pickled = b'(]q\xffh\xfft.'
1091        unpickled = self.loads(pickled)
1092        self.assertEqual(unpickled, ([],)*2)
1093        self.assertIs(unpickled[0], unpickled[1])
1094
1095    def test_long_binget(self):
1096        pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.'
1097        unpickled = self.loads(pickled)
1098        self.assertEqual(unpickled, ([],)*2)
1099        self.assertIs(unpickled[0], unpickled[1])
1100
1101    def test_dup(self):
1102        pickled = b'((l2t.'
1103        unpickled = self.loads(pickled)
1104        self.assertEqual(unpickled, ([],)*2)
1105        self.assertIs(unpickled[0], unpickled[1])
1106
1107    def test_negative_put(self):
1108        # Issue #12847
1109        dumped = b'Va\np-1\n.'
1110        self.check_unpickling_error(ValueError, dumped)
1111
1112    @requires_32b
1113    def test_negative_32b_binput(self):
1114        # Issue #12847
1115        dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.'
1116        self.check_unpickling_error(ValueError, dumped)
1117
1118    def test_badly_escaped_string(self):
1119        self.check_unpickling_error(ValueError, b"S'\\'\n.")
1120
1121    def test_badly_quoted_string(self):
1122        # Issue #17710
1123        badpickles = [b"S'\n.",
1124                      b'S"\n.',
1125                      b'S\' \n.',
1126                      b'S" \n.',
1127                      b'S\'"\n.',
1128                      b'S"\'\n.',
1129                      b"S' ' \n.",
1130                      b'S" " \n.',
1131                      b"S ''\n.",
1132                      b'S ""\n.',
1133                      b'S \n.',
1134                      b'S\n.',
1135                      b'S.']
1136        for p in badpickles:
1137            self.check_unpickling_error(pickle.UnpicklingError, p)
1138
1139    def test_correctly_quoted_string(self):
1140        goodpickles = [(b"S''\n.", ''),
1141                       (b'S""\n.', ''),
1142                       (b'S"\\n"\n.', '\n'),
1143                       (b"S'\\n'\n.", '\n')]
1144        for p, expected in goodpickles:
1145            self.assertEqual(self.loads(p), expected)
1146
1147    def test_frame_readline(self):
1148        pickled = b'\x80\x04\x95\x05\x00\x00\x00\x00\x00\x00\x00I42\n.'
1149        #    0: \x80 PROTO      4
1150        #    2: \x95 FRAME      5
1151        #   11: I    INT        42
1152        #   15: .    STOP
1153        self.assertEqual(self.loads(pickled), 42)
1154
1155    def test_compat_unpickle(self):
1156        # xrange(1, 7)
1157        pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.'
1158        unpickled = self.loads(pickled)
1159        self.assertIs(type(unpickled), range)
1160        self.assertEqual(unpickled, range(1, 7))
1161        self.assertEqual(list(unpickled), [1, 2, 3, 4, 5, 6])
1162        # reduce
1163        pickled = b'\x80\x02c__builtin__\nreduce\n.'
1164        self.assertIs(self.loads(pickled), functools.reduce)
1165        # whichdb.whichdb
1166        pickled = b'\x80\x02cwhichdb\nwhichdb\n.'
1167        self.assertIs(self.loads(pickled), dbm.whichdb)
1168        # Exception(), StandardError()
1169        for name in (b'Exception', b'StandardError'):
1170            pickled = (b'\x80\x02cexceptions\n' + name + b'\nU\x03ugh\x85R.')
1171            unpickled = self.loads(pickled)
1172            self.assertIs(type(unpickled), Exception)
1173            self.assertEqual(str(unpickled), 'ugh')
1174        # UserDict.UserDict({1: 2}), UserDict.IterableUserDict({1: 2})
1175        for name in (b'UserDict', b'IterableUserDict'):
1176            pickled = (b'\x80\x02(cUserDict\n' + name +
1177                       b'\no}U\x04data}K\x01K\x02ssb.')
1178            unpickled = self.loads(pickled)
1179            self.assertIs(type(unpickled), collections.UserDict)
1180            self.assertEqual(unpickled, collections.UserDict({1: 2}))
1181
1182    def test_bad_reduce(self):
1183        self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0)
1184        self.check_unpickling_error(TypeError, b'N)R.')
1185        self.check_unpickling_error(TypeError, b'cbuiltins\nint\nNR.')
1186
1187    def test_bad_newobj(self):
1188        error = (pickle.UnpicklingError, TypeError)
1189        self.assertEqual(self.loads(b'cbuiltins\nint\n)\x81.'), 0)
1190        self.check_unpickling_error(error, b'cbuiltins\nlen\n)\x81.')
1191        self.check_unpickling_error(error, b'cbuiltins\nint\nN\x81.')
1192
1193    def test_bad_newobj_ex(self):
1194        error = (pickle.UnpicklingError, TypeError)
1195        self.assertEqual(self.loads(b'cbuiltins\nint\n)}\x92.'), 0)
1196        self.check_unpickling_error(error, b'cbuiltins\nlen\n)}\x92.')
1197        self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.')
1198        self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.')
1199
1200    def test_bad_stack(self):
1201        badpickles = [
1202            b'.',                       # STOP
1203            b'0',                       # POP
1204            b'1',                       # POP_MARK
1205            b'2',                       # DUP
1206            b'(2',
1207            b'R',                       # REDUCE
1208            b')R',
1209            b'a',                       # APPEND
1210            b'Na',
1211            b'b',                       # BUILD
1212            b'Nb',
1213            b'd',                       # DICT
1214            b'e',                       # APPENDS
1215            b'(e',
1216            b'ibuiltins\nlist\n',       # INST
1217            b'l',                       # LIST
1218            b'o',                       # OBJ
1219            b'(o',
1220            b'p1\n',                    # PUT
1221            b'q\x00',                   # BINPUT
1222            b'r\x00\x00\x00\x00',       # LONG_BINPUT
1223            b's',                       # SETITEM
1224            b'Ns',
1225            b'NNs',
1226            b't',                       # TUPLE
1227            b'u',                       # SETITEMS
1228            b'(u',
1229            b'}(Nu',
1230            b'\x81',                    # NEWOBJ
1231            b')\x81',
1232            b'\x85',                    # TUPLE1
1233            b'\x86',                    # TUPLE2
1234            b'N\x86',
1235            b'\x87',                    # TUPLE3
1236            b'N\x87',
1237            b'NN\x87',
1238            b'\x90',                    # ADDITEMS
1239            b'(\x90',
1240            b'\x91',                    # FROZENSET
1241            b'\x92',                    # NEWOBJ_EX
1242            b')}\x92',
1243            b'\x93',                    # STACK_GLOBAL
1244            b'Vlist\n\x93',
1245            b'\x94',                    # MEMOIZE
1246        ]
1247        for p in badpickles:
1248            self.check_unpickling_error(self.bad_stack_errors, p)
1249
1250    def test_bad_mark(self):
1251        badpickles = [
1252            b'N(.',                     # STOP
1253            b'N(2',                     # DUP
1254            b'cbuiltins\nlist\n)(R',    # REDUCE
1255            b'cbuiltins\nlist\n()R',
1256            b']N(a',                    # APPEND
1257                                        # BUILD
1258            b'cbuiltins\nValueError\n)R}(b',
1259            b'cbuiltins\nValueError\n)R(}b',
1260            b'(Nd',                     # DICT
1261            b'N(p1\n',                  # PUT
1262            b'N(q\x00',                 # BINPUT
1263            b'N(r\x00\x00\x00\x00',     # LONG_BINPUT
1264            b'}NN(s',                   # SETITEM
1265            b'}N(Ns',
1266            b'}(NNs',
1267            b'}((u',                    # SETITEMS
1268            b'cbuiltins\nlist\n)(\x81', # NEWOBJ
1269            b'cbuiltins\nlist\n()\x81',
1270            b'N(\x85',                  # TUPLE1
1271            b'NN(\x86',                 # TUPLE2
1272            b'N(N\x86',
1273            b'NNN(\x87',                # TUPLE3
1274            b'NN(N\x87',
1275            b'N(NN\x87',
1276            b']((\x90',                 # ADDITEMS
1277                                        # NEWOBJ_EX
1278            b'cbuiltins\nlist\n)}(\x92',
1279            b'cbuiltins\nlist\n)(}\x92',
1280            b'cbuiltins\nlist\n()}\x92',
1281                                        # STACK_GLOBAL
1282            b'Vbuiltins\n(Vlist\n\x93',
1283            b'Vbuiltins\nVlist\n(\x93',
1284            b'N(\x94',                  # MEMOIZE
1285        ]
1286        for p in badpickles:
1287            self.check_unpickling_error(self.bad_stack_errors, p)
1288
1289    def test_truncated_data(self):
1290        self.check_unpickling_error(EOFError, b'')
1291        self.check_unpickling_error(EOFError, b'N')
1292        badpickles = [
1293            b'B',                       # BINBYTES
1294            b'B\x03\x00\x00',
1295            b'B\x03\x00\x00\x00',
1296            b'B\x03\x00\x00\x00ab',
1297            b'C',                       # SHORT_BINBYTES
1298            b'C\x03',
1299            b'C\x03ab',
1300            b'F',                       # FLOAT
1301            b'F0.0',
1302            b'F0.00',
1303            b'G',                       # BINFLOAT
1304            b'G\x00\x00\x00\x00\x00\x00\x00',
1305            b'I',                       # INT
1306            b'I0',
1307            b'J',                       # BININT
1308            b'J\x00\x00\x00',
1309            b'K',                       # BININT1
1310            b'L',                       # LONG
1311            b'L0',
1312            b'L10',
1313            b'L0L',
1314            b'L10L',
1315            b'M',                       # BININT2
1316            b'M\x00',
1317            # b'P',                       # PERSID
1318            # b'Pabc',
1319            b'S',                       # STRING
1320            b"S'abc'",
1321            b'T',                       # BINSTRING
1322            b'T\x03\x00\x00',
1323            b'T\x03\x00\x00\x00',
1324            b'T\x03\x00\x00\x00ab',
1325            b'U',                       # SHORT_BINSTRING
1326            b'U\x03',
1327            b'U\x03ab',
1328            b'V',                       # UNICODE
1329            b'Vabc',
1330            b'X',                       # BINUNICODE
1331            b'X\x03\x00\x00',
1332            b'X\x03\x00\x00\x00',
1333            b'X\x03\x00\x00\x00ab',
1334            b'(c',                      # GLOBAL
1335            b'(cbuiltins',
1336            b'(cbuiltins\n',
1337            b'(cbuiltins\nlist',
1338            b'Ng',                      # GET
1339            b'Ng0',
1340            b'(i',                      # INST
1341            b'(ibuiltins',
1342            b'(ibuiltins\n',
1343            b'(ibuiltins\nlist',
1344            b'Nh',                      # BINGET
1345            b'Nj',                      # LONG_BINGET
1346            b'Nj\x00\x00\x00',
1347            b'Np',                      # PUT
1348            b'Np0',
1349            b'Nq',                      # BINPUT
1350            b'Nr',                      # LONG_BINPUT
1351            b'Nr\x00\x00\x00',
1352            b'\x80',                    # PROTO
1353            b'\x82',                    # EXT1
1354            b'\x83',                    # EXT2
1355            b'\x84\x01',
1356            b'\x84',                    # EXT4
1357            b'\x84\x01\x00\x00',
1358            b'\x8a',                    # LONG1
1359            b'\x8b',                    # LONG4
1360            b'\x8b\x00\x00\x00',
1361            b'\x8c',                    # SHORT_BINUNICODE
1362            b'\x8c\x03',
1363            b'\x8c\x03ab',
1364            b'\x8d',                    # BINUNICODE8
1365            b'\x8d\x03\x00\x00\x00\x00\x00\x00',
1366            b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00',
1367            b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab',
1368            b'\x8e',                    # BINBYTES8
1369            b'\x8e\x03\x00\x00\x00\x00\x00\x00',
1370            b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00',
1371            b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab',
1372            b'\x96',                    # BYTEARRAY8
1373            b'\x96\x03\x00\x00\x00\x00\x00\x00',
1374            b'\x96\x03\x00\x00\x00\x00\x00\x00\x00',
1375            b'\x96\x03\x00\x00\x00\x00\x00\x00\x00ab',
1376            b'\x95',                    # FRAME
1377            b'\x95\x02\x00\x00\x00\x00\x00\x00',
1378            b'\x95\x02\x00\x00\x00\x00\x00\x00\x00',
1379            b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N',
1380        ]
1381        for p in badpickles:
1382            self.check_unpickling_error(self.truncated_errors, p)
1383
1384    @threading_helper.reap_threads
1385    @threading_helper.requires_working_threading()
1386    def test_unpickle_module_race(self):
1387        # https://bugs.python.org/issue34572
1388        locker_module = dedent("""
1389        import threading
1390        barrier = threading.Barrier(2)
1391        """)
1392        locking_import_module = dedent("""
1393        import locker
1394        locker.barrier.wait()
1395        class ToBeUnpickled(object):
1396            pass
1397        """)
1398
1399        os.mkdir(TESTFN)
1400        self.addCleanup(shutil.rmtree, TESTFN)
1401        sys.path.insert(0, TESTFN)
1402        self.addCleanup(sys.path.remove, TESTFN)
1403        with open(os.path.join(TESTFN, "locker.py"), "wb") as f:
1404            f.write(locker_module.encode('utf-8'))
1405        with open(os.path.join(TESTFN, "locking_import.py"), "wb") as f:
1406            f.write(locking_import_module.encode('utf-8'))
1407        self.addCleanup(forget, "locker")
1408        self.addCleanup(forget, "locking_import")
1409
1410        import locker
1411
1412        pickle_bytes = (
1413            b'\x80\x03clocking_import\nToBeUnpickled\nq\x00)\x81q\x01.')
1414
1415        # Then try to unpickle two of these simultaneously
1416        # One of them will cause the module import, and we want it to block
1417        # until the other one either:
1418        #   - fails (before the patch for this issue)
1419        #   - blocks on the import lock for the module, as it should
1420        results = []
1421        barrier = threading.Barrier(3)
1422        def t():
1423            # This ensures the threads have all started
1424            # presumably barrier release is faster than thread startup
1425            barrier.wait()
1426            results.append(pickle.loads(pickle_bytes))
1427
1428        t1 = threading.Thread(target=t)
1429        t2 = threading.Thread(target=t)
1430        t1.start()
1431        t2.start()
1432
1433        barrier.wait()
1434        # could have delay here
1435        locker.barrier.wait()
1436
1437        t1.join()
1438        t2.join()
1439
1440        from locking_import import ToBeUnpickled
1441        self.assertEqual(
1442            [type(x) for x in results],
1443            [ToBeUnpickled] * 2)
1444
1445
1446
1447class AbstractPickleTests:
1448    # Subclass must define self.dumps, self.loads.
1449
1450    optimized = False
1451
1452    _testdata = AbstractUnpickleTests._testdata
1453
1454    def setUp(self):
1455        pass
1456
1457    assert_is_copy = AbstractUnpickleTests.assert_is_copy
1458
1459    def test_misc(self):
1460        # test various datatypes not tested by testdata
1461        for proto in protocols:
1462            x = myint(4)
1463            s = self.dumps(x, proto)
1464            y = self.loads(s)
1465            self.assert_is_copy(x, y)
1466
1467            x = (1, ())
1468            s = self.dumps(x, proto)
1469            y = self.loads(s)
1470            self.assert_is_copy(x, y)
1471
1472            x = initarg(1, x)
1473            s = self.dumps(x, proto)
1474            y = self.loads(s)
1475            self.assert_is_copy(x, y)
1476
1477        # XXX test __reduce__ protocol?
1478
1479    def test_roundtrip_equality(self):
1480        expected = self._testdata
1481        for proto in protocols:
1482            s = self.dumps(expected, proto)
1483            got = self.loads(s)
1484            self.assert_is_copy(expected, got)
1485
1486    # There are gratuitous differences between pickles produced by
1487    # pickle and cPickle, largely because cPickle starts PUT indices at
1488    # 1 and pickle starts them at 0.  See XXX comment in cPickle's put2() --
1489    # there's a comment with an exclamation point there whose meaning
1490    # is a mystery.  cPickle also suppresses PUT for objects with a refcount
1491    # of 1.
1492    def dont_test_disassembly(self):
1493        from io import StringIO
1494        from pickletools import dis
1495
1496        for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS):
1497            s = self.dumps(self._testdata, proto)
1498            filelike = StringIO()
1499            dis(s, out=filelike)
1500            got = filelike.getvalue()
1501            self.assertEqual(expected, got)
1502
1503    def _test_recursive_list(self, cls, aslist=identity, minprotocol=0):
1504        # List containing itself.
1505        l = cls()
1506        l.append(l)
1507        for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
1508            s = self.dumps(l, proto)
1509            x = self.loads(s)
1510            self.assertIsInstance(x, cls)
1511            y = aslist(x)
1512            self.assertEqual(len(y), 1)
1513            self.assertIs(y[0], x)
1514
1515    def test_recursive_list(self):
1516        self._test_recursive_list(list)
1517
1518    def test_recursive_list_subclass(self):
1519        self._test_recursive_list(MyList, minprotocol=2)
1520
1521    def test_recursive_list_like(self):
1522        self._test_recursive_list(REX_six, aslist=lambda x: x.items)
1523
1524    def _test_recursive_tuple_and_list(self, cls, aslist=identity, minprotocol=0):
1525        # Tuple containing a list containing the original tuple.
1526        t = (cls(),)
1527        t[0].append(t)
1528        for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
1529            s = self.dumps(t, proto)
1530            x = self.loads(s)
1531            self.assertIsInstance(x, tuple)
1532            self.assertEqual(len(x), 1)
1533            self.assertIsInstance(x[0], cls)
1534            y = aslist(x[0])
1535            self.assertEqual(len(y), 1)
1536            self.assertIs(y[0], x)
1537
1538        # List containing a tuple containing the original list.
1539        t, = t
1540        for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
1541            s = self.dumps(t, proto)
1542            x = self.loads(s)
1543            self.assertIsInstance(x, cls)
1544            y = aslist(x)
1545            self.assertEqual(len(y), 1)
1546            self.assertIsInstance(y[0], tuple)
1547            self.assertEqual(len(y[0]), 1)
1548            self.assertIs(y[0][0], x)
1549
1550    def test_recursive_tuple_and_list(self):
1551        self._test_recursive_tuple_and_list(list)
1552
1553    def test_recursive_tuple_and_list_subclass(self):
1554        self._test_recursive_tuple_and_list(MyList, minprotocol=2)
1555
1556    def test_recursive_tuple_and_list_like(self):
1557        self._test_recursive_tuple_and_list(REX_six, aslist=lambda x: x.items)
1558
1559    def _test_recursive_dict(self, cls, asdict=identity, minprotocol=0):
1560        # Dict containing itself.
1561        d = cls()
1562        d[1] = d
1563        for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
1564            s = self.dumps(d, proto)
1565            x = self.loads(s)
1566            self.assertIsInstance(x, cls)
1567            y = asdict(x)
1568            self.assertEqual(list(y.keys()), [1])
1569            self.assertIs(y[1], x)
1570
1571    def test_recursive_dict(self):
1572        self._test_recursive_dict(dict)
1573
1574    def test_recursive_dict_subclass(self):
1575        self._test_recursive_dict(MyDict, minprotocol=2)
1576
1577    def test_recursive_dict_like(self):
1578        self._test_recursive_dict(REX_seven, asdict=lambda x: x.table)
1579
1580    def _test_recursive_tuple_and_dict(self, cls, asdict=identity, minprotocol=0):
1581        # Tuple containing a dict containing the original tuple.
1582        t = (cls(),)
1583        t[0][1] = t
1584        for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
1585            s = self.dumps(t, proto)
1586            x = self.loads(s)
1587            self.assertIsInstance(x, tuple)
1588            self.assertEqual(len(x), 1)
1589            self.assertIsInstance(x[0], cls)
1590            y = asdict(x[0])
1591            self.assertEqual(list(y), [1])
1592            self.assertIs(y[1], x)
1593
1594        # Dict containing a tuple containing the original dict.
1595        t, = t
1596        for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
1597            s = self.dumps(t, proto)
1598            x = self.loads(s)
1599            self.assertIsInstance(x, cls)
1600            y = asdict(x)
1601            self.assertEqual(list(y), [1])
1602            self.assertIsInstance(y[1], tuple)
1603            self.assertEqual(len(y[1]), 1)
1604            self.assertIs(y[1][0], x)
1605
1606    def test_recursive_tuple_and_dict(self):
1607        self._test_recursive_tuple_and_dict(dict)
1608
1609    def test_recursive_tuple_and_dict_subclass(self):
1610        self._test_recursive_tuple_and_dict(MyDict, minprotocol=2)
1611
1612    def test_recursive_tuple_and_dict_like(self):
1613        self._test_recursive_tuple_and_dict(REX_seven, asdict=lambda x: x.table)
1614
1615    def _test_recursive_dict_key(self, cls, asdict=identity, minprotocol=0):
1616        # Dict containing an immutable object (as key) containing the original
1617        # dict.
1618        d = cls()
1619        d[K(d)] = 1
1620        for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
1621            s = self.dumps(d, proto)
1622            x = self.loads(s)
1623            self.assertIsInstance(x, cls)
1624            y = asdict(x)
1625            self.assertEqual(len(y.keys()), 1)
1626            self.assertIsInstance(list(y.keys())[0], K)
1627            self.assertIs(list(y.keys())[0].value, x)
1628
1629    def test_recursive_dict_key(self):
1630        self._test_recursive_dict_key(dict)
1631
1632    def test_recursive_dict_subclass_key(self):
1633        self._test_recursive_dict_key(MyDict, minprotocol=2)
1634
1635    def test_recursive_dict_like_key(self):
1636        self._test_recursive_dict_key(REX_seven, asdict=lambda x: x.table)
1637
1638    def _test_recursive_tuple_and_dict_key(self, cls, asdict=identity, minprotocol=0):
1639        # Tuple containing a dict containing an immutable object (as key)
1640        # containing the original tuple.
1641        t = (cls(),)
1642        t[0][K(t)] = 1
1643        for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
1644            s = self.dumps(t, proto)
1645            x = self.loads(s)
1646            self.assertIsInstance(x, tuple)
1647            self.assertEqual(len(x), 1)
1648            self.assertIsInstance(x[0], cls)
1649            y = asdict(x[0])
1650            self.assertEqual(len(y), 1)
1651            self.assertIsInstance(list(y.keys())[0], K)
1652            self.assertIs(list(y.keys())[0].value, x)
1653
1654        # Dict containing an immutable object (as key) containing a tuple
1655        # containing the original dict.
1656        t, = t
1657        for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1):
1658            s = self.dumps(t, proto)
1659            x = self.loads(s)
1660            self.assertIsInstance(x, cls)
1661            y = asdict(x)
1662            self.assertEqual(len(y), 1)
1663            self.assertIsInstance(list(y.keys())[0], K)
1664            self.assertIs(list(y.keys())[0].value[0], x)
1665
1666    def test_recursive_tuple_and_dict_key(self):
1667        self._test_recursive_tuple_and_dict_key(dict)
1668
1669    def test_recursive_tuple_and_dict_subclass_key(self):
1670        self._test_recursive_tuple_and_dict_key(MyDict, minprotocol=2)
1671
1672    def test_recursive_tuple_and_dict_like_key(self):
1673        self._test_recursive_tuple_and_dict_key(REX_seven, asdict=lambda x: x.table)
1674
1675    def test_recursive_set(self):
1676        # Set containing an immutable object containing the original set.
1677        y = set()
1678        y.add(K(y))
1679        for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1680            s = self.dumps(y, proto)
1681            x = self.loads(s)
1682            self.assertIsInstance(x, set)
1683            self.assertEqual(len(x), 1)
1684            self.assertIsInstance(list(x)[0], K)
1685            self.assertIs(list(x)[0].value, x)
1686
1687        # Immutable object containing a set containing the original object.
1688        y, = y
1689        for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1690            s = self.dumps(y, proto)
1691            x = self.loads(s)
1692            self.assertIsInstance(x, K)
1693            self.assertIsInstance(x.value, set)
1694            self.assertEqual(len(x.value), 1)
1695            self.assertIs(list(x.value)[0], x)
1696
1697    def test_recursive_inst(self):
1698        # Mutable object containing itself.
1699        i = Object()
1700        i.attr = i
1701        for proto in protocols:
1702            s = self.dumps(i, proto)
1703            x = self.loads(s)
1704            self.assertIsInstance(x, Object)
1705            self.assertEqual(dir(x), dir(i))
1706            self.assertIs(x.attr, x)
1707
1708    def test_recursive_multi(self):
1709        l = []
1710        d = {1:l}
1711        i = Object()
1712        i.attr = d
1713        l.append(i)
1714        for proto in protocols:
1715            s = self.dumps(l, proto)
1716            x = self.loads(s)
1717            self.assertIsInstance(x, list)
1718            self.assertEqual(len(x), 1)
1719            self.assertEqual(dir(x[0]), dir(i))
1720            self.assertEqual(list(x[0].attr.keys()), [1])
1721            self.assertIs(x[0].attr[1], x)
1722
1723    def _test_recursive_collection_and_inst(self, factory):
1724        # Mutable object containing a collection containing the original
1725        # object.
1726        o = Object()
1727        o.attr = factory([o])
1728        t = type(o.attr)
1729        for proto in protocols:
1730            s = self.dumps(o, proto)
1731            x = self.loads(s)
1732            self.assertIsInstance(x.attr, t)
1733            self.assertEqual(len(x.attr), 1)
1734            self.assertIsInstance(list(x.attr)[0], Object)
1735            self.assertIs(list(x.attr)[0], x)
1736
1737        # Collection containing a mutable object containing the original
1738        # collection.
1739        o = o.attr
1740        for proto in protocols:
1741            s = self.dumps(o, proto)
1742            x = self.loads(s)
1743            self.assertIsInstance(x, t)
1744            self.assertEqual(len(x), 1)
1745            self.assertIsInstance(list(x)[0], Object)
1746            self.assertIs(list(x)[0].attr, x)
1747
1748    def test_recursive_list_and_inst(self):
1749        self._test_recursive_collection_and_inst(list)
1750
1751    def test_recursive_tuple_and_inst(self):
1752        self._test_recursive_collection_and_inst(tuple)
1753
1754    def test_recursive_dict_and_inst(self):
1755        self._test_recursive_collection_and_inst(dict.fromkeys)
1756
1757    def test_recursive_set_and_inst(self):
1758        self._test_recursive_collection_and_inst(set)
1759
1760    def test_recursive_frozenset_and_inst(self):
1761        self._test_recursive_collection_and_inst(frozenset)
1762
1763    def test_recursive_list_subclass_and_inst(self):
1764        self._test_recursive_collection_and_inst(MyList)
1765
1766    def test_recursive_tuple_subclass_and_inst(self):
1767        self._test_recursive_collection_and_inst(MyTuple)
1768
1769    def test_recursive_dict_subclass_and_inst(self):
1770        self._test_recursive_collection_and_inst(MyDict.fromkeys)
1771
1772    def test_recursive_set_subclass_and_inst(self):
1773        self._test_recursive_collection_and_inst(MySet)
1774
1775    def test_recursive_frozenset_subclass_and_inst(self):
1776        self._test_recursive_collection_and_inst(MyFrozenSet)
1777
1778    def test_recursive_inst_state(self):
1779        # Mutable object containing itself.
1780        y = REX_state()
1781        y.state = y
1782        for proto in protocols:
1783            s = self.dumps(y, proto)
1784            x = self.loads(s)
1785            self.assertIsInstance(x, REX_state)
1786            self.assertIs(x.state, x)
1787
1788    def test_recursive_tuple_and_inst_state(self):
1789        # Tuple containing a mutable object containing the original tuple.
1790        t = (REX_state(),)
1791        t[0].state = t
1792        for proto in protocols:
1793            s = self.dumps(t, proto)
1794            x = self.loads(s)
1795            self.assertIsInstance(x, tuple)
1796            self.assertEqual(len(x), 1)
1797            self.assertIsInstance(x[0], REX_state)
1798            self.assertIs(x[0].state, x)
1799
1800        # Mutable object containing a tuple containing the object.
1801        t, = t
1802        for proto in protocols:
1803            s = self.dumps(t, proto)
1804            x = self.loads(s)
1805            self.assertIsInstance(x, REX_state)
1806            self.assertIsInstance(x.state, tuple)
1807            self.assertEqual(len(x.state), 1)
1808            self.assertIs(x.state[0], x)
1809
1810    def test_unicode(self):
1811        endcases = ['', '<\\u>', '<\\\u1234>', '<\n>',
1812                    '<\\>', '<\\\U00012345>',
1813                    # surrogates
1814                    '<\udc80>']
1815        for proto in protocols:
1816            for u in endcases:
1817                p = self.dumps(u, proto)
1818                u2 = self.loads(p)
1819                self.assert_is_copy(u, u2)
1820
1821    def test_unicode_high_plane(self):
1822        t = '\U00012345'
1823        for proto in protocols:
1824            p = self.dumps(t, proto)
1825            t2 = self.loads(p)
1826            self.assert_is_copy(t, t2)
1827
1828    def test_bytes(self):
1829        for proto in protocols:
1830            for s in b'', b'xyz', b'xyz'*100:
1831                p = self.dumps(s, proto)
1832                self.assert_is_copy(s, self.loads(p))
1833            for s in [bytes([i]) for i in range(256)]:
1834                p = self.dumps(s, proto)
1835                self.assert_is_copy(s, self.loads(p))
1836            for s in [bytes([i, i]) for i in range(256)]:
1837                p = self.dumps(s, proto)
1838                self.assert_is_copy(s, self.loads(p))
1839
1840    def test_bytearray(self):
1841        for proto in protocols:
1842            for s in b'', b'xyz', b'xyz'*100:
1843                b = bytearray(s)
1844                p = self.dumps(b, proto)
1845                bb = self.loads(p)
1846                self.assertIsNot(bb, b)
1847                self.assert_is_copy(b, bb)
1848                if proto <= 3:
1849                    # bytearray is serialized using a global reference
1850                    self.assertIn(b'bytearray', p)
1851                    self.assertTrue(opcode_in_pickle(pickle.GLOBAL, p))
1852                elif proto == 4:
1853                    self.assertIn(b'bytearray', p)
1854                    self.assertTrue(opcode_in_pickle(pickle.STACK_GLOBAL, p))
1855                elif proto == 5:
1856                    self.assertNotIn(b'bytearray', p)
1857                    self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p))
1858
1859    def test_bytearray_memoization_bug(self):
1860        for proto in protocols:
1861            for s in b'', b'xyz', b'xyz'*100:
1862                b = bytearray(s)
1863                p = self.dumps((b, b), proto)
1864                b1, b2 = self.loads(p)
1865                self.assertIs(b1, b2)
1866
1867    def test_ints(self):
1868        for proto in protocols:
1869            n = sys.maxsize
1870            while n:
1871                for expected in (-n, n):
1872                    s = self.dumps(expected, proto)
1873                    n2 = self.loads(s)
1874                    self.assert_is_copy(expected, n2)
1875                n = n >> 1
1876
1877    def test_long(self):
1878        for proto in protocols:
1879            # 256 bytes is where LONG4 begins.
1880            for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257:
1881                nbase = 1 << nbits
1882                for npos in nbase-1, nbase, nbase+1:
1883                    for n in npos, -npos:
1884                        pickle = self.dumps(n, proto)
1885                        got = self.loads(pickle)
1886                        self.assert_is_copy(n, got)
1887        # Try a monster.  This is quadratic-time in protos 0 & 1, so don't
1888        # bother with those.
1889        nbase = int("deadbeeffeedface", 16)
1890        nbase += nbase << 1000000
1891        for n in nbase, -nbase:
1892            p = self.dumps(n, 2)
1893            got = self.loads(p)
1894            # assert_is_copy is very expensive here as it precomputes
1895            # a failure message by computing the repr() of n and got,
1896            # we just do the check ourselves.
1897            self.assertIs(type(got), int)
1898            self.assertEqual(n, got)
1899
1900    def test_float(self):
1901        test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5,
1902                       3.14, 263.44582062374053, 6.022e23, 1e30]
1903        test_values = test_values + [-x for x in test_values]
1904        for proto in protocols:
1905            for value in test_values:
1906                pickle = self.dumps(value, proto)
1907                got = self.loads(pickle)
1908                self.assert_is_copy(value, got)
1909
1910    @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
1911    def test_float_format(self):
1912        # make sure that floats are formatted locale independent with proto 0
1913        self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
1914
1915    def test_reduce(self):
1916        for proto in protocols:
1917            inst = AAA()
1918            dumped = self.dumps(inst, proto)
1919            loaded = self.loads(dumped)
1920            self.assertEqual(loaded, REDUCE_A)
1921
1922    def test_getinitargs(self):
1923        for proto in protocols:
1924            inst = initarg(1, 2)
1925            dumped = self.dumps(inst, proto)
1926            loaded = self.loads(dumped)
1927            self.assert_is_copy(inst, loaded)
1928
1929    def test_metaclass(self):
1930        a = use_metaclass()
1931        for proto in protocols:
1932            s = self.dumps(a, proto)
1933            b = self.loads(s)
1934            self.assertEqual(a.__class__, b.__class__)
1935
1936    def test_dynamic_class(self):
1937        a = create_dynamic_class("my_dynamic_class", (object,))
1938        copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__)
1939        for proto in protocols:
1940            s = self.dumps(a, proto)
1941            b = self.loads(s)
1942            self.assertEqual(a, b)
1943            self.assertIs(type(a), type(b))
1944
1945    def test_structseq(self):
1946        import time
1947        import os
1948
1949        t = time.localtime()
1950        for proto in protocols:
1951            s = self.dumps(t, proto)
1952            u = self.loads(s)
1953            self.assert_is_copy(t, u)
1954            t = os.stat(os.curdir)
1955            s = self.dumps(t, proto)
1956            u = self.loads(s)
1957            self.assert_is_copy(t, u)
1958            if hasattr(os, "statvfs"):
1959                t = os.statvfs(os.curdir)
1960                s = self.dumps(t, proto)
1961                u = self.loads(s)
1962                self.assert_is_copy(t, u)
1963
1964    def test_ellipsis(self):
1965        for proto in protocols:
1966            s = self.dumps(..., proto)
1967            u = self.loads(s)
1968            self.assertIs(..., u)
1969
1970    def test_notimplemented(self):
1971        for proto in protocols:
1972            s = self.dumps(NotImplemented, proto)
1973            u = self.loads(s)
1974            self.assertIs(NotImplemented, u)
1975
1976    def test_singleton_types(self):
1977        # Issue #6477: Test that types of built-in singletons can be pickled.
1978        singletons = [None, ..., NotImplemented]
1979        for singleton in singletons:
1980            for proto in protocols:
1981                s = self.dumps(type(singleton), proto)
1982                u = self.loads(s)
1983                self.assertIs(type(singleton), u)
1984
1985    def test_builtin_types(self):
1986        for t in builtins.__dict__.values():
1987            if isinstance(t, type) and not issubclass(t, BaseException):
1988                for proto in protocols:
1989                    s = self.dumps(t, proto)
1990                    self.assertIs(self.loads(s), t)
1991
1992    def test_builtin_exceptions(self):
1993        for t in builtins.__dict__.values():
1994            if isinstance(t, type) and issubclass(t, BaseException):
1995                for proto in protocols:
1996                    s = self.dumps(t, proto)
1997                    u = self.loads(s)
1998                    if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError:
1999                        self.assertIs(u, OSError)
2000                    elif proto <= 2 and issubclass(t, ImportError):
2001                        self.assertIs(u, ImportError)
2002                    else:
2003                        self.assertIs(u, t)
2004
2005    def test_builtin_functions(self):
2006        for t in builtins.__dict__.values():
2007            if isinstance(t, types.BuiltinFunctionType):
2008                for proto in protocols:
2009                    s = self.dumps(t, proto)
2010                    self.assertIs(self.loads(s), t)
2011
2012    # Tests for protocol 2
2013
2014    def test_proto(self):
2015        for proto in protocols:
2016            pickled = self.dumps(None, proto)
2017            if proto >= 2:
2018                proto_header = pickle.PROTO + bytes([proto])
2019                self.assertTrue(pickled.startswith(proto_header))
2020            else:
2021                self.assertEqual(count_opcode(pickle.PROTO, pickled), 0)
2022
2023        oob = protocols[-1] + 1     # a future protocol
2024        build_none = pickle.NONE + pickle.STOP
2025        badpickle = pickle.PROTO + bytes([oob]) + build_none
2026        try:
2027            self.loads(badpickle)
2028        except ValueError as err:
2029            self.assertIn("unsupported pickle protocol", str(err))
2030        else:
2031            self.fail("expected bad protocol number to raise ValueError")
2032
2033    def test_long1(self):
2034        x = 12345678910111213141516178920
2035        for proto in protocols:
2036            s = self.dumps(x, proto)
2037            y = self.loads(s)
2038            self.assert_is_copy(x, y)
2039            self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2)
2040
2041    def test_long4(self):
2042        x = 12345678910111213141516178920 << (256*8)
2043        for proto in protocols:
2044            s = self.dumps(x, proto)
2045            y = self.loads(s)
2046            self.assert_is_copy(x, y)
2047            self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2)
2048
2049    def test_short_tuples(self):
2050        # Map (proto, len(tuple)) to expected opcode.
2051        expected_opcode = {(0, 0): pickle.TUPLE,
2052                           (0, 1): pickle.TUPLE,
2053                           (0, 2): pickle.TUPLE,
2054                           (0, 3): pickle.TUPLE,
2055                           (0, 4): pickle.TUPLE,
2056
2057                           (1, 0): pickle.EMPTY_TUPLE,
2058                           (1, 1): pickle.TUPLE,
2059                           (1, 2): pickle.TUPLE,
2060                           (1, 3): pickle.TUPLE,
2061                           (1, 4): pickle.TUPLE,
2062
2063                           (2, 0): pickle.EMPTY_TUPLE,
2064                           (2, 1): pickle.TUPLE1,
2065                           (2, 2): pickle.TUPLE2,
2066                           (2, 3): pickle.TUPLE3,
2067                           (2, 4): pickle.TUPLE,
2068
2069                           (3, 0): pickle.EMPTY_TUPLE,
2070                           (3, 1): pickle.TUPLE1,
2071                           (3, 2): pickle.TUPLE2,
2072                           (3, 3): pickle.TUPLE3,
2073                           (3, 4): pickle.TUPLE,
2074                          }
2075        a = ()
2076        b = (1,)
2077        c = (1, 2)
2078        d = (1, 2, 3)
2079        e = (1, 2, 3, 4)
2080        for proto in protocols:
2081            for x in a, b, c, d, e:
2082                s = self.dumps(x, proto)
2083                y = self.loads(s)
2084                self.assert_is_copy(x, y)
2085                expected = expected_opcode[min(proto, 3), len(x)]
2086                self.assertTrue(opcode_in_pickle(expected, s))
2087
2088    def test_singletons(self):
2089        # Map (proto, singleton) to expected opcode.
2090        expected_opcode = {(0, None): pickle.NONE,
2091                           (1, None): pickle.NONE,
2092                           (2, None): pickle.NONE,
2093                           (3, None): pickle.NONE,
2094
2095                           (0, True): pickle.INT,
2096                           (1, True): pickle.INT,
2097                           (2, True): pickle.NEWTRUE,
2098                           (3, True): pickle.NEWTRUE,
2099
2100                           (0, False): pickle.INT,
2101                           (1, False): pickle.INT,
2102                           (2, False): pickle.NEWFALSE,
2103                           (3, False): pickle.NEWFALSE,
2104                          }
2105        for proto in protocols:
2106            for x in None, False, True:
2107                s = self.dumps(x, proto)
2108                y = self.loads(s)
2109                self.assertTrue(x is y, (proto, x, s, y))
2110                expected = expected_opcode[min(proto, 3), x]
2111                self.assertTrue(opcode_in_pickle(expected, s))
2112
2113    def test_newobj_tuple(self):
2114        x = MyTuple([1, 2, 3])
2115        x.foo = 42
2116        x.bar = "hello"
2117        for proto in protocols:
2118            s = self.dumps(x, proto)
2119            y = self.loads(s)
2120            self.assert_is_copy(x, y)
2121
2122    def test_newobj_list(self):
2123        x = MyList([1, 2, 3])
2124        x.foo = 42
2125        x.bar = "hello"
2126        for proto in protocols:
2127            s = self.dumps(x, proto)
2128            y = self.loads(s)
2129            self.assert_is_copy(x, y)
2130
2131    def test_newobj_generic(self):
2132        for proto in protocols:
2133            for C in myclasses:
2134                B = C.__base__
2135                x = C(C.sample)
2136                x.foo = 42
2137                s = self.dumps(x, proto)
2138                y = self.loads(s)
2139                detail = (proto, C, B, x, y, type(y))
2140                self.assert_is_copy(x, y) # XXX revisit
2141                self.assertEqual(B(x), B(y), detail)
2142                self.assertEqual(x.__dict__, y.__dict__, detail)
2143
2144    def test_newobj_proxies(self):
2145        # NEWOBJ should use the __class__ rather than the raw type
2146        classes = myclasses[:]
2147        # Cannot create weakproxies to these classes
2148        for c in (MyInt, MyTuple):
2149            classes.remove(c)
2150        for proto in protocols:
2151            for C in classes:
2152                B = C.__base__
2153                x = C(C.sample)
2154                x.foo = 42
2155                p = weakref.proxy(x)
2156                s = self.dumps(p, proto)
2157                y = self.loads(s)
2158                self.assertEqual(type(y), type(x))  # rather than type(p)
2159                detail = (proto, C, B, x, y, type(y))
2160                self.assertEqual(B(x), B(y), detail)
2161                self.assertEqual(x.__dict__, y.__dict__, detail)
2162
2163    def test_newobj_overridden_new(self):
2164        # Test that Python class with C implemented __new__ is pickleable
2165        for proto in protocols:
2166            x = MyIntWithNew2(1)
2167            x.foo = 42
2168            s = self.dumps(x, proto)
2169            y = self.loads(s)
2170            self.assertIs(type(y), MyIntWithNew2)
2171            self.assertEqual(int(y), 1)
2172            self.assertEqual(y.foo, 42)
2173
2174    def test_newobj_not_class(self):
2175        # Issue 24552
2176        global SimpleNewObj
2177        save = SimpleNewObj
2178        o = SimpleNewObj.__new__(SimpleNewObj)
2179        b = self.dumps(o, 4)
2180        try:
2181            SimpleNewObj = 42
2182            self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b)
2183        finally:
2184            SimpleNewObj = save
2185
2186    # Register a type with copyreg, with extension code extcode.  Pickle
2187    # an object of that type.  Check that the resulting pickle uses opcode
2188    # (EXT[124]) under proto 2, and not in proto 1.
2189
2190    def produce_global_ext(self, extcode, opcode):
2191        e = ExtensionSaver(extcode)
2192        try:
2193            copyreg.add_extension(__name__, "MyList", extcode)
2194            x = MyList([1, 2, 3])
2195            x.foo = 42
2196            x.bar = "hello"
2197
2198            # Dump using protocol 1 for comparison.
2199            s1 = self.dumps(x, 1)
2200            self.assertIn(__name__.encode("utf-8"), s1)
2201            self.assertIn(b"MyList", s1)
2202            self.assertFalse(opcode_in_pickle(opcode, s1))
2203
2204            y = self.loads(s1)
2205            self.assert_is_copy(x, y)
2206
2207            # Dump using protocol 2 for test.
2208            s2 = self.dumps(x, 2)
2209            self.assertNotIn(__name__.encode("utf-8"), s2)
2210            self.assertNotIn(b"MyList", s2)
2211            self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2))
2212
2213            y = self.loads(s2)
2214            self.assert_is_copy(x, y)
2215        finally:
2216            e.restore()
2217
2218    def test_global_ext1(self):
2219        self.produce_global_ext(0x00000001, pickle.EXT1)  # smallest EXT1 code
2220        self.produce_global_ext(0x000000ff, pickle.EXT1)  # largest EXT1 code
2221
2222    def test_global_ext2(self):
2223        self.produce_global_ext(0x00000100, pickle.EXT2)  # smallest EXT2 code
2224        self.produce_global_ext(0x0000ffff, pickle.EXT2)  # largest EXT2 code
2225        self.produce_global_ext(0x0000abcd, pickle.EXT2)  # check endianness
2226
2227    def test_global_ext4(self):
2228        self.produce_global_ext(0x00010000, pickle.EXT4)  # smallest EXT4 code
2229        self.produce_global_ext(0x7fffffff, pickle.EXT4)  # largest EXT4 code
2230        self.produce_global_ext(0x12abcdef, pickle.EXT4)  # check endianness
2231
2232    def test_list_chunking(self):
2233        n = 10  # too small to chunk
2234        x = list(range(n))
2235        for proto in protocols:
2236            s = self.dumps(x, proto)
2237            y = self.loads(s)
2238            self.assert_is_copy(x, y)
2239            num_appends = count_opcode(pickle.APPENDS, s)
2240            self.assertEqual(num_appends, proto > 0)
2241
2242        n = 2500  # expect at least two chunks when proto > 0
2243        x = list(range(n))
2244        for proto in protocols:
2245            s = self.dumps(x, proto)
2246            y = self.loads(s)
2247            self.assert_is_copy(x, y)
2248            num_appends = count_opcode(pickle.APPENDS, s)
2249            if proto == 0:
2250                self.assertEqual(num_appends, 0)
2251            else:
2252                self.assertTrue(num_appends >= 2)
2253
2254    def test_dict_chunking(self):
2255        n = 10  # too small to chunk
2256        x = dict.fromkeys(range(n))
2257        for proto in protocols:
2258            s = self.dumps(x, proto)
2259            self.assertIsInstance(s, bytes_types)
2260            y = self.loads(s)
2261            self.assert_is_copy(x, y)
2262            num_setitems = count_opcode(pickle.SETITEMS, s)
2263            self.assertEqual(num_setitems, proto > 0)
2264
2265        n = 2500  # expect at least two chunks when proto > 0
2266        x = dict.fromkeys(range(n))
2267        for proto in protocols:
2268            s = self.dumps(x, proto)
2269            y = self.loads(s)
2270            self.assert_is_copy(x, y)
2271            num_setitems = count_opcode(pickle.SETITEMS, s)
2272            if proto == 0:
2273                self.assertEqual(num_setitems, 0)
2274            else:
2275                self.assertTrue(num_setitems >= 2)
2276
2277    def test_set_chunking(self):
2278        n = 10  # too small to chunk
2279        x = set(range(n))
2280        for proto in protocols:
2281            s = self.dumps(x, proto)
2282            y = self.loads(s)
2283            self.assert_is_copy(x, y)
2284            num_additems = count_opcode(pickle.ADDITEMS, s)
2285            if proto < 4:
2286                self.assertEqual(num_additems, 0)
2287            else:
2288                self.assertEqual(num_additems, 1)
2289
2290        n = 2500  # expect at least two chunks when proto >= 4
2291        x = set(range(n))
2292        for proto in protocols:
2293            s = self.dumps(x, proto)
2294            y = self.loads(s)
2295            self.assert_is_copy(x, y)
2296            num_additems = count_opcode(pickle.ADDITEMS, s)
2297            if proto < 4:
2298                self.assertEqual(num_additems, 0)
2299            else:
2300                self.assertGreaterEqual(num_additems, 2)
2301
2302    def test_simple_newobj(self):
2303        x = SimpleNewObj.__new__(SimpleNewObj, 0xface)  # avoid __init__
2304        x.abc = 666
2305        for proto in protocols:
2306            with self.subTest(proto=proto):
2307                s = self.dumps(x, proto)
2308                if proto < 1:
2309                    self.assertIn(b'\nI64206', s)  # INT
2310                else:
2311                    self.assertIn(b'M\xce\xfa', s)  # BININT2
2312                self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
2313                                 2 <= proto)
2314                self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s))
2315                y = self.loads(s)   # will raise TypeError if __init__ called
2316                self.assert_is_copy(x, y)
2317
2318    def test_complex_newobj(self):
2319        x = ComplexNewObj.__new__(ComplexNewObj, 0xface)  # avoid __init__
2320        x.abc = 666
2321        for proto in protocols:
2322            with self.subTest(proto=proto):
2323                s = self.dumps(x, proto)
2324                if proto < 1:
2325                    self.assertIn(b'\nI64206', s)  # INT
2326                elif proto < 2:
2327                    self.assertIn(b'M\xce\xfa', s)  # BININT2
2328                elif proto < 4:
2329                    self.assertIn(b'X\x04\x00\x00\x00FACE', s)  # BINUNICODE
2330                else:
2331                    self.assertIn(b'\x8c\x04FACE', s)  # SHORT_BINUNICODE
2332                self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
2333                                 2 <= proto)
2334                self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s))
2335                y = self.loads(s)   # will raise TypeError if __init__ called
2336                self.assert_is_copy(x, y)
2337
2338    def test_complex_newobj_ex(self):
2339        x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface)  # avoid __init__
2340        x.abc = 666
2341        for proto in protocols:
2342            with self.subTest(proto=proto):
2343                s = self.dumps(x, proto)
2344                if proto < 1:
2345                    self.assertIn(b'\nI64206', s)  # INT
2346                elif proto < 2:
2347                    self.assertIn(b'M\xce\xfa', s)  # BININT2
2348                elif proto < 4:
2349                    self.assertIn(b'X\x04\x00\x00\x00FACE', s)  # BINUNICODE
2350                else:
2351                    self.assertIn(b'\x8c\x04FACE', s)  # SHORT_BINUNICODE
2352                self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s))
2353                self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s),
2354                                 4 <= proto)
2355                y = self.loads(s)   # will raise TypeError if __init__ called
2356                self.assert_is_copy(x, y)
2357
2358    def test_newobj_list_slots(self):
2359        x = SlotList([1, 2, 3])
2360        x.foo = 42
2361        x.bar = "hello"
2362        s = self.dumps(x, 2)
2363        y = self.loads(s)
2364        self.assert_is_copy(x, y)
2365
2366    def test_reduce_overrides_default_reduce_ex(self):
2367        for proto in protocols:
2368            x = REX_one()
2369            self.assertEqual(x._reduce_called, 0)
2370            s = self.dumps(x, proto)
2371            self.assertEqual(x._reduce_called, 1)
2372            y = self.loads(s)
2373            self.assertEqual(y._reduce_called, 0)
2374
2375    def test_reduce_ex_called(self):
2376        for proto in protocols:
2377            x = REX_two()
2378            self.assertEqual(x._proto, None)
2379            s = self.dumps(x, proto)
2380            self.assertEqual(x._proto, proto)
2381            y = self.loads(s)
2382            self.assertEqual(y._proto, None)
2383
2384    def test_reduce_ex_overrides_reduce(self):
2385        for proto in protocols:
2386            x = REX_three()
2387            self.assertEqual(x._proto, None)
2388            s = self.dumps(x, proto)
2389            self.assertEqual(x._proto, proto)
2390            y = self.loads(s)
2391            self.assertEqual(y._proto, None)
2392
2393    def test_reduce_ex_calls_base(self):
2394        for proto in protocols:
2395            x = REX_four()
2396            self.assertEqual(x._proto, None)
2397            s = self.dumps(x, proto)
2398            self.assertEqual(x._proto, proto)
2399            y = self.loads(s)
2400            self.assertEqual(y._proto, proto)
2401
2402    def test_reduce_calls_base(self):
2403        for proto in protocols:
2404            x = REX_five()
2405            self.assertEqual(x._reduce_called, 0)
2406            s = self.dumps(x, proto)
2407            self.assertEqual(x._reduce_called, 1)
2408            y = self.loads(s)
2409            self.assertEqual(y._reduce_called, 1)
2410
2411    @no_tracing
2412    def test_bad_getattr(self):
2413        # Issue #3514: crash when there is an infinite loop in __getattr__
2414        x = BadGetattr()
2415        for proto in range(2):
2416            with support.infinite_recursion():
2417                self.assertRaises(RuntimeError, self.dumps, x, proto)
2418        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
2419            s = self.dumps(x, proto)
2420
2421    def test_reduce_bad_iterator(self):
2422        # Issue4176: crash when 4th and 5th items of __reduce__()
2423        # are not iterators
2424        class C(object):
2425            def __reduce__(self):
2426                # 4th item is not an iterator
2427                return list, (), None, [], None
2428        class D(object):
2429            def __reduce__(self):
2430                # 5th item is not an iterator
2431                return dict, (), None, None, []
2432
2433        # Python implementation is less strict and also accepts iterables.
2434        for proto in protocols:
2435            try:
2436                self.dumps(C(), proto)
2437            except pickle.PicklingError:
2438                pass
2439            try:
2440                self.dumps(D(), proto)
2441            except pickle.PicklingError:
2442                pass
2443
2444    def test_many_puts_and_gets(self):
2445        # Test that internal data structures correctly deal with lots of
2446        # puts/gets.
2447        keys = ("aaa" + str(i) for i in range(100))
2448        large_dict = dict((k, [4, 5, 6]) for k in keys)
2449        obj = [dict(large_dict), dict(large_dict), dict(large_dict)]
2450
2451        for proto in protocols:
2452            with self.subTest(proto=proto):
2453                dumped = self.dumps(obj, proto)
2454                loaded = self.loads(dumped)
2455                self.assert_is_copy(obj, loaded)
2456
2457    def test_attribute_name_interning(self):
2458        # Test that attribute names of pickled objects are interned when
2459        # unpickling.
2460        for proto in protocols:
2461            x = C()
2462            x.foo = 42
2463            x.bar = "hello"
2464            s = self.dumps(x, proto)
2465            y = self.loads(s)
2466            x_keys = sorted(x.__dict__)
2467            y_keys = sorted(y.__dict__)
2468            for x_key, y_key in zip(x_keys, y_keys):
2469                self.assertIs(x_key, y_key)
2470
2471    def test_pickle_to_2x(self):
2472        # Pickle non-trivial data with protocol 2, expecting that it yields
2473        # the same result as Python 2.x did.
2474        # NOTE: this test is a bit too strong since we can produce different
2475        # bytecode that 2.x will still understand.
2476        dumped = self.dumps(range(5), 2)
2477        self.assertEqual(dumped, DATA_XRANGE)
2478        dumped = self.dumps(set([3]), 2)
2479        self.assertEqual(dumped, DATA_SET2)
2480
2481    def test_large_pickles(self):
2482        # Test the correctness of internal buffering routines when handling
2483        # large data.
2484        for proto in protocols:
2485            data = (1, min, b'xy' * (30 * 1024), len)
2486            dumped = self.dumps(data, proto)
2487            loaded = self.loads(dumped)
2488            self.assertEqual(len(loaded), len(data))
2489            self.assertEqual(loaded, data)
2490
2491    def test_int_pickling_efficiency(self):
2492        # Test compacity of int representation (see issue #12744)
2493        for proto in protocols:
2494            with self.subTest(proto=proto):
2495                pickles = [self.dumps(2**n, proto) for n in range(70)]
2496                sizes = list(map(len, pickles))
2497                # the size function is monotonic
2498                self.assertEqual(sorted(sizes), sizes)
2499                if proto >= 2:
2500                    for p in pickles:
2501                        self.assertFalse(opcode_in_pickle(pickle.LONG, p))
2502
2503    def _check_pickling_with_opcode(self, obj, opcode, proto):
2504        pickled = self.dumps(obj, proto)
2505        self.assertTrue(opcode_in_pickle(opcode, pickled))
2506        unpickled = self.loads(pickled)
2507        self.assertEqual(obj, unpickled)
2508
2509    def test_appends_on_non_lists(self):
2510        # Issue #17720
2511        obj = REX_six([1, 2, 3])
2512        for proto in protocols:
2513            if proto == 0:
2514                self._check_pickling_with_opcode(obj, pickle.APPEND, proto)
2515            else:
2516                self._check_pickling_with_opcode(obj, pickle.APPENDS, proto)
2517
2518    def test_setitems_on_non_dicts(self):
2519        obj = REX_seven({1: -1, 2: -2, 3: -3})
2520        for proto in protocols:
2521            if proto == 0:
2522                self._check_pickling_with_opcode(obj, pickle.SETITEM, proto)
2523            else:
2524                self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto)
2525
2526    # Exercise framing (proto >= 4) for significant workloads
2527
2528    FRAME_SIZE_MIN = 4
2529    FRAME_SIZE_TARGET = 64 * 1024
2530
2531    def check_frame_opcodes(self, pickled):
2532        """
2533        Check the arguments of FRAME opcodes in a protocol 4+ pickle.
2534
2535        Note that binary objects that are larger than FRAME_SIZE_TARGET are not
2536        framed by default and are therefore considered a frame by themselves in
2537        the following consistency check.
2538        """
2539        frame_end = frameless_start = None
2540        frameless_opcodes = {'BINBYTES', 'BINUNICODE', 'BINBYTES8',
2541                             'BINUNICODE8', 'BYTEARRAY8'}
2542        for op, arg, pos in pickletools.genops(pickled):
2543            if frame_end is not None:
2544                self.assertLessEqual(pos, frame_end)
2545                if pos == frame_end:
2546                    frame_end = None
2547
2548            if frame_end is not None:  # framed
2549                self.assertNotEqual(op.name, 'FRAME')
2550                if op.name in frameless_opcodes:
2551                    # Only short bytes and str objects should be written
2552                    # in a frame
2553                    self.assertLessEqual(len(arg), self.FRAME_SIZE_TARGET)
2554
2555            else:  # not framed
2556                if (op.name == 'FRAME' or
2557                    (op.name in frameless_opcodes and
2558                     len(arg) > self.FRAME_SIZE_TARGET)):
2559                    # Frame or large bytes or str object
2560                    if frameless_start is not None:
2561                        # Only short data should be written outside of a frame
2562                        self.assertLess(pos - frameless_start,
2563                                        self.FRAME_SIZE_MIN)
2564                        frameless_start = None
2565                elif frameless_start is None and op.name != 'PROTO':
2566                    frameless_start = pos
2567
2568            if op.name == 'FRAME':
2569                self.assertGreaterEqual(arg, self.FRAME_SIZE_MIN)
2570                frame_end = pos + 9 + arg
2571
2572        pos = len(pickled)
2573        if frame_end is not None:
2574            self.assertEqual(frame_end, pos)
2575        elif frameless_start is not None:
2576            self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN)
2577
2578    @support.skip_if_pgo_task
2579    def test_framing_many_objects(self):
2580        obj = list(range(10**5))
2581        for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2582            with self.subTest(proto=proto):
2583                pickled = self.dumps(obj, proto)
2584                unpickled = self.loads(pickled)
2585                self.assertEqual(obj, unpickled)
2586                bytes_per_frame = (len(pickled) /
2587                                   count_opcode(pickle.FRAME, pickled))
2588                self.assertGreater(bytes_per_frame,
2589                                   self.FRAME_SIZE_TARGET / 2)
2590                self.assertLessEqual(bytes_per_frame,
2591                                     self.FRAME_SIZE_TARGET * 1)
2592                self.check_frame_opcodes(pickled)
2593
2594    def test_framing_large_objects(self):
2595        N = 1024 * 1024
2596        small_items = [[i] for i in range(10)]
2597        obj = [b'x' * N, *small_items, b'y' * N, 'z' * N]
2598        for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2599            for fast in [False, True]:
2600                with self.subTest(proto=proto, fast=fast):
2601                    if not fast:
2602                        # fast=False by default.
2603                        # This covers in-memory pickling with pickle.dumps().
2604                        pickled = self.dumps(obj, proto)
2605                    else:
2606                        # Pickler is required when fast=True.
2607                        if not hasattr(self, 'pickler'):
2608                            continue
2609                        buf = io.BytesIO()
2610                        pickler = self.pickler(buf, protocol=proto)
2611                        pickler.fast = fast
2612                        pickler.dump(obj)
2613                        pickled = buf.getvalue()
2614                    unpickled = self.loads(pickled)
2615                    # More informative error message in case of failure.
2616                    self.assertEqual([len(x) for x in obj],
2617                                     [len(x) for x in unpickled])
2618                    # Perform full equality check if the lengths match.
2619                    self.assertEqual(obj, unpickled)
2620                    n_frames = count_opcode(pickle.FRAME, pickled)
2621                    # A single frame for small objects between
2622                    # first two large objects.
2623                    self.assertEqual(n_frames, 1)
2624                    self.check_frame_opcodes(pickled)
2625
2626    def test_optional_frames(self):
2627        if pickle.HIGHEST_PROTOCOL < 4:
2628            return
2629
2630        def remove_frames(pickled, keep_frame=None):
2631            """Remove frame opcodes from the given pickle."""
2632            frame_starts = []
2633            # 1 byte for the opcode and 8 for the argument
2634            frame_opcode_size = 9
2635            for opcode, _, pos in pickletools.genops(pickled):
2636                if opcode.name == 'FRAME':
2637                    frame_starts.append(pos)
2638
2639            newpickle = bytearray()
2640            last_frame_end = 0
2641            for i, pos in enumerate(frame_starts):
2642                if keep_frame and keep_frame(i):
2643                    continue
2644                newpickle += pickled[last_frame_end:pos]
2645                last_frame_end = pos + frame_opcode_size
2646            newpickle += pickled[last_frame_end:]
2647            return newpickle
2648
2649        frame_size = self.FRAME_SIZE_TARGET
2650        num_frames = 20
2651        # Large byte objects (dict values) intermittent with small objects
2652        # (dict keys)
2653        for bytes_type in (bytes, bytearray):
2654            obj = {i: bytes_type([i]) * frame_size for i in range(num_frames)}
2655
2656            for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2657                pickled = self.dumps(obj, proto)
2658
2659                frameless_pickle = remove_frames(pickled)
2660                self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0)
2661                self.assertEqual(obj, self.loads(frameless_pickle))
2662
2663                some_frames_pickle = remove_frames(pickled, lambda i: i % 2)
2664                self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle),
2665                                count_opcode(pickle.FRAME, pickled))
2666                self.assertEqual(obj, self.loads(some_frames_pickle))
2667
2668    @support.skip_if_pgo_task
2669    def test_framed_write_sizes_with_delayed_writer(self):
2670        class ChunkAccumulator:
2671            """Accumulate pickler output in a list of raw chunks."""
2672            def __init__(self):
2673                self.chunks = []
2674            def write(self, chunk):
2675                self.chunks.append(chunk)
2676            def concatenate_chunks(self):
2677                return b"".join(self.chunks)
2678
2679        for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2680            objects = [(str(i).encode('ascii'), i % 42, {'i': str(i)})
2681                       for i in range(int(1e4))]
2682            # Add a large unique ASCII string
2683            objects.append('0123456789abcdef' *
2684                           (self.FRAME_SIZE_TARGET // 16 + 1))
2685
2686            # Protocol 4 packs groups of small objects into frames and issues
2687            # calls to write only once or twice per frame:
2688            # The C pickler issues one call to write per-frame (header and
2689            # contents) while Python pickler issues two calls to write: one for
2690            # the frame header and one for the frame binary contents.
2691            writer = ChunkAccumulator()
2692            self.pickler(writer, proto).dump(objects)
2693
2694            # Actually read the binary content of the chunks after the end
2695            # of the call to dump: any memoryview passed to write should not
2696            # be released otherwise this delayed access would not be possible.
2697            pickled = writer.concatenate_chunks()
2698            reconstructed = self.loads(pickled)
2699            self.assertEqual(reconstructed, objects)
2700            self.assertGreater(len(writer.chunks), 1)
2701
2702            # memoryviews should own the memory.
2703            del objects
2704            support.gc_collect()
2705            self.assertEqual(writer.concatenate_chunks(), pickled)
2706
2707            n_frames = (len(pickled) - 1) // self.FRAME_SIZE_TARGET + 1
2708            # There should be at least one call to write per frame
2709            self.assertGreaterEqual(len(writer.chunks), n_frames)
2710
2711            # but not too many either: there can be one for the proto,
2712            # one per-frame header, one per frame for the actual contents,
2713            # and two for the header.
2714            self.assertLessEqual(len(writer.chunks), 2 * n_frames + 3)
2715
2716            chunk_sizes = [len(c) for c in writer.chunks]
2717            large_sizes = [s for s in chunk_sizes
2718                           if s >= self.FRAME_SIZE_TARGET]
2719            medium_sizes = [s for s in chunk_sizes
2720                           if 9 < s < self.FRAME_SIZE_TARGET]
2721            small_sizes = [s for s in chunk_sizes if s <= 9]
2722
2723            # Large chunks should not be too large:
2724            for chunk_size in large_sizes:
2725                self.assertLess(chunk_size, 2 * self.FRAME_SIZE_TARGET,
2726                                chunk_sizes)
2727            # There shouldn't bee too many small chunks: the protocol header,
2728            # the frame headers and the large string headers are written
2729            # in small chunks.
2730            self.assertLessEqual(len(small_sizes),
2731                                 len(large_sizes) + len(medium_sizes) + 3,
2732                                 chunk_sizes)
2733
2734    def test_nested_names(self):
2735        global Nested
2736        class Nested:
2737            class A:
2738                class B:
2739                    class C:
2740                        pass
2741        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2742            for obj in [Nested.A, Nested.A.B, Nested.A.B.C]:
2743                with self.subTest(proto=proto, obj=obj):
2744                    unpickled = self.loads(self.dumps(obj, proto))
2745                    self.assertIs(obj, unpickled)
2746
2747    def test_recursive_nested_names(self):
2748        global Recursive
2749        class Recursive:
2750            pass
2751        Recursive.mod = sys.modules[Recursive.__module__]
2752        Recursive.__qualname__ = 'Recursive.mod.Recursive'
2753        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2754            with self.subTest(proto=proto):
2755                unpickled = self.loads(self.dumps(Recursive, proto))
2756                self.assertIs(unpickled, Recursive)
2757        del Recursive.mod # break reference loop
2758
2759    def test_py_methods(self):
2760        global PyMethodsTest
2761        class PyMethodsTest:
2762            @staticmethod
2763            def cheese():
2764                return "cheese"
2765            @classmethod
2766            def wine(cls):
2767                assert cls is PyMethodsTest
2768                return "wine"
2769            def biscuits(self):
2770                assert isinstance(self, PyMethodsTest)
2771                return "biscuits"
2772            class Nested:
2773                "Nested class"
2774                @staticmethod
2775                def ketchup():
2776                    return "ketchup"
2777                @classmethod
2778                def maple(cls):
2779                    assert cls is PyMethodsTest.Nested
2780                    return "maple"
2781                def pie(self):
2782                    assert isinstance(self, PyMethodsTest.Nested)
2783                    return "pie"
2784
2785        py_methods = (
2786            PyMethodsTest.cheese,
2787            PyMethodsTest.wine,
2788            PyMethodsTest().biscuits,
2789            PyMethodsTest.Nested.ketchup,
2790            PyMethodsTest.Nested.maple,
2791            PyMethodsTest.Nested().pie
2792        )
2793        py_unbound_methods = (
2794            (PyMethodsTest.biscuits, PyMethodsTest),
2795            (PyMethodsTest.Nested.pie, PyMethodsTest.Nested)
2796        )
2797        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2798            for method in py_methods:
2799                with self.subTest(proto=proto, method=method):
2800                    unpickled = self.loads(self.dumps(method, proto))
2801                    self.assertEqual(method(), unpickled())
2802            for method, cls in py_unbound_methods:
2803                obj = cls()
2804                with self.subTest(proto=proto, method=method):
2805                    unpickled = self.loads(self.dumps(method, proto))
2806                    self.assertEqual(method(obj), unpickled(obj))
2807
2808        descriptors = (
2809            PyMethodsTest.__dict__['cheese'],  # static method descriptor
2810            PyMethodsTest.__dict__['wine'],  # class method descriptor
2811        )
2812        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2813            for descr in descriptors:
2814                with self.subTest(proto=proto, descr=descr):
2815                    self.assertRaises(TypeError, self.dumps, descr, proto)
2816
2817    def test_c_methods(self):
2818        global Subclass
2819        class Subclass(tuple):
2820            class Nested(str):
2821                pass
2822
2823        c_methods = (
2824            # bound built-in method
2825            ("abcd".index, ("c",)),
2826            # unbound built-in method
2827            (str.index, ("abcd", "c")),
2828            # bound "slot" method
2829            ([1, 2, 3].__len__, ()),
2830            # unbound "slot" method
2831            (list.__len__, ([1, 2, 3],)),
2832            # bound "coexist" method
2833            ({1, 2}.__contains__, (2,)),
2834            # unbound "coexist" method
2835            (set.__contains__, ({1, 2}, 2)),
2836            # built-in class method
2837            (dict.fromkeys, (("a", 1), ("b", 2))),
2838            # built-in static method
2839            (bytearray.maketrans, (b"abc", b"xyz")),
2840            # subclass methods
2841            (Subclass([1,2,2]).count, (2,)),
2842            (Subclass.count, (Subclass([1,2,2]), 2)),
2843            (Subclass.Nested("sweet").count, ("e",)),
2844            (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")),
2845        )
2846        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2847            for method, args in c_methods:
2848                with self.subTest(proto=proto, method=method):
2849                    unpickled = self.loads(self.dumps(method, proto))
2850                    self.assertEqual(method(*args), unpickled(*args))
2851
2852        descriptors = (
2853            bytearray.__dict__['maketrans'],  # built-in static method descriptor
2854            dict.__dict__['fromkeys'],  # built-in class method descriptor
2855        )
2856        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2857            for descr in descriptors:
2858                with self.subTest(proto=proto, descr=descr):
2859                    self.assertRaises(TypeError, self.dumps, descr, proto)
2860
2861    def test_compat_pickle(self):
2862        tests = [
2863            (range(1, 7), '__builtin__', 'xrange'),
2864            (map(int, '123'), 'itertools', 'imap'),
2865            (functools.reduce, '__builtin__', 'reduce'),
2866            (dbm.whichdb, 'whichdb', 'whichdb'),
2867            (Exception(), 'exceptions', 'Exception'),
2868            (collections.UserDict(), 'UserDict', 'IterableUserDict'),
2869            (collections.UserList(), 'UserList', 'UserList'),
2870            (collections.defaultdict(), 'collections', 'defaultdict'),
2871        ]
2872        for val, mod, name in tests:
2873            for proto in range(3):
2874                with self.subTest(type=type(val), proto=proto):
2875                    pickled = self.dumps(val, proto)
2876                    self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled)
2877                    self.assertIs(type(self.loads(pickled)), type(val))
2878
2879    def test_local_lookup_error(self):
2880        # Test that whichmodule() errors out cleanly when looking up
2881        # an assumed globally-reachable object fails.
2882        def f():
2883            pass
2884        # Since the function is local, lookup will fail
2885        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2886            with self.assertRaises((AttributeError, pickle.PicklingError)):
2887                pickletools.dis(self.dumps(f, proto))
2888        # Same without a __module__ attribute (exercises a different path
2889        # in _pickle.c).
2890        del f.__module__
2891        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2892            with self.assertRaises((AttributeError, pickle.PicklingError)):
2893                pickletools.dis(self.dumps(f, proto))
2894        # Yet a different path.
2895        f.__name__ = f.__qualname__
2896        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2897            with self.assertRaises((AttributeError, pickle.PicklingError)):
2898                pickletools.dis(self.dumps(f, proto))
2899
2900    #
2901    # PEP 574 tests below
2902    #
2903
2904    def buffer_like_objects(self):
2905        # Yield buffer-like objects with the bytestring "abcdef" in them
2906        bytestring = b"abcdefgh"
2907        yield ZeroCopyBytes(bytestring)
2908        yield ZeroCopyBytearray(bytestring)
2909        if _testbuffer is not None:
2910            items = list(bytestring)
2911            value = int.from_bytes(bytestring, byteorder='little')
2912            for flags in (0, _testbuffer.ND_WRITABLE):
2913                # 1-D, contiguous
2914                yield PicklableNDArray(items, format='B', shape=(8,),
2915                                       flags=flags)
2916                # 2-D, C-contiguous
2917                yield PicklableNDArray(items, format='B', shape=(4, 2),
2918                                       strides=(2, 1), flags=flags)
2919                # 2-D, Fortran-contiguous
2920                yield PicklableNDArray(items, format='B',
2921                                       shape=(4, 2), strides=(1, 4),
2922                                       flags=flags)
2923
2924    def test_in_band_buffers(self):
2925        # Test in-band buffers (PEP 574)
2926        for obj in self.buffer_like_objects():
2927            for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2928                data = self.dumps(obj, proto)
2929                if obj.c_contiguous and proto >= 5:
2930                    # The raw memory bytes are serialized in physical order
2931                    self.assertIn(b"abcdefgh", data)
2932                self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 0)
2933                if proto >= 5:
2934                    self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data),
2935                                     1 if obj.readonly else 0)
2936                    self.assertEqual(count_opcode(pickle.BYTEARRAY8, data),
2937                                     0 if obj.readonly else 1)
2938                    # Return a true value from buffer_callback should have
2939                    # the same effect
2940                    def buffer_callback(obj):
2941                        return True
2942                    data2 = self.dumps(obj, proto,
2943                                       buffer_callback=buffer_callback)
2944                    self.assertEqual(data2, data)
2945
2946                new = self.loads(data)
2947                # It's a copy
2948                self.assertIsNot(new, obj)
2949                self.assertIs(type(new), type(obj))
2950                self.assertEqual(new, obj)
2951
2952    # XXX Unfortunately cannot test non-contiguous array
2953    # (see comment in PicklableNDArray.__reduce_ex__)
2954
2955    def test_oob_buffers(self):
2956        # Test out-of-band buffers (PEP 574)
2957        for obj in self.buffer_like_objects():
2958            for proto in range(0, 5):
2959                # Need protocol >= 5 for buffer_callback
2960                with self.assertRaises(ValueError):
2961                    self.dumps(obj, proto,
2962                               buffer_callback=[].append)
2963            for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2964                buffers = []
2965                buffer_callback = lambda pb: buffers.append(pb.raw())
2966                data = self.dumps(obj, proto,
2967                                  buffer_callback=buffer_callback)
2968                self.assertNotIn(b"abcdefgh", data)
2969                self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data), 0)
2970                self.assertEqual(count_opcode(pickle.BYTEARRAY8, data), 0)
2971                self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 1)
2972                self.assertEqual(count_opcode(pickle.READONLY_BUFFER, data),
2973                                 1 if obj.readonly else 0)
2974
2975                if obj.c_contiguous:
2976                    self.assertEqual(bytes(buffers[0]), b"abcdefgh")
2977                # Need buffers argument to unpickle properly
2978                with self.assertRaises(pickle.UnpicklingError):
2979                    self.loads(data)
2980
2981                new = self.loads(data, buffers=buffers)
2982                if obj.zero_copy_reconstruct:
2983                    # Zero-copy achieved
2984                    self.assertIs(new, obj)
2985                else:
2986                    self.assertIs(type(new), type(obj))
2987                    self.assertEqual(new, obj)
2988                # Non-sequence buffers accepted too
2989                new = self.loads(data, buffers=iter(buffers))
2990                if obj.zero_copy_reconstruct:
2991                    # Zero-copy achieved
2992                    self.assertIs(new, obj)
2993                else:
2994                    self.assertIs(type(new), type(obj))
2995                    self.assertEqual(new, obj)
2996
2997    def test_oob_buffers_writable_to_readonly(self):
2998        # Test reconstructing readonly object from writable buffer
2999        obj = ZeroCopyBytes(b"foobar")
3000        for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
3001            buffers = []
3002            buffer_callback = buffers.append
3003            data = self.dumps(obj, proto, buffer_callback=buffer_callback)
3004
3005            buffers = map(bytearray, buffers)
3006            new = self.loads(data, buffers=buffers)
3007            self.assertIs(type(new), type(obj))
3008            self.assertEqual(new, obj)
3009
3010    def test_picklebuffer_error(self):
3011        # PickleBuffer forbidden with protocol < 5
3012        pb = pickle.PickleBuffer(b"foobar")
3013        for proto in range(0, 5):
3014            with self.assertRaises(pickle.PickleError):
3015                self.dumps(pb, proto)
3016
3017    def test_buffer_callback_error(self):
3018        def buffer_callback(buffers):
3019            1/0
3020        pb = pickle.PickleBuffer(b"foobar")
3021        with self.assertRaises(ZeroDivisionError):
3022            self.dumps(pb, 5, buffer_callback=buffer_callback)
3023
3024    def test_buffers_error(self):
3025        pb = pickle.PickleBuffer(b"foobar")
3026        for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
3027            data = self.dumps(pb, proto, buffer_callback=[].append)
3028            # Non iterable buffers
3029            with self.assertRaises(TypeError):
3030                self.loads(data, buffers=object())
3031            # Buffer iterable exhausts too early
3032            with self.assertRaises(pickle.UnpicklingError):
3033                self.loads(data, buffers=[])
3034
3035    def test_inband_accept_default_buffers_argument(self):
3036        for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
3037            data_pickled = self.dumps(1, proto, buffer_callback=None)
3038            data = self.loads(data_pickled, buffers=None)
3039
3040    @unittest.skipIf(np is None, "Test needs Numpy")
3041    def test_buffers_numpy(self):
3042        def check_no_copy(x, y):
3043            np.testing.assert_equal(x, y)
3044            self.assertEqual(x.ctypes.data, y.ctypes.data)
3045
3046        def check_copy(x, y):
3047            np.testing.assert_equal(x, y)
3048            self.assertNotEqual(x.ctypes.data, y.ctypes.data)
3049
3050        def check_array(arr):
3051            # In-band
3052            for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
3053                data = self.dumps(arr, proto)
3054                new = self.loads(data)
3055                check_copy(arr, new)
3056            for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
3057                buffer_callback = lambda _: True
3058                data = self.dumps(arr, proto, buffer_callback=buffer_callback)
3059                new = self.loads(data)
3060                check_copy(arr, new)
3061            # Out-of-band
3062            for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
3063                buffers = []
3064                buffer_callback = buffers.append
3065                data = self.dumps(arr, proto, buffer_callback=buffer_callback)
3066                new = self.loads(data, buffers=buffers)
3067                if arr.flags.c_contiguous or arr.flags.f_contiguous:
3068                    check_no_copy(arr, new)
3069                else:
3070                    check_copy(arr, new)
3071
3072        # 1-D
3073        arr = np.arange(6)
3074        check_array(arr)
3075        # 1-D, non-contiguous
3076        check_array(arr[::2])
3077        # 2-D, C-contiguous
3078        arr = np.arange(12).reshape((3, 4))
3079        check_array(arr)
3080        # 2-D, F-contiguous
3081        check_array(arr.T)
3082        # 2-D, non-contiguous
3083        check_array(arr[::2])
3084
3085    def test_evil_class_mutating_dict(self):
3086        # https://github.com/python/cpython/issues/92930
3087        from random import getrandbits
3088
3089        global Bad
3090        class Bad:
3091            def __eq__(self, other):
3092                return ENABLED
3093            def __hash__(self):
3094                return 42
3095            def __reduce__(self):
3096                if getrandbits(6) == 0:
3097                    collection.clear()
3098                return (Bad, ())
3099
3100        for proto in protocols:
3101            for _ in range(20):
3102                ENABLED = False
3103                collection = {Bad(): Bad() for _ in range(20)}
3104                for bad in collection:
3105                    bad.bad = bad
3106                    bad.collection = collection
3107                ENABLED = True
3108                try:
3109                    data = self.dumps(collection, proto)
3110                    self.loads(data)
3111                except RuntimeError as e:
3112                    expected = "changed size during iteration"
3113                    self.assertIn(expected, str(e))
3114
3115    def test_evil_pickler_mutating_collection(self):
3116        # https://github.com/python/cpython/issues/92930
3117        if not hasattr(self, "pickler"):
3118            raise self.skipTest(f"{type(self)} has no associated pickler type")
3119
3120        global Clearer
3121        class Clearer:
3122            pass
3123
3124        def check(collection):
3125            class EvilPickler(self.pickler):
3126                def persistent_id(self, obj):
3127                    if isinstance(obj, Clearer):
3128                        collection.clear()
3129                    return None
3130            pickler = EvilPickler(io.BytesIO(), proto)
3131            try:
3132                pickler.dump(collection)
3133            except RuntimeError as e:
3134                expected = "changed size during iteration"
3135                self.assertIn(expected, str(e))
3136
3137        for proto in protocols:
3138            check([Clearer()])
3139            check([Clearer(), Clearer()])
3140            check({Clearer()})
3141            check({Clearer(), Clearer()})
3142            check({Clearer(): 1})
3143            check({Clearer(): 1, Clearer(): 2})
3144            check({1: Clearer(), 2: Clearer()})
3145
3146
3147class BigmemPickleTests:
3148
3149    # Binary protocols can serialize longs of up to 2 GiB-1
3150
3151    @bigmemtest(size=_2G, memuse=3.6, dry_run=False)
3152    def test_huge_long_32b(self, size):
3153        data = 1 << (8 * size)
3154        try:
3155            for proto in protocols:
3156                if proto < 2:
3157                    continue
3158                with self.subTest(proto=proto):
3159                    with self.assertRaises((ValueError, OverflowError)):
3160                        self.dumps(data, protocol=proto)
3161        finally:
3162            data = None
3163
3164    # Protocol 3 can serialize up to 4 GiB-1 as a bytes object
3165    # (older protocols don't have a dedicated opcode for bytes and are
3166    # too inefficient)
3167
3168    @bigmemtest(size=_2G, memuse=2.5, dry_run=False)
3169    def test_huge_bytes_32b(self, size):
3170        data = b"abcd" * (size // 4)
3171        try:
3172            for proto in protocols:
3173                if proto < 3:
3174                    continue
3175                with self.subTest(proto=proto):
3176                    try:
3177                        pickled = self.dumps(data, protocol=proto)
3178                        header = (pickle.BINBYTES +
3179                                  struct.pack("<I", len(data)))
3180                        data_start = pickled.index(data)
3181                        self.assertEqual(
3182                            header,
3183                            pickled[data_start-len(header):data_start])
3184                    finally:
3185                        pickled = None
3186        finally:
3187            data = None
3188
3189    @bigmemtest(size=_4G, memuse=2.5, dry_run=False)
3190    def test_huge_bytes_64b(self, size):
3191        data = b"acbd" * (size // 4)
3192        try:
3193            for proto in protocols:
3194                if proto < 3:
3195                    continue
3196                with self.subTest(proto=proto):
3197                    if proto == 3:
3198                        # Protocol 3 does not support large bytes objects.
3199                        # Verify that we do not crash when processing one.
3200                        with self.assertRaises((ValueError, OverflowError)):
3201                            self.dumps(data, protocol=proto)
3202                        continue
3203                    try:
3204                        pickled = self.dumps(data, protocol=proto)
3205                        header = (pickle.BINBYTES8 +
3206                                  struct.pack("<Q", len(data)))
3207                        data_start = pickled.index(data)
3208                        self.assertEqual(
3209                            header,
3210                            pickled[data_start-len(header):data_start])
3211                    finally:
3212                        pickled = None
3213        finally:
3214            data = None
3215
3216    # All protocols use 1-byte per printable ASCII character; we add another
3217    # byte because the encoded form has to be copied into the internal buffer.
3218
3219    @bigmemtest(size=_2G, memuse=8, dry_run=False)
3220    def test_huge_str_32b(self, size):
3221        data = "abcd" * (size // 4)
3222        try:
3223            for proto in protocols:
3224                if proto == 0:
3225                    continue
3226                with self.subTest(proto=proto):
3227                    try:
3228                        pickled = self.dumps(data, protocol=proto)
3229                        header = (pickle.BINUNICODE +
3230                                  struct.pack("<I", len(data)))
3231                        data_start = pickled.index(b'abcd')
3232                        self.assertEqual(
3233                            header,
3234                            pickled[data_start-len(header):data_start])
3235                        self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
3236                                          pickled.index(b"abcd")), len(data))
3237                    finally:
3238                        pickled = None
3239        finally:
3240            data = None
3241
3242    # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes
3243    # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge
3244    # unicode strings however.
3245
3246    @bigmemtest(size=_4G, memuse=8, dry_run=False)
3247    def test_huge_str_64b(self, size):
3248        data = "abcd" * (size // 4)
3249        try:
3250            for proto in protocols:
3251                if proto == 0:
3252                    continue
3253                with self.subTest(proto=proto):
3254                    if proto < 4:
3255                        with self.assertRaises((ValueError, OverflowError)):
3256                            self.dumps(data, protocol=proto)
3257                        continue
3258                    try:
3259                        pickled = self.dumps(data, protocol=proto)
3260                        header = (pickle.BINUNICODE8 +
3261                                  struct.pack("<Q", len(data)))
3262                        data_start = pickled.index(b'abcd')
3263                        self.assertEqual(
3264                            header,
3265                            pickled[data_start-len(header):data_start])
3266                        self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
3267                                          pickled.index(b"abcd")), len(data))
3268                    finally:
3269                        pickled = None
3270        finally:
3271            data = None
3272
3273
3274# Test classes for reduce_ex
3275
3276class REX_one(object):
3277    """No __reduce_ex__ here, but inheriting it from object"""
3278    _reduce_called = 0
3279    def __reduce__(self):
3280        self._reduce_called = 1
3281        return REX_one, ()
3282
3283class REX_two(object):
3284    """No __reduce__ here, but inheriting it from object"""
3285    _proto = None
3286    def __reduce_ex__(self, proto):
3287        self._proto = proto
3288        return REX_two, ()
3289
3290class REX_three(object):
3291    _proto = None
3292    def __reduce_ex__(self, proto):
3293        self._proto = proto
3294        return REX_two, ()
3295    def __reduce__(self):
3296        raise TestFailed("This __reduce__ shouldn't be called")
3297
3298class REX_four(object):
3299    """Calling base class method should succeed"""
3300    _proto = None
3301    def __reduce_ex__(self, proto):
3302        self._proto = proto
3303        return object.__reduce_ex__(self, proto)
3304
3305class REX_five(object):
3306    """This one used to fail with infinite recursion"""
3307    _reduce_called = 0
3308    def __reduce__(self):
3309        self._reduce_called = 1
3310        return object.__reduce__(self)
3311
3312class REX_six(object):
3313    """This class is used to check the 4th argument (list iterator) of
3314    the reduce protocol.
3315    """
3316    def __init__(self, items=None):
3317        self.items = items if items is not None else []
3318    def __eq__(self, other):
3319        return type(self) is type(other) and self.items == other.items
3320    def append(self, item):
3321        self.items.append(item)
3322    def __reduce__(self):
3323        return type(self), (), None, iter(self.items), None
3324
3325class REX_seven(object):
3326    """This class is used to check the 5th argument (dict iterator) of
3327    the reduce protocol.
3328    """
3329    def __init__(self, table=None):
3330        self.table = table if table is not None else {}
3331    def __eq__(self, other):
3332        return type(self) is type(other) and self.table == other.table
3333    def __setitem__(self, key, value):
3334        self.table[key] = value
3335    def __reduce__(self):
3336        return type(self), (), None, None, iter(self.table.items())
3337
3338class REX_state(object):
3339    """This class is used to check the 3th argument (state) of
3340    the reduce protocol.
3341    """
3342    def __init__(self, state=None):
3343        self.state = state
3344    def __eq__(self, other):
3345        return type(self) is type(other) and self.state == other.state
3346    def __setstate__(self, state):
3347        self.state = state
3348    def __reduce__(self):
3349        return type(self), (), self.state
3350
3351
3352# Test classes for newobj
3353
3354class MyInt(int):
3355    sample = 1
3356
3357class MyFloat(float):
3358    sample = 1.0
3359
3360class MyComplex(complex):
3361    sample = 1.0 + 0.0j
3362
3363class MyStr(str):
3364    sample = "hello"
3365
3366class MyUnicode(str):
3367    sample = "hello \u1234"
3368
3369class MyTuple(tuple):
3370    sample = (1, 2, 3)
3371
3372class MyList(list):
3373    sample = [1, 2, 3]
3374
3375class MyDict(dict):
3376    sample = {"a": 1, "b": 2}
3377
3378class MySet(set):
3379    sample = {"a", "b"}
3380
3381class MyFrozenSet(frozenset):
3382    sample = frozenset({"a", "b"})
3383
3384myclasses = [MyInt, MyFloat,
3385             MyComplex,
3386             MyStr, MyUnicode,
3387             MyTuple, MyList, MyDict, MySet, MyFrozenSet]
3388
3389class MyIntWithNew(int):
3390    def __new__(cls, value):
3391        raise AssertionError
3392
3393class MyIntWithNew2(MyIntWithNew):
3394    __new__ = int.__new__
3395
3396
3397class SlotList(MyList):
3398    __slots__ = ["foo"]
3399
3400class SimpleNewObj(int):
3401    def __init__(self, *args, **kwargs):
3402        # raise an error, to make sure this isn't called
3403        raise TypeError("SimpleNewObj.__init__() didn't expect to get called")
3404    def __eq__(self, other):
3405        return int(self) == int(other) and self.__dict__ == other.__dict__
3406
3407class ComplexNewObj(SimpleNewObj):
3408    def __getnewargs__(self):
3409        return ('%X' % self, 16)
3410
3411class ComplexNewObjEx(SimpleNewObj):
3412    def __getnewargs_ex__(self):
3413        return ('%X' % self,), {'base': 16}
3414
3415class BadGetattr:
3416    def __getattr__(self, key):
3417        self.foo
3418
3419
3420class AbstractPickleModuleTests:
3421
3422    def test_dump_closed_file(self):
3423        f = open(TESTFN, "wb")
3424        try:
3425            f.close()
3426            self.assertRaises(ValueError, self.dump, 123, f)
3427        finally:
3428            os_helper.unlink(TESTFN)
3429
3430    def test_load_closed_file(self):
3431        f = open(TESTFN, "wb")
3432        try:
3433            f.close()
3434            self.assertRaises(ValueError, self.dump, 123, f)
3435        finally:
3436            os_helper.unlink(TESTFN)
3437
3438    def test_load_from_and_dump_to_file(self):
3439        stream = io.BytesIO()
3440        data = [123, {}, 124]
3441        self.dump(data, stream)
3442        stream.seek(0)
3443        unpickled = self.load(stream)
3444        self.assertEqual(unpickled, data)
3445
3446    def test_highest_protocol(self):
3447        # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
3448        self.assertEqual(pickle.HIGHEST_PROTOCOL, 5)
3449
3450    def test_callapi(self):
3451        f = io.BytesIO()
3452        # With and without keyword arguments
3453        self.dump(123, f, -1)
3454        self.dump(123, file=f, protocol=-1)
3455        self.dumps(123, -1)
3456        self.dumps(123, protocol=-1)
3457        self.Pickler(f, -1)
3458        self.Pickler(f, protocol=-1)
3459
3460    def test_dump_text_file(self):
3461        f = open(TESTFN, "w")
3462        try:
3463            for proto in protocols:
3464                self.assertRaises(TypeError, self.dump, 123, f, proto)
3465        finally:
3466            f.close()
3467            os_helper.unlink(TESTFN)
3468
3469    def test_incomplete_input(self):
3470        s = io.BytesIO(b"X''.")
3471        self.assertRaises((EOFError, struct.error, pickle.UnpicklingError), self.load, s)
3472
3473    def test_bad_init(self):
3474        # Test issue3664 (pickle can segfault from a badly initialized Pickler).
3475        # Override initialization without calling __init__() of the superclass.
3476        class BadPickler(self.Pickler):
3477            def __init__(self): pass
3478
3479        class BadUnpickler(self.Unpickler):
3480            def __init__(self): pass
3481
3482        self.assertRaises(pickle.PicklingError, BadPickler().dump, 0)
3483        self.assertRaises(pickle.UnpicklingError, BadUnpickler().load)
3484
3485    def check_dumps_loads_oob_buffers(self, dumps, loads):
3486        # No need to do the full gamut of tests here, just enough to
3487        # check that dumps() and loads() redirect their arguments
3488        # to the underlying Pickler and Unpickler, respectively.
3489        obj = ZeroCopyBytes(b"foo")
3490
3491        for proto in range(0, 5):
3492            # Need protocol >= 5 for buffer_callback
3493            with self.assertRaises(ValueError):
3494                dumps(obj, protocol=proto,
3495                      buffer_callback=[].append)
3496        for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
3497            buffers = []
3498            buffer_callback = buffers.append
3499            data = dumps(obj, protocol=proto,
3500                         buffer_callback=buffer_callback)
3501            self.assertNotIn(b"foo", data)
3502            self.assertEqual(bytes(buffers[0]), b"foo")
3503            # Need buffers argument to unpickle properly
3504            with self.assertRaises(pickle.UnpicklingError):
3505                loads(data)
3506            new = loads(data, buffers=buffers)
3507            self.assertIs(new, obj)
3508
3509    def test_dumps_loads_oob_buffers(self):
3510        # Test out-of-band buffers (PEP 574) with top-level dumps() and loads()
3511        self.check_dumps_loads_oob_buffers(self.dumps, self.loads)
3512
3513    def test_dump_load_oob_buffers(self):
3514        # Test out-of-band buffers (PEP 574) with top-level dump() and load()
3515        def dumps(obj, **kwargs):
3516            f = io.BytesIO()
3517            self.dump(obj, f, **kwargs)
3518            return f.getvalue()
3519
3520        def loads(data, **kwargs):
3521            f = io.BytesIO(data)
3522            return self.load(f, **kwargs)
3523
3524        self.check_dumps_loads_oob_buffers(dumps, loads)
3525
3526
3527class AbstractPersistentPicklerTests:
3528
3529    # This class defines persistent_id() and persistent_load()
3530    # functions that should be used by the pickler.  All even integers
3531    # are pickled using persistent ids.
3532
3533    def persistent_id(self, object):
3534        if isinstance(object, int) and object % 2 == 0:
3535            self.id_count += 1
3536            return str(object)
3537        elif object == "test_false_value":
3538            self.false_count += 1
3539            return ""
3540        else:
3541            return None
3542
3543    def persistent_load(self, oid):
3544        if not oid:
3545            self.load_false_count += 1
3546            return "test_false_value"
3547        else:
3548            self.load_count += 1
3549            object = int(oid)
3550            assert object % 2 == 0
3551            return object
3552
3553    def test_persistence(self):
3554        L = list(range(10)) + ["test_false_value"]
3555        for proto in protocols:
3556            self.id_count = 0
3557            self.false_count = 0
3558            self.load_false_count = 0
3559            self.load_count = 0
3560            self.assertEqual(self.loads(self.dumps(L, proto)), L)
3561            self.assertEqual(self.id_count, 5)
3562            self.assertEqual(self.false_count, 1)
3563            self.assertEqual(self.load_count, 5)
3564            self.assertEqual(self.load_false_count, 1)
3565
3566
3567class AbstractIdentityPersistentPicklerTests:
3568
3569    def persistent_id(self, obj):
3570        return obj
3571
3572    def persistent_load(self, pid):
3573        return pid
3574
3575    def _check_return_correct_type(self, obj, proto):
3576        unpickled = self.loads(self.dumps(obj, proto))
3577        self.assertIsInstance(unpickled, type(obj))
3578        self.assertEqual(unpickled, obj)
3579
3580    def test_return_correct_type(self):
3581        for proto in protocols:
3582            # Protocol 0 supports only ASCII strings.
3583            if proto == 0:
3584                self._check_return_correct_type("abc", 0)
3585            else:
3586                for obj in [b"abc\n", "abc\n", -1, -1.1 * 0.1, str]:
3587                    self._check_return_correct_type(obj, proto)
3588
3589    def test_protocol0_is_ascii_only(self):
3590        non_ascii_str = "\N{EMPTY SET}"
3591        self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0)
3592        pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.'
3593        self.assertRaises(pickle.UnpicklingError, self.loads, pickled)
3594
3595
3596class AbstractPicklerUnpicklerObjectTests:
3597
3598    pickler_class = None
3599    unpickler_class = None
3600
3601    def setUp(self):
3602        assert self.pickler_class
3603        assert self.unpickler_class
3604
3605    def test_clear_pickler_memo(self):
3606        # To test whether clear_memo() has any effect, we pickle an object,
3607        # then pickle it again without clearing the memo; the two serialized
3608        # forms should be different. If we clear_memo() and then pickle the
3609        # object again, the third serialized form should be identical to the
3610        # first one we obtained.
3611        data = ["abcdefg", "abcdefg", 44]
3612        for proto in protocols:
3613            f = io.BytesIO()
3614            pickler = self.pickler_class(f, proto)
3615
3616            pickler.dump(data)
3617            first_pickled = f.getvalue()
3618
3619            # Reset BytesIO object.
3620            f.seek(0)
3621            f.truncate()
3622
3623            pickler.dump(data)
3624            second_pickled = f.getvalue()
3625
3626            # Reset the Pickler and BytesIO objects.
3627            pickler.clear_memo()
3628            f.seek(0)
3629            f.truncate()
3630
3631            pickler.dump(data)
3632            third_pickled = f.getvalue()
3633
3634            self.assertNotEqual(first_pickled, second_pickled)
3635            self.assertEqual(first_pickled, third_pickled)
3636
3637    def test_priming_pickler_memo(self):
3638        # Verify that we can set the Pickler's memo attribute.
3639        data = ["abcdefg", "abcdefg", 44]
3640        f = io.BytesIO()
3641        pickler = self.pickler_class(f)
3642
3643        pickler.dump(data)
3644        first_pickled = f.getvalue()
3645
3646        f = io.BytesIO()
3647        primed = self.pickler_class(f)
3648        primed.memo = pickler.memo
3649
3650        primed.dump(data)
3651        primed_pickled = f.getvalue()
3652
3653        self.assertNotEqual(first_pickled, primed_pickled)
3654
3655    def test_priming_unpickler_memo(self):
3656        # Verify that we can set the Unpickler's memo attribute.
3657        data = ["abcdefg", "abcdefg", 44]
3658        f = io.BytesIO()
3659        pickler = self.pickler_class(f)
3660
3661        pickler.dump(data)
3662        first_pickled = f.getvalue()
3663
3664        f = io.BytesIO()
3665        primed = self.pickler_class(f)
3666        primed.memo = pickler.memo
3667
3668        primed.dump(data)
3669        primed_pickled = f.getvalue()
3670
3671        unpickler = self.unpickler_class(io.BytesIO(first_pickled))
3672        unpickled_data1 = unpickler.load()
3673
3674        self.assertEqual(unpickled_data1, data)
3675
3676        primed = self.unpickler_class(io.BytesIO(primed_pickled))
3677        primed.memo = unpickler.memo
3678        unpickled_data2 = primed.load()
3679
3680        primed.memo.clear()
3681
3682        self.assertEqual(unpickled_data2, data)
3683        self.assertTrue(unpickled_data2 is unpickled_data1)
3684
3685    def test_reusing_unpickler_objects(self):
3686        data1 = ["abcdefg", "abcdefg", 44]
3687        f = io.BytesIO()
3688        pickler = self.pickler_class(f)
3689        pickler.dump(data1)
3690        pickled1 = f.getvalue()
3691
3692        data2 = ["abcdefg", 44, 44]
3693        f = io.BytesIO()
3694        pickler = self.pickler_class(f)
3695        pickler.dump(data2)
3696        pickled2 = f.getvalue()
3697
3698        f = io.BytesIO()
3699        f.write(pickled1)
3700        f.seek(0)
3701        unpickler = self.unpickler_class(f)
3702        self.assertEqual(unpickler.load(), data1)
3703
3704        f.seek(0)
3705        f.truncate()
3706        f.write(pickled2)
3707        f.seek(0)
3708        self.assertEqual(unpickler.load(), data2)
3709
3710    def _check_multiple_unpicklings(self, ioclass, *, seekable=True):
3711        for proto in protocols:
3712            with self.subTest(proto=proto):
3713                data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len]
3714                f = ioclass()
3715                pickler = self.pickler_class(f, protocol=proto)
3716                pickler.dump(data1)
3717                pickled = f.getvalue()
3718
3719                N = 5
3720                f = ioclass(pickled * N)
3721                unpickler = self.unpickler_class(f)
3722                for i in range(N):
3723                    if seekable:
3724                        pos = f.tell()
3725                    self.assertEqual(unpickler.load(), data1)
3726                    if seekable:
3727                        self.assertEqual(f.tell(), pos + len(pickled))
3728                self.assertRaises(EOFError, unpickler.load)
3729
3730    def test_multiple_unpicklings_seekable(self):
3731        self._check_multiple_unpicklings(io.BytesIO)
3732
3733    def test_multiple_unpicklings_unseekable(self):
3734        self._check_multiple_unpicklings(UnseekableIO, seekable=False)
3735
3736    def test_multiple_unpicklings_minimal(self):
3737        # File-like object that doesn't support peek() and readinto()
3738        # (bpo-39681)
3739        self._check_multiple_unpicklings(MinimalIO, seekable=False)
3740
3741    def test_unpickling_buffering_readline(self):
3742        # Issue #12687: the unpickler's buffering logic could fail with
3743        # text mode opcodes.
3744        data = list(range(10))
3745        for proto in protocols:
3746            for buf_size in range(1, 11):
3747                f = io.BufferedRandom(io.BytesIO(), buffer_size=buf_size)
3748                pickler = self.pickler_class(f, protocol=proto)
3749                pickler.dump(data)
3750                f.seek(0)
3751                unpickler = self.unpickler_class(f)
3752                self.assertEqual(unpickler.load(), data)
3753
3754
3755# Tests for dispatch_table attribute
3756
3757REDUCE_A = 'reduce_A'
3758
3759class AAA(object):
3760    def __reduce__(self):
3761        return str, (REDUCE_A,)
3762
3763class BBB(object):
3764    def __init__(self):
3765        # Add an instance attribute to enable state-saving routines at pickling
3766        # time.
3767        self.a = "some attribute"
3768
3769    def __setstate__(self, state):
3770        self.a = "BBB.__setstate__"
3771
3772
3773def setstate_bbb(obj, state):
3774    """Custom state setter for BBB objects
3775
3776    Such callable may be created by other persons than the ones who created the
3777    BBB class. If passed as the state_setter item of a custom reducer, this
3778    allows for custom state setting behavior of BBB objects. One can think of
3779    it as the analogous of list_setitems or dict_setitems but for foreign
3780    classes/functions.
3781    """
3782    obj.a = "custom state_setter"
3783
3784
3785
3786class AbstractCustomPicklerClass:
3787    """Pickler implementing a reducing hook using reducer_override."""
3788    def reducer_override(self, obj):
3789        obj_name = getattr(obj, "__name__", None)
3790
3791        if obj_name == 'f':
3792            # asking the pickler to save f as 5
3793            return int, (5, )
3794
3795        if obj_name == 'MyClass':
3796            return str, ('some str',)
3797
3798        elif obj_name == 'g':
3799            # in this case, the callback returns an invalid result (not a 2-5
3800            # tuple or a string), the pickler should raise a proper error.
3801            return False
3802
3803        elif obj_name == 'h':
3804            # Simulate a case when the reducer fails. The error should
3805            # be propagated to the original ``dump`` call.
3806            raise ValueError('The reducer just failed')
3807
3808        return NotImplemented
3809
3810class AbstractHookTests:
3811    def test_pickler_hook(self):
3812        # test the ability of a custom, user-defined CPickler subclass to
3813        # override the default reducing routines of any type using the method
3814        # reducer_override
3815
3816        def f():
3817            pass
3818
3819        def g():
3820            pass
3821
3822        def h():
3823            pass
3824
3825        class MyClass:
3826            pass
3827
3828        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
3829            with self.subTest(proto=proto):
3830                bio = io.BytesIO()
3831                p = self.pickler_class(bio, proto)
3832
3833                p.dump([f, MyClass, math.log])
3834                new_f, some_str, math_log = pickle.loads(bio.getvalue())
3835
3836                self.assertEqual(new_f, 5)
3837                self.assertEqual(some_str, 'some str')
3838                # math.log does not have its usual reducer overridden, so the
3839                # custom reduction callback should silently direct the pickler
3840                # to the default pickling by attribute, by returning
3841                # NotImplemented
3842                self.assertIs(math_log, math.log)
3843
3844                with self.assertRaises(pickle.PicklingError):
3845                    p.dump(g)
3846
3847                with self.assertRaisesRegex(
3848                        ValueError, 'The reducer just failed'):
3849                    p.dump(h)
3850
3851    @support.cpython_only
3852    def test_reducer_override_no_reference_cycle(self):
3853        # bpo-39492: reducer_override used to induce a spurious reference cycle
3854        # inside the Pickler object, that could prevent all serialized objects
3855        # from being garbage-collected without explicitly invoking gc.collect.
3856
3857        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
3858            with self.subTest(proto=proto):
3859                def f():
3860                    pass
3861
3862                wr = weakref.ref(f)
3863
3864                bio = io.BytesIO()
3865                p = self.pickler_class(bio, proto)
3866                p.dump(f)
3867                new_f = pickle.loads(bio.getvalue())
3868                assert new_f == 5
3869
3870                del p
3871                del f
3872
3873                self.assertIsNone(wr())
3874
3875
3876class AbstractDispatchTableTests:
3877
3878    def test_default_dispatch_table(self):
3879        # No dispatch_table attribute by default
3880        f = io.BytesIO()
3881        p = self.pickler_class(f, 0)
3882        with self.assertRaises(AttributeError):
3883            p.dispatch_table
3884        self.assertFalse(hasattr(p, 'dispatch_table'))
3885
3886    def test_class_dispatch_table(self):
3887        # A dispatch_table attribute can be specified class-wide
3888        dt = self.get_dispatch_table()
3889
3890        class MyPickler(self.pickler_class):
3891            dispatch_table = dt
3892
3893        def dumps(obj, protocol=None):
3894            f = io.BytesIO()
3895            p = MyPickler(f, protocol)
3896            self.assertEqual(p.dispatch_table, dt)
3897            p.dump(obj)
3898            return f.getvalue()
3899
3900        self._test_dispatch_table(dumps, dt)
3901
3902    def test_instance_dispatch_table(self):
3903        # A dispatch_table attribute can also be specified instance-wide
3904        dt = self.get_dispatch_table()
3905
3906        def dumps(obj, protocol=None):
3907            f = io.BytesIO()
3908            p = self.pickler_class(f, protocol)
3909            p.dispatch_table = dt
3910            self.assertEqual(p.dispatch_table, dt)
3911            p.dump(obj)
3912            return f.getvalue()
3913
3914        self._test_dispatch_table(dumps, dt)
3915
3916    def _test_dispatch_table(self, dumps, dispatch_table):
3917        def custom_load_dump(obj):
3918            return pickle.loads(dumps(obj, 0))
3919
3920        def default_load_dump(obj):
3921            return pickle.loads(pickle.dumps(obj, 0))
3922
3923        # pickling complex numbers using protocol 0 relies on copyreg
3924        # so check pickling a complex number still works
3925        z = 1 + 2j
3926        self.assertEqual(custom_load_dump(z), z)
3927        self.assertEqual(default_load_dump(z), z)
3928
3929        # modify pickling of complex
3930        REDUCE_1 = 'reduce_1'
3931        def reduce_1(obj):
3932            return str, (REDUCE_1,)
3933        dispatch_table[complex] = reduce_1
3934        self.assertEqual(custom_load_dump(z), REDUCE_1)
3935        self.assertEqual(default_load_dump(z), z)
3936
3937        # check picklability of AAA and BBB
3938        a = AAA()
3939        b = BBB()
3940        self.assertEqual(custom_load_dump(a), REDUCE_A)
3941        self.assertIsInstance(custom_load_dump(b), BBB)
3942        self.assertEqual(default_load_dump(a), REDUCE_A)
3943        self.assertIsInstance(default_load_dump(b), BBB)
3944
3945        # modify pickling of BBB
3946        dispatch_table[BBB] = reduce_1
3947        self.assertEqual(custom_load_dump(a), REDUCE_A)
3948        self.assertEqual(custom_load_dump(b), REDUCE_1)
3949        self.assertEqual(default_load_dump(a), REDUCE_A)
3950        self.assertIsInstance(default_load_dump(b), BBB)
3951
3952        # revert pickling of BBB and modify pickling of AAA
3953        REDUCE_2 = 'reduce_2'
3954        def reduce_2(obj):
3955            return str, (REDUCE_2,)
3956        dispatch_table[AAA] = reduce_2
3957        del dispatch_table[BBB]
3958        self.assertEqual(custom_load_dump(a), REDUCE_2)
3959        self.assertIsInstance(custom_load_dump(b), BBB)
3960        self.assertEqual(default_load_dump(a), REDUCE_A)
3961        self.assertIsInstance(default_load_dump(b), BBB)
3962
3963        # End-to-end testing of save_reduce with the state_setter keyword
3964        # argument. This is a dispatch_table test as the primary goal of
3965        # state_setter is to tweak objects reduction behavior.
3966        # In particular, state_setter is useful when the default __setstate__
3967        # behavior is not flexible enough.
3968
3969        # No custom reducer for b has been registered for now, so
3970        # BBB.__setstate__ should be used at unpickling time
3971        self.assertEqual(default_load_dump(b).a, "BBB.__setstate__")
3972
3973        def reduce_bbb(obj):
3974            return BBB, (), obj.__dict__, None, None, setstate_bbb
3975
3976        dispatch_table[BBB] = reduce_bbb
3977
3978        # The custom reducer reduce_bbb includes a state setter, that should
3979        # have priority over BBB.__setstate__
3980        self.assertEqual(custom_load_dump(b).a, "custom state_setter")
3981
3982
3983if __name__ == "__main__":
3984    # Print some stuff that can be used to rewrite DATA{0,1,2}
3985    from pickletools import dis
3986    x = create_data()
3987    for i in range(pickle.HIGHEST_PROTOCOL+1):
3988        p = pickle.dumps(x, i)
3989        print("DATA{0} = (".format(i))
3990        for j in range(0, len(p), 20):
3991            b = bytes(p[j:j+20])
3992            print("    {0!r}".format(b))
3993        print(")")
3994        print()
3995        print("# Disassembly of DATA{0}".format(i))
3996        print("DATA{0}_DIS = \"\"\"\\".format(i))
3997        dis(p)
3998        print("\"\"\"")
3999        print()
4000