1"""The optional bytecode cache system. This is useful if you have very
2complex template situations and the compilation of all those templates
3slows down your application too much.
4
5Situations where this is useful are often forking web applications that
6are initialized on the first request.
7"""
8import errno
9import fnmatch
10import marshal
11import os
12import pickle
13import stat
14import sys
15import tempfile
16from hashlib import sha1
17from io import BytesIO
18
19from .utils import open_if_exists
20
21bc_version = 5
22# Magic bytes to identify Jinja bytecode cache files. Contains the
23# Python major and minor version to avoid loading incompatible bytecode
24# if a project upgrades its Python version.
25bc_magic = (
26    b"j2"
27    + pickle.dumps(bc_version, 2)
28    + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2)
29)
30
31
32class Bucket:
33    """Buckets are used to store the bytecode for one template.  It's created
34    and initialized by the bytecode cache and passed to the loading functions.
35
36    The buckets get an internal checksum from the cache assigned and use this
37    to automatically reject outdated cache material.  Individual bytecode
38    cache subclasses don't have to care about cache invalidation.
39    """
40
41    def __init__(self, environment, key, checksum):
42        self.environment = environment
43        self.key = key
44        self.checksum = checksum
45        self.reset()
46
47    def reset(self):
48        """Resets the bucket (unloads the bytecode)."""
49        self.code = None
50
51    def load_bytecode(self, f):
52        """Loads bytecode from a file or file like object."""
53        # make sure the magic header is correct
54        magic = f.read(len(bc_magic))
55        if magic != bc_magic:
56            self.reset()
57            return
58        # the source code of the file changed, we need to reload
59        checksum = pickle.load(f)
60        if self.checksum != checksum:
61            self.reset()
62            return
63        # if marshal_load fails then we need to reload
64        try:
65            self.code = marshal.load(f)
66        except (EOFError, ValueError, TypeError):
67            self.reset()
68            return
69
70    def write_bytecode(self, f):
71        """Dump the bytecode into the file or file like object passed."""
72        if self.code is None:
73            raise TypeError("can't write empty bucket")
74        f.write(bc_magic)
75        pickle.dump(self.checksum, f, 2)
76        marshal.dump(self.code, f)
77
78    def bytecode_from_string(self, string):
79        """Load bytecode from a string."""
80        self.load_bytecode(BytesIO(string))
81
82    def bytecode_to_string(self):
83        """Return the bytecode as string."""
84        out = BytesIO()
85        self.write_bytecode(out)
86        return out.getvalue()
87
88
89class BytecodeCache:
90    """To implement your own bytecode cache you have to subclass this class
91    and override :meth:`load_bytecode` and :meth:`dump_bytecode`.  Both of
92    these methods are passed a :class:`~jinja2.bccache.Bucket`.
93
94    A very basic bytecode cache that saves the bytecode on the file system::
95
96        from os import path
97
98        class MyCache(BytecodeCache):
99
100            def __init__(self, directory):
101                self.directory = directory
102
103            def load_bytecode(self, bucket):
104                filename = path.join(self.directory, bucket.key)
105                if path.exists(filename):
106                    with open(filename, 'rb') as f:
107                        bucket.load_bytecode(f)
108
109            def dump_bytecode(self, bucket):
110                filename = path.join(self.directory, bucket.key)
111                with open(filename, 'wb') as f:
112                    bucket.write_bytecode(f)
113
114    A more advanced version of a filesystem based bytecode cache is part of
115    Jinja.
116    """
117
118    def load_bytecode(self, bucket):
119        """Subclasses have to override this method to load bytecode into a
120        bucket.  If they are not able to find code in the cache for the
121        bucket, it must not do anything.
122        """
123        raise NotImplementedError()
124
125    def dump_bytecode(self, bucket):
126        """Subclasses have to override this method to write the bytecode
127        from a bucket back to the cache.  If it unable to do so it must not
128        fail silently but raise an exception.
129        """
130        raise NotImplementedError()
131
132    def clear(self):
133        """Clears the cache.  This method is not used by Jinja but should be
134        implemented to allow applications to clear the bytecode cache used
135        by a particular environment.
136        """
137
138    def get_cache_key(self, name, filename=None):
139        """Returns the unique hash key for this template name."""
140        hash = sha1(name.encode("utf-8"))
141        if filename is not None:
142            filename = "|" + filename
143            if isinstance(filename, str):
144                filename = filename.encode("utf-8")
145            hash.update(filename)
146        return hash.hexdigest()
147
148    def get_source_checksum(self, source):
149        """Returns a checksum for the source."""
150        return sha1(source.encode("utf-8")).hexdigest()
151
152    def get_bucket(self, environment, name, filename, source):
153        """Return a cache bucket for the given template.  All arguments are
154        mandatory but filename may be `None`.
155        """
156        key = self.get_cache_key(name, filename)
157        checksum = self.get_source_checksum(source)
158        bucket = Bucket(environment, key, checksum)
159        self.load_bytecode(bucket)
160        return bucket
161
162    def set_bucket(self, bucket):
163        """Put the bucket into the cache."""
164        self.dump_bytecode(bucket)
165
166
167class FileSystemBytecodeCache(BytecodeCache):
168    """A bytecode cache that stores bytecode on the filesystem.  It accepts
169    two arguments: The directory where the cache items are stored and a
170    pattern string that is used to build the filename.
171
172    If no directory is specified a default cache directory is selected.  On
173    Windows the user's temp directory is used, on UNIX systems a directory
174    is created for the user in the system temp directory.
175
176    The pattern can be used to have multiple separate caches operate on the
177    same directory.  The default pattern is ``'__jinja2_%s.cache'``.  ``%s``
178    is replaced with the cache key.
179
180    >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
181
182    This bytecode cache supports clearing of the cache using the clear method.
183    """
184
185    def __init__(self, directory=None, pattern="__jinja2_%s.cache"):
186        if directory is None:
187            directory = self._get_default_cache_dir()
188        self.directory = directory
189        self.pattern = pattern
190
191    def _get_default_cache_dir(self):
192        def _unsafe_dir():
193            raise RuntimeError(
194                "Cannot determine safe temp directory.  You "
195                "need to explicitly provide one."
196            )
197
198        tmpdir = tempfile.gettempdir()
199
200        # On windows the temporary directory is used specific unless
201        # explicitly forced otherwise.  We can just use that.
202        if os.name == "nt":
203            return tmpdir
204        if not hasattr(os, "getuid"):
205            _unsafe_dir()
206
207        dirname = f"_jinja2-cache-{os.getuid()}"
208        actual_dir = os.path.join(tmpdir, dirname)
209
210        try:
211            os.mkdir(actual_dir, stat.S_IRWXU)
212        except OSError as e:
213            if e.errno != errno.EEXIST:
214                raise
215        try:
216            os.chmod(actual_dir, stat.S_IRWXU)
217            actual_dir_stat = os.lstat(actual_dir)
218            if (
219                actual_dir_stat.st_uid != os.getuid()
220                or not stat.S_ISDIR(actual_dir_stat.st_mode)
221                or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU
222            ):
223                _unsafe_dir()
224        except OSError as e:
225            if e.errno != errno.EEXIST:
226                raise
227
228        actual_dir_stat = os.lstat(actual_dir)
229        if (
230            actual_dir_stat.st_uid != os.getuid()
231            or not stat.S_ISDIR(actual_dir_stat.st_mode)
232            or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU
233        ):
234            _unsafe_dir()
235
236        return actual_dir
237
238    def _get_cache_filename(self, bucket):
239        return os.path.join(self.directory, self.pattern % (bucket.key,))
240
241    def load_bytecode(self, bucket):
242        f = open_if_exists(self._get_cache_filename(bucket), "rb")
243        if f is not None:
244            try:
245                bucket.load_bytecode(f)
246            finally:
247                f.close()
248
249    def dump_bytecode(self, bucket):
250        f = open(self._get_cache_filename(bucket), "wb")
251        try:
252            bucket.write_bytecode(f)
253        finally:
254            f.close()
255
256    def clear(self):
257        # imported lazily here because google app-engine doesn't support
258        # write access on the file system and the function does not exist
259        # normally.
260        from os import remove
261
262        files = fnmatch.filter(os.listdir(self.directory), self.pattern % ("*",))
263        for filename in files:
264            try:
265                remove(os.path.join(self.directory, filename))
266            except OSError:
267                pass
268
269
270class MemcachedBytecodeCache(BytecodeCache):
271    """This class implements a bytecode cache that uses a memcache cache for
272    storing the information.  It does not enforce a specific memcache library
273    (tummy's memcache or cmemcache) but will accept any class that provides
274    the minimal interface required.
275
276    Libraries compatible with this class:
277
278    -   `cachelib <https://github.com/pallets/cachelib>`_
279    -   `python-memcached <https://pypi.org/project/python-memcached/>`_
280
281    (Unfortunately the django cache interface is not compatible because it
282    does not support storing binary data, only text. You can however pass
283    the underlying cache client to the bytecode cache which is available
284    as `django.core.cache.cache._client`.)
285
286    The minimal interface for the client passed to the constructor is this:
287
288    .. class:: MinimalClientInterface
289
290        .. method:: set(key, value[, timeout])
291
292            Stores the bytecode in the cache.  `value` is a string and
293            `timeout` the timeout of the key.  If timeout is not provided
294            a default timeout or no timeout should be assumed, if it's
295            provided it's an integer with the number of seconds the cache
296            item should exist.
297
298        .. method:: get(key)
299
300            Returns the value for the cache key.  If the item does not
301            exist in the cache the return value must be `None`.
302
303    The other arguments to the constructor are the prefix for all keys that
304    is added before the actual cache key and the timeout for the bytecode in
305    the cache system.  We recommend a high (or no) timeout.
306
307    This bytecode cache does not support clearing of used items in the cache.
308    The clear method is a no-operation function.
309
310    .. versionadded:: 2.7
311       Added support for ignoring memcache errors through the
312       `ignore_memcache_errors` parameter.
313    """
314
315    def __init__(
316        self,
317        client,
318        prefix="jinja2/bytecode/",
319        timeout=None,
320        ignore_memcache_errors=True,
321    ):
322        self.client = client
323        self.prefix = prefix
324        self.timeout = timeout
325        self.ignore_memcache_errors = ignore_memcache_errors
326
327    def load_bytecode(self, bucket):
328        try:
329            code = self.client.get(self.prefix + bucket.key)
330        except Exception:
331            if not self.ignore_memcache_errors:
332                raise
333            code = None
334        if code is not None:
335            bucket.bytecode_from_string(code)
336
337    def dump_bytecode(self, bucket):
338        args = (self.prefix + bucket.key, bucket.bytecode_to_string())
339        if self.timeout is not None:
340            args += (self.timeout,)
341        try:
342            self.client.set(*args)
343        except Exception:
344            if not self.ignore_memcache_errors:
345                raise
346