1#
2# Copyright 2015 Google Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Integration tests for uploading and downloading to GCS.
17
18These tests exercise most of the corner cases for upload/download of
19files in apitools, via GCS. There are no performance tests here yet.
20"""
21
22import json
23import os
24import unittest
25
26import six
27
28from apitools.base.py import exceptions
29import storage
30
31_CLIENT = None
32
33
34def _GetClient():
35    global _CLIENT  # pylint: disable=global-statement
36    if _CLIENT is None:
37        _CLIENT = storage.StorageV1()
38    return _CLIENT
39
40
41class DownloadsTest(unittest.TestCase):
42    _DEFAULT_BUCKET = 'apitools'
43    _TESTDATA_PREFIX = 'testdata'
44
45    def setUp(self):
46        self.__client = _GetClient()
47        self.__ResetDownload()
48
49    def __ResetDownload(self, auto_transfer=False):
50        self.__buffer = six.StringIO()
51        self.__download = storage.Download.FromStream(
52            self.__buffer, auto_transfer=auto_transfer)
53
54    def __GetTestdataFileContents(self, filename):
55        file_path = os.path.join(
56            os.path.dirname(__file__), self._TESTDATA_PREFIX, filename)
57        file_contents = open(file_path).read()
58        self.assertIsNotNone(
59            file_contents, msg=('Could not read file %s' % filename))
60        return file_contents
61
62    @classmethod
63    def __GetRequest(cls, filename):
64        object_name = os.path.join(cls._TESTDATA_PREFIX, filename)
65        return storage.StorageObjectsGetRequest(
66            bucket=cls._DEFAULT_BUCKET, object=object_name)
67
68    def __GetFile(self, request):
69        response = self.__client.objects.Get(request, download=self.__download)
70        self.assertIsNone(response, msg=(
71            'Unexpected nonempty response for file download: %s' % response))
72
73    def __GetAndStream(self, request):
74        self.__GetFile(request)
75        self.__download.StreamInChunks()
76
77    def testZeroBytes(self):
78        request = self.__GetRequest('zero_byte_file')
79        self.__GetAndStream(request)
80        self.assertEqual(0, self.__buffer.tell())
81
82    def testObjectDoesNotExist(self):
83        self.__ResetDownload(auto_transfer=True)
84        with self.assertRaises(exceptions.HttpError):
85            self.__GetFile(self.__GetRequest('nonexistent_file'))
86
87    def testAutoTransfer(self):
88        self.__ResetDownload(auto_transfer=True)
89        self.__GetFile(self.__GetRequest('fifteen_byte_file'))
90        file_contents = self.__GetTestdataFileContents('fifteen_byte_file')
91        self.assertEqual(15, self.__buffer.tell())
92        self.__buffer.seek(0)
93        self.assertEqual(file_contents, self.__buffer.read())
94
95    def testFilenameWithSpaces(self):
96        self.__ResetDownload(auto_transfer=True)
97        self.__GetFile(self.__GetRequest('filename with spaces'))
98        # NOTE(craigcitro): We add _ here to make this play nice with blaze.
99        file_contents = self.__GetTestdataFileContents('filename_with_spaces')
100        self.assertEqual(15, self.__buffer.tell())
101        self.__buffer.seek(0)
102        self.assertEqual(file_contents, self.__buffer.read())
103
104    def testGetRange(self):
105        # TODO(craigcitro): Test about a thousand more corner cases.
106        file_contents = self.__GetTestdataFileContents('fifteen_byte_file')
107        self.__GetFile(self.__GetRequest('fifteen_byte_file'))
108        self.__download.GetRange(5, 10)
109        self.assertEqual(6, self.__buffer.tell())
110        self.__buffer.seek(0)
111        self.assertEqual(file_contents[5:11], self.__buffer.read())
112
113    def testGetRangeWithNegativeStart(self):
114        file_contents = self.__GetTestdataFileContents('fifteen_byte_file')
115        self.__GetFile(self.__GetRequest('fifteen_byte_file'))
116        self.__download.GetRange(-3)
117        self.assertEqual(3, self.__buffer.tell())
118        self.__buffer.seek(0)
119        self.assertEqual(file_contents[-3:], self.__buffer.read())
120
121    def testGetRangeWithPositiveStart(self):
122        file_contents = self.__GetTestdataFileContents('fifteen_byte_file')
123        self.__GetFile(self.__GetRequest('fifteen_byte_file'))
124        self.__download.GetRange(2)
125        self.assertEqual(13, self.__buffer.tell())
126        self.__buffer.seek(0)
127        self.assertEqual(file_contents[2:15], self.__buffer.read())
128
129    def testSmallChunksizes(self):
130        file_contents = self.__GetTestdataFileContents('fifteen_byte_file')
131        request = self.__GetRequest('fifteen_byte_file')
132        for chunksize in (2, 3, 15, 100):
133            self.__ResetDownload()
134            self.__download.chunksize = chunksize
135            self.__GetAndStream(request)
136            self.assertEqual(15, self.__buffer.tell())
137            self.__buffer.seek(0)
138            self.assertEqual(file_contents, self.__buffer.read(15))
139
140    def testLargeFileChunksizes(self):
141        request = self.__GetRequest('thirty_meg_file')
142        for chunksize in (1048576, 40 * 1048576):
143            self.__ResetDownload()
144            self.__download.chunksize = chunksize
145            self.__GetAndStream(request)
146            self.__buffer.seek(0)
147
148    def testAutoGzipObject(self):
149        # TODO(craigcitro): Move this to a new object once we have a more
150        # permanent one, see: http://b/12250275
151        request = storage.StorageObjectsGetRequest(
152            bucket='ottenl-gzip', object='50K.txt')
153        # First, try without auto-transfer.
154        self.__GetFile(request)
155        self.assertEqual(0, self.__buffer.tell())
156        self.__download.StreamInChunks()
157        self.assertEqual(50000, self.__buffer.tell())
158        # Next, try with auto-transfer.
159        self.__ResetDownload(auto_transfer=True)
160        self.__GetFile(request)
161        self.assertEqual(50000, self.__buffer.tell())
162
163    def testSmallGzipObject(self):
164        request = self.__GetRequest('zero-gzipd.html')
165        self.__GetFile(request)
166        self.assertEqual(0, self.__buffer.tell())
167        additional_headers = {'accept-encoding': 'gzip, deflate'}
168        self.__download.StreamInChunks(additional_headers=additional_headers)
169        self.assertEqual(0, self.__buffer.tell())
170
171    def testSerializedDownload(self):
172
173        def _ProgressCallback(unused_response, download_object):
174            print('Progress %s' % download_object.progress)
175
176        file_contents = self.__GetTestdataFileContents('fifteen_byte_file')
177        object_name = os.path.join(self._TESTDATA_PREFIX, 'fifteen_byte_file')
178        request = storage.StorageObjectsGetRequest(
179            bucket=self._DEFAULT_BUCKET, object=object_name)
180        response = self.__client.objects.Get(request)
181        # pylint: disable=attribute-defined-outside-init
182        self.__buffer = six.StringIO()
183        download_data = json.dumps({
184            'auto_transfer': False,
185            'progress': 0,
186            'total_size': response.size,
187            'url': response.mediaLink,
188        })
189        self.__download = storage.Download.FromData(
190            self.__buffer, download_data, http=self.__client.http)
191        self.__download.StreamInChunks(callback=_ProgressCallback)
192        self.assertEqual(15, self.__buffer.tell())
193        self.__buffer.seek(0)
194        self.assertEqual(file_contents, self.__buffer.read(15))
195
196if __name__ == '__main__':
197    unittest.main()
198