1"""Tests to cover the Tools/i18n package"""
2
3import os
4import sys
5import unittest
6from textwrap import dedent
7
8from test.support.script_helper import assert_python_ok
9from test.test_tools import skip_if_missing, toolsdir
10from test.support.os_helper import temp_cwd, temp_dir
11
12
13skip_if_missing()
14
15
16class Test_pygettext(unittest.TestCase):
17    """Tests for the pygettext.py tool"""
18
19    script = os.path.join(toolsdir,'i18n', 'pygettext.py')
20
21    def get_header(self, data):
22        """ utility: return the header of a .po file as a dictionary """
23        headers = {}
24        for line in data.split('\n'):
25            if not line or line.startswith(('#', 'msgid','msgstr')):
26                continue
27            line = line.strip('"')
28            key, val = line.split(':',1)
29            headers[key] = val.strip()
30        return headers
31
32    def get_msgids(self, data):
33        """ utility: return all msgids in .po file as a list of strings """
34        msgids = []
35        reading_msgid = False
36        cur_msgid = []
37        for line in data.split('\n'):
38            if reading_msgid:
39                if line.startswith('"'):
40                    cur_msgid.append(line.strip('"'))
41                else:
42                    msgids.append('\n'.join(cur_msgid))
43                    cur_msgid = []
44                    reading_msgid = False
45                    continue
46            if line.startswith('msgid '):
47                line = line[len('msgid '):]
48                cur_msgid.append(line.strip('"'))
49                reading_msgid = True
50        else:
51            if reading_msgid:
52                msgids.append('\n'.join(cur_msgid))
53
54        return msgids
55
56    def extract_docstrings_from_str(self, module_content):
57        """ utility: return all msgids extracted from module_content """
58        filename = 'test_docstrings.py'
59        with temp_cwd(None) as cwd:
60            with open(filename, 'w', encoding='utf-8') as fp:
61                fp.write(module_content)
62            assert_python_ok(self.script, '-D', filename)
63            with open('messages.pot', encoding='utf-8') as fp:
64                data = fp.read()
65        return self.get_msgids(data)
66
67    def test_header(self):
68        """Make sure the required fields are in the header, according to:
69           http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry
70        """
71        with temp_cwd(None) as cwd:
72            assert_python_ok(self.script)
73            with open('messages.pot', encoding='utf-8') as fp:
74                data = fp.read()
75            header = self.get_header(data)
76
77            self.assertIn("Project-Id-Version", header)
78            self.assertIn("POT-Creation-Date", header)
79            self.assertIn("PO-Revision-Date", header)
80            self.assertIn("Last-Translator", header)
81            self.assertIn("Language-Team", header)
82            self.assertIn("MIME-Version", header)
83            self.assertIn("Content-Type", header)
84            self.assertIn("Content-Transfer-Encoding", header)
85            self.assertIn("Generated-By", header)
86
87            # not clear if these should be required in POT (template) files
88            #self.assertIn("Report-Msgid-Bugs-To", header)
89            #self.assertIn("Language", header)
90
91            #"Plural-Forms" is optional
92
93    @unittest.skipIf(sys.platform.startswith('aix'),
94                     'bpo-29972: broken test on AIX')
95    def test_POT_Creation_Date(self):
96        """ Match the date format from xgettext for POT-Creation-Date """
97        from datetime import datetime
98        with temp_cwd(None) as cwd:
99            assert_python_ok(self.script)
100            with open('messages.pot', encoding='utf-8') as fp:
101                data = fp.read()
102            header = self.get_header(data)
103            creationDate = header['POT-Creation-Date']
104
105            # peel off the escaped newline at the end of string
106            if creationDate.endswith('\\n'):
107                creationDate = creationDate[:-len('\\n')]
108
109            # This will raise if the date format does not exactly match.
110            datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z')
111
112    def test_funcdocstring(self):
113        for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
114            with self.subTest(doc):
115                msgids = self.extract_docstrings_from_str(dedent('''\
116                def foo(bar):
117                    %s
118                ''' % doc))
119                self.assertIn('doc', msgids)
120
121    def test_funcdocstring_bytes(self):
122        msgids = self.extract_docstrings_from_str(dedent('''\
123        def foo(bar):
124            b"""doc"""
125        '''))
126        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
127
128    def test_funcdocstring_fstring(self):
129        msgids = self.extract_docstrings_from_str(dedent('''\
130        def foo(bar):
131            f"""doc"""
132        '''))
133        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
134
135    def test_classdocstring(self):
136        for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
137            with self.subTest(doc):
138                msgids = self.extract_docstrings_from_str(dedent('''\
139                class C:
140                    %s
141                ''' % doc))
142                self.assertIn('doc', msgids)
143
144    def test_classdocstring_bytes(self):
145        msgids = self.extract_docstrings_from_str(dedent('''\
146        class C:
147            b"""doc"""
148        '''))
149        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
150
151    def test_classdocstring_fstring(self):
152        msgids = self.extract_docstrings_from_str(dedent('''\
153        class C:
154            f"""doc"""
155        '''))
156        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
157
158    def test_moduledocstring(self):
159        for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
160            with self.subTest(doc):
161                msgids = self.extract_docstrings_from_str(dedent('''\
162                %s
163                ''' % doc))
164                self.assertIn('doc', msgids)
165
166    def test_moduledocstring_bytes(self):
167        msgids = self.extract_docstrings_from_str(dedent('''\
168        b"""doc"""
169        '''))
170        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
171
172    def test_moduledocstring_fstring(self):
173        msgids = self.extract_docstrings_from_str(dedent('''\
174        f"""doc"""
175        '''))
176        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
177
178    def test_msgid(self):
179        msgids = self.extract_docstrings_from_str(
180                '''_("""doc""" r'str' u"ing")''')
181        self.assertIn('docstring', msgids)
182
183    def test_msgid_bytes(self):
184        msgids = self.extract_docstrings_from_str('_(b"""doc""")')
185        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
186
187    def test_msgid_fstring(self):
188        msgids = self.extract_docstrings_from_str('_(f"""doc""")')
189        self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
190
191    def test_funcdocstring_annotated_args(self):
192        """ Test docstrings for functions with annotated args """
193        msgids = self.extract_docstrings_from_str(dedent('''\
194        def foo(bar: str):
195            """doc"""
196        '''))
197        self.assertIn('doc', msgids)
198
199    def test_funcdocstring_annotated_return(self):
200        """ Test docstrings for functions with annotated return type """
201        msgids = self.extract_docstrings_from_str(dedent('''\
202        def foo(bar) -> str:
203            """doc"""
204        '''))
205        self.assertIn('doc', msgids)
206
207    def test_funcdocstring_defvalue_args(self):
208        """ Test docstring for functions with default arg values """
209        msgids = self.extract_docstrings_from_str(dedent('''\
210        def foo(bar=()):
211            """doc"""
212        '''))
213        self.assertIn('doc', msgids)
214
215    def test_funcdocstring_multiple_funcs(self):
216        """ Test docstring extraction for multiple functions combining
217        annotated args, annotated return types and default arg values
218        """
219        msgids = self.extract_docstrings_from_str(dedent('''\
220        def foo1(bar: tuple=()) -> str:
221            """doc1"""
222
223        def foo2(bar: List[1:2]) -> (lambda x: x):
224            """doc2"""
225
226        def foo3(bar: 'func'=lambda x: x) -> {1: 2}:
227            """doc3"""
228        '''))
229        self.assertIn('doc1', msgids)
230        self.assertIn('doc2', msgids)
231        self.assertIn('doc3', msgids)
232
233    def test_classdocstring_early_colon(self):
234        """ Test docstring extraction for a class with colons occurring within
235        the parentheses.
236        """
237        msgids = self.extract_docstrings_from_str(dedent('''\
238        class D(L[1:2], F({1: 2}), metaclass=M(lambda x: x)):
239            """doc"""
240        '''))
241        self.assertIn('doc', msgids)
242
243    def test_calls_in_fstrings(self):
244        msgids = self.extract_docstrings_from_str(dedent('''\
245        f"{_('foo bar')}"
246        '''))
247        self.assertIn('foo bar', msgids)
248
249    def test_calls_in_fstrings_raw(self):
250        msgids = self.extract_docstrings_from_str(dedent('''\
251        rf"{_('foo bar')}"
252        '''))
253        self.assertIn('foo bar', msgids)
254
255    def test_calls_in_fstrings_nested(self):
256        msgids = self.extract_docstrings_from_str(dedent('''\
257        f"""{f'{_("foo bar")}'}"""
258        '''))
259        self.assertIn('foo bar', msgids)
260
261    def test_calls_in_fstrings_attribute(self):
262        msgids = self.extract_docstrings_from_str(dedent('''\
263        f"{obj._('foo bar')}"
264        '''))
265        self.assertIn('foo bar', msgids)
266
267    def test_calls_in_fstrings_with_call_on_call(self):
268        msgids = self.extract_docstrings_from_str(dedent('''\
269        f"{type(str)('foo bar')}"
270        '''))
271        self.assertNotIn('foo bar', msgids)
272
273    def test_calls_in_fstrings_with_format(self):
274        msgids = self.extract_docstrings_from_str(dedent('''\
275        f"{_('foo {bar}').format(bar='baz')}"
276        '''))
277        self.assertIn('foo {bar}', msgids)
278
279    def test_calls_in_fstrings_with_wrong_input_1(self):
280        msgids = self.extract_docstrings_from_str(dedent('''\
281        f"{_(f'foo {bar}')}"
282        '''))
283        self.assertFalse([msgid for msgid in msgids if 'foo {bar}' in msgid])
284
285    def test_calls_in_fstrings_with_wrong_input_2(self):
286        msgids = self.extract_docstrings_from_str(dedent('''\
287        f"{_(1)}"
288        '''))
289        self.assertNotIn(1, msgids)
290
291    def test_calls_in_fstring_with_multiple_args(self):
292        msgids = self.extract_docstrings_from_str(dedent('''\
293        f"{_('foo', 'bar')}"
294        '''))
295        self.assertNotIn('foo', msgids)
296        self.assertNotIn('bar', msgids)
297
298    def test_calls_in_fstring_with_keyword_args(self):
299        msgids = self.extract_docstrings_from_str(dedent('''\
300        f"{_('foo', bar='baz')}"
301        '''))
302        self.assertNotIn('foo', msgids)
303        self.assertNotIn('bar', msgids)
304        self.assertNotIn('baz', msgids)
305
306    def test_calls_in_fstring_with_partially_wrong_expression(self):
307        msgids = self.extract_docstrings_from_str(dedent('''\
308        f"{_(f'foo') + _('bar')}"
309        '''))
310        self.assertNotIn('foo', msgids)
311        self.assertIn('bar', msgids)
312
313    def test_files_list(self):
314        """Make sure the directories are inspected for source files
315           bpo-31920
316        """
317        text1 = 'Text to translate1'
318        text2 = 'Text to translate2'
319        text3 = 'Text to ignore'
320        with temp_cwd(None), temp_dir(None) as sdir:
321            os.mkdir(os.path.join(sdir, 'pypkg'))
322            with open(os.path.join(sdir, 'pypkg', 'pymod.py'), 'w',
323                      encoding='utf-8') as sfile:
324                sfile.write(f'_({text1!r})')
325            os.mkdir(os.path.join(sdir, 'pkg.py'))
326            with open(os.path.join(sdir, 'pkg.py', 'pymod2.py'), 'w',
327                      encoding='utf-8') as sfile:
328                sfile.write(f'_({text2!r})')
329            os.mkdir(os.path.join(sdir, 'CVS'))
330            with open(os.path.join(sdir, 'CVS', 'pymod3.py'), 'w',
331                      encoding='utf-8') as sfile:
332                sfile.write(f'_({text3!r})')
333            assert_python_ok(self.script, sdir)
334            with open('messages.pot', encoding='utf-8') as fp:
335                data = fp.read()
336            self.assertIn(f'msgid "{text1}"', data)
337            self.assertIn(f'msgid "{text2}"', data)
338            self.assertNotIn(text3, data)
339