xref: /aosp_15_r20/external/tensorflow/tensorflow/python/debug/cli/evaluator_test.py (revision b6fb3261f9314811a0f4371741dbb8839866f948) 
1# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Tests for arbitrary expression evaluator."""
16import numpy as np
17
18from tensorflow.python.debug.cli import evaluator
19from tensorflow.python.debug.lib import debug_data
20from tensorflow.python.framework import test_util
21from tensorflow.python.platform import test
22
23
24class ParseDebugTensorNameTest(test_util.TensorFlowTestCase):
25
26  def testParseNamesWithoutPrefixOrSuffix(self):
27    device_name, node_name, output_slot, debug_op, exec_index = (
28        evaluator._parse_debug_tensor_name("foo:1"))
29    self.assertIsNone(device_name)
30    self.assertEqual("foo", node_name)
31    self.assertEqual(1, output_slot)
32    self.assertEqual("DebugIdentity", debug_op)
33    self.assertEqual(0, exec_index)
34
35    device_name, node_name, output_slot, debug_op, exec_index = (
36        evaluator._parse_debug_tensor_name("hidden_0/Weights:0"))
37    self.assertIsNone(device_name)
38    self.assertEqual("hidden_0/Weights", node_name)
39    self.assertEqual(0, output_slot)
40    self.assertEqual("DebugIdentity", debug_op)
41    self.assertEqual(0, exec_index)
42
43  def testParseNamesWithoutPrefixWithDebugOpSuffix(self):
44    device_name, node_name, output_slot, debug_op, exec_index = (
45        evaluator._parse_debug_tensor_name("foo:1:DebugNanCount"))
46    self.assertIsNone(device_name)
47    self.assertEqual("foo", node_name)
48    self.assertEqual(1, output_slot)
49    self.assertEqual("DebugNanCount", debug_op)
50    self.assertEqual(0, exec_index)
51
52    device_name, node_name, output_slot, debug_op, exec_index = (
53        evaluator._parse_debug_tensor_name(
54            "hidden_0/Weights:0:DebugNumericSummary"))
55    self.assertIsNone(device_name)
56    self.assertEqual("hidden_0/Weights", node_name)
57    self.assertEqual(0, output_slot)
58    self.assertEqual("DebugNumericSummary", debug_op)
59    self.assertEqual(0, exec_index)
60
61  def testParseNamesWithDeviceNamePrefixWithoutDebugOpSuffix(self):
62    device_name, node_name, output_slot, debug_op, exec_index = (
63        evaluator._parse_debug_tensor_name(
64            "/job:ps/replica:0/task:2/cpu:0:foo:1"))
65    self.assertEqual("/job:ps/replica:0/task:2/cpu:0", device_name)
66    self.assertEqual("foo", node_name)
67    self.assertEqual(1, output_slot)
68    self.assertEqual("DebugIdentity", debug_op)
69    self.assertEqual(0, exec_index)
70
71    device_name, node_name, output_slot, debug_op, exec_index = (
72        evaluator._parse_debug_tensor_name(
73            "/job:worker/replica:0/task:3/gpu:0:hidden_0/Weights:0"))
74    self.assertEqual("/job:worker/replica:0/task:3/gpu:0", device_name)
75    self.assertEqual("hidden_0/Weights", node_name)
76    self.assertEqual(0, output_slot)
77    self.assertEqual("DebugIdentity", debug_op)
78    self.assertEqual(0, exec_index)
79
80  def testParseNamesWithDeviceNamePrefixWithDebugOpSuffix(self):
81    device_name, node_name, output_slot, debug_op, exec_index = (
82        evaluator._parse_debug_tensor_name(
83            "/job:ps/replica:0/task:2/cpu:0:foo:1:DebugNanCount"))
84    self.assertEqual("/job:ps/replica:0/task:2/cpu:0", device_name)
85    self.assertEqual("foo", node_name)
86    self.assertEqual(1, output_slot)
87    self.assertEqual("DebugNanCount", debug_op)
88    self.assertEqual(0, exec_index)
89
90    device_name, node_name, output_slot, debug_op, exec_index = (
91        evaluator._parse_debug_tensor_name(
92            "/job:worker/replica:0/task:3/gpu:0:"
93            "hidden_0/Weights:0:DebugNumericSummary"))
94    self.assertEqual("/job:worker/replica:0/task:3/gpu:0", device_name)
95    self.assertEqual("hidden_0/Weights", node_name)
96    self.assertEqual(0, output_slot)
97    self.assertEqual("DebugNumericSummary", debug_op)
98    self.assertEqual(0, exec_index)
99
100  def testParseMalformedDebugTensorName(self):
101    with self.assertRaisesRegex(
102        ValueError,
103        r"The debug tensor name in the to-be-evaluated expression is "
104        r"malformed:"):
105      evaluator._parse_debug_tensor_name(
106          "/job:ps/replica:0/task:2/cpu:0:foo:1:DebugNanCount:1337")
107
108    with self.assertRaisesRegex(
109        ValueError,
110        r"The debug tensor name in the to-be-evaluated expression is "
111        r"malformed:"):
112      evaluator._parse_debug_tensor_name(
113          "/job:ps/replica:0/cpu:0:foo:1:DebugNanCount")
114
115    with self.assertRaises(ValueError):
116      evaluator._parse_debug_tensor_name(
117          "foo:1:DebugNanCount[]")
118
119    with self.assertRaises(ValueError):
120      evaluator._parse_debug_tensor_name(
121          "foo:1[DebugNanCount]")
122
123  def testParseNamesWithExecIndex(self):
124    device_name, node_name, output_slot, debug_op, exec_index = (
125        evaluator._parse_debug_tensor_name("foo:1[20]"))
126    self.assertIsNone(device_name)
127    self.assertEqual("foo", node_name)
128    self.assertEqual(1, output_slot)
129    self.assertEqual("DebugIdentity", debug_op)
130    self.assertEqual(20, exec_index)
131
132    device_name, node_name, output_slot, debug_op, exec_index = (
133        evaluator._parse_debug_tensor_name("hidden_0/Weights:0[3]"))
134    self.assertIsNone(device_name)
135    self.assertEqual("hidden_0/Weights", node_name)
136    self.assertEqual(0, output_slot)
137    self.assertEqual("DebugIdentity", debug_op)
138    self.assertEqual(3, exec_index)
139
140
141class EvaluatorTest(test_util.TensorFlowTestCase):
142
143  def testEvaluateSingleTensor(self):
144    dump = test.mock.MagicMock()
145    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
146      del node_name, output_slot, debug_op, device_name  # Unused.
147      return [np.array([[1.0, 2.0, 3.0]])]
148
149    with test.mock.patch.object(
150        dump, "get_tensors", side_effect=fake_get_tensors):
151      ev = evaluator.ExpressionEvaluator(dump)
152      self.assertEqual(3, ev.evaluate("np.size(`a:0`)"))
153
154      # Whitespace in backticks should be tolerated.
155      self.assertEqual(3, ev.evaluate("np.size(` a:0 `)"))
156
157  def testEvaluateTwoTensors(self):
158    dump = test.mock.MagicMock()
159    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
160      del debug_op, device_name  # Unused.
161      if node_name == "a" and output_slot == 0:
162        return [np.array([[1.0, -2.0], [0.0, 1.0]])]
163      elif node_name == "b" and output_slot == 0:
164        return [np.array([[-1.0], [1.0]])]
165
166    with test.mock.patch.object(
167        dump, "get_tensors", side_effect=fake_get_tensors):
168      ev = evaluator.ExpressionEvaluator(dump)
169      self.assertAllClose([[-3.0], [1.0]],
170                          ev.evaluate("np.matmul(`a:0`, `b:0`)"))
171      self.assertAllClose(
172          [[-4.0], [2.0]], ev.evaluate("np.matmul(`a:0`, `b:0`) + `b:0`"))
173
174  def testEvaluateNoneExistentTensorGeneratesError(self):
175    dump = test.mock.MagicMock()
176    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
177      del node_name, output_slot, debug_op, device_name  # Unused.
178      raise debug_data.WatchKeyDoesNotExistInDebugDumpDirError()
179
180    with test.mock.patch.object(
181        dump, "get_tensors", side_effect=fake_get_tensors):
182      ev = evaluator.ExpressionEvaluator(dump)
183      with self.assertRaisesRegex(
184          ValueError, "Eval failed due to the value of .* being unavailable"):
185        ev.evaluate("np.matmul(`a:0`, `b:0`)")
186
187  def testEvaluateWithMultipleDevicesContainingTheSameTensorName(self):
188    dump = test.mock.MagicMock()
189    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
190      del output_slot, debug_op  # Unused.
191      if node_name == "a" and device_name is None:
192        raise ValueError(
193            "There are multiple (2) devices with nodes named 'a' but "
194            "device_name is not specified")
195      elif (node_name == "a" and
196            device_name == "/job:worker/replica:0/task:0/cpu:0"):
197        return [np.array(10.0)]
198      elif (node_name == "a" and
199            device_name == "/job:worker/replica:0/task:1/cpu:0"):
200        return [np.array(20.0)]
201
202    with test.mock.patch.object(
203        dump, "get_tensors", side_effect=fake_get_tensors):
204      ev = evaluator.ExpressionEvaluator(dump)
205      with self.assertRaisesRegex(ValueError, r"multiple \(2\) devices"):
206        ev.evaluate("`a:0` + `a:0`")
207
208      self.assertAllClose(
209          30.0,
210          ev.evaluate("`/job:worker/replica:0/task:0/cpu:0:a:0` + "
211                      "`/job:worker/replica:0/task:1/cpu:0:a:0`"))
212
213  def testEvaluateWithNonDefaultDebugOp(self):
214    dump = test.mock.MagicMock()
215    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
216      del device_name  # Unused.
217      if node_name == "a" and output_slot == 0 and debug_op == "DebugIdentity":
218        return [np.array([[-1.0], [1.0]])]
219      elif node_name == "a" and output_slot == 0 and debug_op == "DebugFoo":
220        return [np.array([[-2.0, 2.0]])]
221
222    with test.mock.patch.object(
223        dump, "get_tensors", side_effect=fake_get_tensors):
224      ev = evaluator.ExpressionEvaluator(dump)
225      self.assertAllClose(
226          [[4.0]],
227          ev.evaluate("np.matmul(`a:0:DebugFoo`, `a:0:DebugIdentity`)"))
228
229  def testEvaluateWithMultipleExecIndexes(self):
230    dump = test.mock.MagicMock()
231    def fake_get_tensors(node_name, output_slot, debug_op, device_name=None):
232      del debug_op, device_name  # Unused.
233      if node_name == "a" and output_slot == 0:
234        return [np.array([[-1.0], [1.0]]), np.array([[-2.0], [2.0]])]
235
236    with test.mock.patch.object(
237        dump, "get_tensors", side_effect=fake_get_tensors):
238      ev = evaluator.ExpressionEvaluator(dump)
239      self.assertAllClose(
240          [[4.0]], ev.evaluate("np.matmul(`a:0[1]`.T, `a:0[0]`)"))
241
242  def testEvaluateExpressionWithUnmatchedBacktick(self):
243    dump = test.mock.MagicMock()
244    ev = evaluator.ExpressionEvaluator(dump)
245    with self.assertRaises(SyntaxError):
246      ev.evaluate("np.matmul(`a:0`, `b:0`) + `b:0")
247
248  def testEvaluateExpressionWithInvalidDebugTensorName(self):
249    dump = test.mock.MagicMock()
250    ev = evaluator.ExpressionEvaluator(dump)
251    with self.assertRaisesRegex(ValueError,
252                                r".* tensor name .* expression .* malformed"):
253      ev.evaluate("np.matmul(`a`, `b`)")
254
255    with self.assertRaisesRegex(ValueError,
256                                r".* tensor name .* expression .* malformed"):
257      ev.evaluate("np.matmul(`a:0:DebugIdentity:0`, `b:1:DebugNanCount:2`)")
258
259    with self.assertRaises(ValueError):
260      ev.evaluate("np.matmul(`a:0[]`, `b:0[]`)")
261
262
263if __name__ == "__main__":
264  test.main()
265