Fix tfds testing pylint

Author: vijayphoenix

tensorflow_datasets/testing/dataset_builder_testing.py

@@ -256,7 +256,7 @@ def _get_dl_download_result(self, url):
     return utils.map_nested(lambda fname: os.path.join(self.example_dir, fname),
                             self.DL_DOWNLOAD_RESULT)
 
-  def _download_checksums(self, url):
+  def _download_checksums(self, _):
     self._stop_record_download = True
 
   def _make_builder(self, config=None):
@@ -267,6 +267,7 @@ def _make_builder(self, config=None):
 
   @test_utils.run_in_graph_and_eager_modes()
   def test_download_and_prepare_as_dataset(self):
+    """Test download and prepare function on fake testing dataset"""
     # If configs specified, ensure they are all valid
     if self.BUILDER_CONFIG_NAMES_TO_TEST:
       for config in self.BUILDER_CONFIG_NAMES_TO_TEST:  # pylint: disable=not-an-iterable
@@ -295,7 +296,7 @@ def test_download_and_prepare_as_dataset(self):
         self._test_checksums()
 
   def _test_checksums(self):
-    # If no call to `dl_manager.download`, then no need to check url presence.
+    """If no call to `dl_manager.download`, then no need to check url presence"""
     if not self._download_urls:
       return
 
@@ -312,13 +313,14 @@ def _test_checksums(self):
     self.assertEmpty(
         missing_urls,
         "Some urls checksums are missing at: {} "
-        "Did you forgot to record checksums with `--register_checksums` ? "
+        "Did you forget to record checksums with `--register_checksums` ? "
         "See instructions at: "
         "https://www.tensorflow.org/datasets/add_dataset#2_run_download_and_prepare_locally"
         "\n{}".format(filepath, err_msg)
     )
 
   def _download_and_prepare_as_dataset(self, builder):
+    """A utility function"""
     # Provide the manual dir only if builder has MANUAL_DOWNLOAD_INSTRUCTIONS
     # set.
 
@@ -367,7 +369,7 @@ def _download_and_prepare_as_dataset(self, builder):
       with self._subTest("as_dataset"):
         self._assertAsDataset(builder_reloaded)
 
-  def _assertAsDataset(self, builder):
+  def _assertAsDataset(self, builder): # pylint: disable = invalid-name, missing-function-docstring
     split_to_checksums = {}  # {"split": set(examples_checksums)}
     for split_name, expected_examples_number in self.SPLITS.items():
       ds = builder.as_dataset(split=split_name)
@@ -392,7 +394,7 @@ def _assertAsDataset(self, builder):
            "have the same objects in those splits? If yes, add one one of "
            "them to OVERLAPPING_SPLITS class attribute.") % (split1, split2))
 
-  def _assertNumSamples(self, builder):
+  def _assertNumSamples(self, builder): # pylint: disable = invalid-name
     for split_name, expected_num_examples in self.SPLITS.items():
       self.assertEqual(
           builder.info.splits[split_name].num_examples,

tensorflow_datasets/testing/fake_data_generation/cifar.py

@@ -58,8 +58,8 @@ def dump(output_dir, fname, images, labels):
   path = os.path.join(output_dir, fname)
   print("Writing %s..." % path)
   with open(path, "wb") as out_file:
-    for image, labels in zip(images, labels):
-      out_file.write(labels.tobytes())
+    for image, label in zip(images, labels):
+      out_file.write(label.tobytes())
       out_file.write(image.tobytes())
 
 

tensorflow_datasets/testing/fake_data_generation/coil100.py

@@ -53,7 +53,7 @@ def create_images(label):
 
 def main(argv):
   del argv
-  label = [x for x in range(0, 25, 5)]
+  label = list(range(0, 25, 5))
   create_images(label)
 
 

tensorflow_datasets/testing/fake_data_generation/flic.py

@@ -93,7 +93,7 @@ def _generate_mat(data, train_fname, test_fname):
   scipy.io.savemat(dirname, data)
 
 
-def main(unused_argv):
+def main(_):
   _generate_image("small", "images", "example_movie00000001.jpg")
   _generate_image("small", "images", "example_movie00000002.jpg")
   _generate_mat("small", "example_movie00000001.jpg",

tensorflow_datasets/testing/fake_data_generation/lsun.py

@@ -35,8 +35,8 @@
 
 from absl import app
 from absl import flags
-import lmdb
 import tensorflow.compat.v2 as tf
+import lmdb
 
 FLAGS = flags.FLAGS
 

tensorflow_datasets/testing/fake_data_generation/smallnorb_test.py

@@ -29,6 +29,7 @@
 
 
 class SmallnorbTest(parameterized.TestCase):
+  """A class for testing Smallnorb dataset"""
 
   @parameterized.named_parameters(
       ("uint8", np.array([[1, 2, 3], [1, 2, 3]], dtype=np.dtype("|u1"))),

tensorflow_datasets/testing/fake_data_generation/starcraft.py

@@ -30,9 +30,9 @@
 from absl import app
 from absl import flags
 import numpy as np
-import png
 import six
 import tensorflow.compat.v2 as tf
+import png
 
 FLAGS = flags.FLAGS
 

tensorflow_datasets/testing/fake_data_generation/sun397.py

@@ -28,9 +28,9 @@
 import tarfile
 import tempfile
 
+import md5
 from absl import app
 from absl import flags
-import md5
 
 import numpy as np
 from tensorflow_datasets.core.utils import py_utils

tensorflow_datasets/testing/mocking.py

@@ -124,8 +124,10 @@ def mock_as_dataset(self, *args, **kwargs):
   if not data_dir:
     data_dir = os.path.join(os.path.dirname(__file__), 'metadata')
 
-  download_and_prepare_path = 'tensorflow_datasets.core.dataset_builder.DatasetBuilder.download_and_prepare'
-  as_dataset_path = 'tensorflow_datasets.core.dataset_builder.FileAdapterBuilder._as_dataset'
+  download_and_prepare_path = \
+  'tensorflow_datasets.core.dataset_builder.DatasetBuilder.download_and_prepare'
+  as_dataset_path = \
+    'tensorflow_datasets.core.dataset_builder.FileAdapterBuilder._as_dataset'
   data_dir_path = 'tensorflow_datasets.core.constants.DATA_DIR'
 
   with absltest.mock.patch(as_dataset_path, as_dataset_fn), \
@@ -135,7 +137,7 @@ def mock_as_dataset(self, *args, **kwargs):
     yield
 
 
-class RandomFakeGenerator(object):
+class RandomFakeGenerator():
   """Generator of fake examples randomly and deterministically generated."""
 
   def __init__(self, builder, num_examples, seed=0):
@@ -163,14 +165,13 @@ def _generate_random_array(self, feature, tensor_info):
     # Generate some random values, depending on the dtype
     if tensor_info.dtype.is_integer:
       return self._rgn.randint(0, max_value, shape)
-    elif tensor_info.dtype.is_floating:
+    if tensor_info.dtype.is_floating:
       return self._rgn.random_sample(shape)
-    elif tensor_info.dtype == tf.string:
+    if tensor_info.dtype == tf.string:
       return ''.join(
           random.choice(' abcdefghij') for _ in range(random.randint(10, 20)))
-    else:
-      raise ValueError('Fake generation not supported for {}'.format(
-          tensor_info.dtype))
+    raise ValueError('Fake generation not supported for {}'.format(
+        tensor_info.dtype))
 
   def _generate_example(self):
     """Generate the next example."""

tensorflow_datasets/testing/mocking_test.py

@@ -37,7 +37,7 @@
 
 
 class MockingTest(test_case.TestCase):
-
+  """A class for testing tensorflow_datasets.testing.mocking"""
   def test_mocking_imagenet(self):
     with mocking.mock_data():
       ds = registered.load('imagenet2012', split='train')
@@ -53,7 +53,7 @@ def test_mocking_lm1b(self):
         self.assertEqual(ex['text'].dtype, tf.int64)
         ex['text'].shape.assert_is_compatible_with((None,))
 
-  def test_custom_as_dataset(self):
+  def test_custom_as_dataset(self):  # pylint: disable = missing-function-docstring
     def _as_dataset(self, *args, **kwargs):  # pylint: disable=unused-argument
       return tf.data.Dataset.from_generator(
           lambda: ({  # pylint: disable=g-long-lambda

tensorflow_datasets/testing/test_case.py

@@ -48,7 +48,7 @@ class TestCase(tf.test.TestCase):
   """
 
   @classmethod
-  def setUpClass(cls):
+  def setUpClass(cls): # pylint: disable = invalid-name
     super(TestCase, cls).setUpClass()
     cls.test_data = os.path.join(os.path.dirname(__file__), "test_data")
     # Test must not communicate with GCS.
@@ -65,12 +65,12 @@ def gcs_access(self):
     gcs_utils.gcs_dataset_info_files = GCS_ACCESS_FNS["dummy_info"]
     gcs_utils.is_dataset_on_gcs = GCS_ACCESS_FNS["dummy_datasets"]
 
-  def setUp(self):
+  def setUp(self): # pylint: disable = invalid-name
     super(TestCase, self).setUp()
     # get_temp_dir is actually the same for all tests, so create a temp sub-dir.
     self.tmp_dir = tempfile.mkdtemp(dir=tf.compat.v1.test.get_temp_dir())
 
-  def assertRaisesWithPredicateMatch(self, err_type, predicate):
+  def assertRaisesWithPredicateMatch(self, err_type, predicate): # pylint: disable = invalid-name
     if isinstance(predicate, six.string_types):
       predicate_fct = lambda err: predicate in str(err)
     else:
@@ -79,7 +79,8 @@ def assertRaisesWithPredicateMatch(self, err_type, predicate):
         err_type, predicate_fct)
 
   @contextlib.contextmanager
-  def assertLogs(self, text, level="info"):
+  def assertLogs(self, text, level="info"): # pylint: disable = invalid-name
+    """Log text messages with their level information"""
     with absltest.mock.patch.object(logging, level) as mock_log:
       yield
       concat_logs = ""

tensorflow_datasets/testing/test_utils.py

@@ -73,7 +73,7 @@ def fake_examples_dir():
   return os.path.join(os.path.dirname(__file__), "test_data", "fake_examples")
 
 
-class FeatureExpectationItem(object):
+class FeatureExpectationItem():
   """Test item of a FeatureExpectation."""
 
   def __init__(
@@ -111,7 +111,7 @@ def setUpClass(cls):
     cls._sub_test_stack = []
 
   @contextlib.contextmanager
-  def _subTest(self, test_str):
+  def _subTest(self, test_str): # pylint: disable = invalid-name, missing-function-docstring
     sub_test_not_implemented = True
     if sub_test_not_implemented:
       yield
@@ -122,7 +122,7 @@ def _subTest(self, test_str):
         yield
       self._sub_test_stack.pop()
 
-  def assertAllEqualNested(self, d1, d2):
+  def assertAllEqualNested(self, d1, d2): # pylint: disable = invalid-name
     """Same as assertAllEqual but compatible with nested dict."""
     if isinstance(d1, dict):
       # assertAllEqual do not works well with dictionaries so assert
@@ -213,7 +213,7 @@ def decorated(self, *args, **kwargs):
   return decorator
 
 
-class RaggedConstant(object):
+class RaggedConstant():
   """Container of tf.ragged.constant values.
 
   This simple wrapper forward the arguments to delay the RaggedTensor
@@ -233,7 +233,7 @@ class FeatureExpectationsTestCase(SubTestCase):
   """Tests FeatureExpectations with full encode-decode."""
 
   @run_in_graph_and_eager_modes()
-  def assertFeature(self, feature, shape, dtype, tests, serialized_info=None):
+  def assertFeature(self, feature, shape, dtype, tests, serialized_info=None): # pylint: disable = invalid-name
     """Test the given feature against the predicates."""
 
     # Check the shape/dtype
@@ -264,7 +264,7 @@ def assertFeature(self, feature, shape, dtype, tests, serialized_info=None):
             dtype=dtype,
         )
 
-  def assertFeatureTest(self, fdict, test, feature, shape, dtype):
+  def assertFeatureTest(self, fdict, test, feature, shape, dtype): # pylint: disable = invalid-name
     """Test that encode=>decoding of a value works correctly."""
     # test feature.encode_example can be pickled and unpickled for beam.
     dill.loads(dill.dumps(feature.encode_example))
@@ -385,7 +385,7 @@ def _split_generators(self, dl_manager):
             gen_kwargs={"range_": range(20, 30)}),
     ]
 
-  def _generate_examples(self, range_):
+  def _generate_examples(self, range_): # pylint: disable = arguments-differ
     for i in range_:
       yield i, {"x": i}
 
@@ -415,7 +415,7 @@ def _split_generators(self, dl_manager):
             gen_kwargs=dict()),
     ]
 
-  def _generate_examples(self):
+  def _generate_examples(self): # pylint: disable = arguments-differ
     for i in range(20):
       yield i, {
           "image": np.ones((28, 28, 1), dtype=np.uint8),
@@ -479,9 +479,8 @@ def make_mock_check_output(filenames, err_msg):
     def check_output(command_args):
       if command_args[2] == "files":
         return files_call(command_args)
-      else:
-        assert command_args[2] == "download"
-        return dl_call(command_args)
+      assert command_args[2] == "download"
+      return dl_call(command_args)
 
     return check_output
 
@@ -490,7 +489,7 @@ def check_output(command_args):
     yield
 
 
-class DummySerializer(object):
+class DummySerializer():
   """To mock example_serializer.ExampleSerializer."""
 
   def __init__(self, specs):
@@ -500,12 +499,11 @@ def serialize_example(self, example):
     return bytes(example)
 
 
-class DummyParser(object):
+class DummyParser():
   """To mock example_parser.ExampleParser."""
 
   def __init__(self, specs):
     del specs
 
   def parse_example(self, ex):
     return ex
-

tensorflow_datasets/testing/test_utils_test.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 # Lint as: python3
+# pylint: disable = missing-class-docstring, missing-function-docstring
 """Tests for tensorflow_datasets.core.test_utils."""
 
 from __future__ import absolute_import
@@ -57,14 +58,14 @@ def test_run_in_graph_and_eager_modes_setup_in_same_mode(self):
 
     class ExampleTest(test_case.TestCase):
 
-      def runTest(self):
+      def runTest(self): # pylint: disable = invalid-name
         pass
 
-      def setUp(self):
+      def setUp(self): # pylint: disable = invalid-name
         modes.append("setup_" + mode_name())
 
       @test_utils.run_in_graph_and_eager_modes
-      def testBody(self):
+      def testBody(self): # pylint: disable = invalid-name
         modes.append("run_" + mode_name())
 
     e = ExampleTest()