diff --git a/tensorflow_datasets/testing/dataset_builder_testing.py b/tensorflow_datasets/testing/dataset_builder_testing.py index ddd8041bf6f..5ca9b3a8cd1 100644 --- a/tensorflow_datasets/testing/dataset_builder_testing.py +++ b/tensorflow_datasets/testing/dataset_builder_testing.py @@ -256,7 +256,7 @@ def _get_dl_download_result(self, url): return utils.map_nested(lambda fname: os.path.join(self.example_dir, fname), self.DL_DOWNLOAD_RESULT) - def _download_checksums(self, url): + def _download_checksums(self, _): self._stop_record_download = True def _make_builder(self, config=None): @@ -267,6 +267,7 @@ def _make_builder(self, config=None): @test_utils.run_in_graph_and_eager_modes() def test_download_and_prepare_as_dataset(self): + """Test download and prepare function on fake testing dataset""" # If configs specified, ensure they are all valid if self.BUILDER_CONFIG_NAMES_TO_TEST: for config in self.BUILDER_CONFIG_NAMES_TO_TEST: # pylint: disable=not-an-iterable @@ -295,7 +296,7 @@ def test_download_and_prepare_as_dataset(self): self._test_checksums() def _test_checksums(self): - # If no call to `dl_manager.download`, then no need to check url presence. + """If no call to `dl_manager.download`, then no need to check url presence""" if not self._download_urls: return @@ -312,13 +313,14 @@ def _test_checksums(self): self.assertEmpty( missing_urls, "Some urls checksums are missing at: {} " - "Did you forgot to record checksums with `--register_checksums` ? " + "Did you forget to record checksums with `--register_checksums` ? " "See instructions at: " "https://www.tensorflow.org/datasets/add_dataset#2_run_download_and_prepare_locally" "\n{}".format(filepath, err_msg) ) def _download_and_prepare_as_dataset(self, builder): + """A utility function""" # Provide the manual dir only if builder has MANUAL_DOWNLOAD_INSTRUCTIONS # set. @@ -367,7 +369,7 @@ def _download_and_prepare_as_dataset(self, builder): with self._subTest("as_dataset"): self._assertAsDataset(builder_reloaded) - def _assertAsDataset(self, builder): + def _assertAsDataset(self, builder): # pylint: disable = invalid-name, missing-function-docstring split_to_checksums = {} # {"split": set(examples_checksums)} for split_name, expected_examples_number in self.SPLITS.items(): ds = builder.as_dataset(split=split_name) @@ -392,7 +394,7 @@ def _assertAsDataset(self, builder): "have the same objects in those splits? If yes, add one one of " "them to OVERLAPPING_SPLITS class attribute.") % (split1, split2)) - def _assertNumSamples(self, builder): + def _assertNumSamples(self, builder): # pylint: disable = invalid-name for split_name, expected_num_examples in self.SPLITS.items(): self.assertEqual( builder.info.splits[split_name].num_examples, diff --git a/tensorflow_datasets/testing/fake_data_generation/cifar.py b/tensorflow_datasets/testing/fake_data_generation/cifar.py index 26a4bc6c3fe..5ad216dcfe5 100644 --- a/tensorflow_datasets/testing/fake_data_generation/cifar.py +++ b/tensorflow_datasets/testing/fake_data_generation/cifar.py @@ -58,8 +58,8 @@ def dump(output_dir, fname, images, labels): path = os.path.join(output_dir, fname) print("Writing %s..." % path) with open(path, "wb") as out_file: - for image, labels in zip(images, labels): - out_file.write(labels.tobytes()) + for image, label in zip(images, labels): + out_file.write(label.tobytes()) out_file.write(image.tobytes()) diff --git a/tensorflow_datasets/testing/fake_data_generation/coil100.py b/tensorflow_datasets/testing/fake_data_generation/coil100.py index 4f1ed9a58ed..46179664501 100644 --- a/tensorflow_datasets/testing/fake_data_generation/coil100.py +++ b/tensorflow_datasets/testing/fake_data_generation/coil100.py @@ -53,7 +53,7 @@ def create_images(label): def main(argv): del argv - label = [x for x in range(0, 25, 5)] + label = list(range(0, 25, 5)) create_images(label) diff --git a/tensorflow_datasets/testing/fake_data_generation/flic.py b/tensorflow_datasets/testing/fake_data_generation/flic.py index 13a5ff5d70b..4a5d6890b76 100644 --- a/tensorflow_datasets/testing/fake_data_generation/flic.py +++ b/tensorflow_datasets/testing/fake_data_generation/flic.py @@ -93,7 +93,7 @@ def _generate_mat(data, train_fname, test_fname): scipy.io.savemat(dirname, data) -def main(unused_argv): +def main(_): _generate_image("small", "images", "example_movie00000001.jpg") _generate_image("small", "images", "example_movie00000002.jpg") _generate_mat("small", "example_movie00000001.jpg", diff --git a/tensorflow_datasets/testing/fake_data_generation/lsun.py b/tensorflow_datasets/testing/fake_data_generation/lsun.py index 631024f3867..2d20f82bc86 100644 --- a/tensorflow_datasets/testing/fake_data_generation/lsun.py +++ b/tensorflow_datasets/testing/fake_data_generation/lsun.py @@ -35,8 +35,8 @@ from absl import app from absl import flags -import lmdb import tensorflow.compat.v2 as tf +import lmdb FLAGS = flags.FLAGS diff --git a/tensorflow_datasets/testing/fake_data_generation/smallnorb_test.py b/tensorflow_datasets/testing/fake_data_generation/smallnorb_test.py index f538cd462a4..70254fe4083 100644 --- a/tensorflow_datasets/testing/fake_data_generation/smallnorb_test.py +++ b/tensorflow_datasets/testing/fake_data_generation/smallnorb_test.py @@ -29,6 +29,7 @@ class SmallnorbTest(parameterized.TestCase): + """A class for testing Smallnorb dataset""" @parameterized.named_parameters( ("uint8", np.array([[1, 2, 3], [1, 2, 3]], dtype=np.dtype("|u1"))), diff --git a/tensorflow_datasets/testing/fake_data_generation/starcraft.py b/tensorflow_datasets/testing/fake_data_generation/starcraft.py index 6818b4b1025..ce5b4bc3294 100644 --- a/tensorflow_datasets/testing/fake_data_generation/starcraft.py +++ b/tensorflow_datasets/testing/fake_data_generation/starcraft.py @@ -30,9 +30,9 @@ from absl import app from absl import flags import numpy as np -import png import six import tensorflow.compat.v2 as tf +import png FLAGS = flags.FLAGS diff --git a/tensorflow_datasets/testing/fake_data_generation/sun397.py b/tensorflow_datasets/testing/fake_data_generation/sun397.py index 64403cb6491..c2994117dfe 100644 --- a/tensorflow_datasets/testing/fake_data_generation/sun397.py +++ b/tensorflow_datasets/testing/fake_data_generation/sun397.py @@ -28,9 +28,9 @@ import tarfile import tempfile +import md5 from absl import app from absl import flags -import md5 import numpy as np from tensorflow_datasets.core.utils import py_utils diff --git a/tensorflow_datasets/testing/mocking.py b/tensorflow_datasets/testing/mocking.py index 35bbd42120e..b28afb90525 100644 --- a/tensorflow_datasets/testing/mocking.py +++ b/tensorflow_datasets/testing/mocking.py @@ -124,8 +124,10 @@ def mock_as_dataset(self, *args, **kwargs): if not data_dir: data_dir = os.path.join(os.path.dirname(__file__), 'metadata') - download_and_prepare_path = 'tensorflow_datasets.core.dataset_builder.DatasetBuilder.download_and_prepare' - as_dataset_path = 'tensorflow_datasets.core.dataset_builder.FileAdapterBuilder._as_dataset' + download_and_prepare_path = \ + 'tensorflow_datasets.core.dataset_builder.DatasetBuilder.download_and_prepare' + as_dataset_path = \ + 'tensorflow_datasets.core.dataset_builder.FileAdapterBuilder._as_dataset' data_dir_path = 'tensorflow_datasets.core.constants.DATA_DIR' with absltest.mock.patch(as_dataset_path, as_dataset_fn), \ @@ -135,7 +137,7 @@ def mock_as_dataset(self, *args, **kwargs): yield -class RandomFakeGenerator(object): +class RandomFakeGenerator(): """Generator of fake examples randomly and deterministically generated.""" def __init__(self, builder, num_examples, seed=0): @@ -163,14 +165,13 @@ def _generate_random_array(self, feature, tensor_info): # Generate some random values, depending on the dtype if tensor_info.dtype.is_integer: return self._rgn.randint(0, max_value, shape) - elif tensor_info.dtype.is_floating: + if tensor_info.dtype.is_floating: return self._rgn.random_sample(shape) - elif tensor_info.dtype == tf.string: + if tensor_info.dtype == tf.string: return ''.join( random.choice(' abcdefghij') for _ in range(random.randint(10, 20))) - else: - raise ValueError('Fake generation not supported for {}'.format( - tensor_info.dtype)) + raise ValueError('Fake generation not supported for {}'.format( + tensor_info.dtype)) def _generate_example(self): """Generate the next example.""" diff --git a/tensorflow_datasets/testing/mocking_test.py b/tensorflow_datasets/testing/mocking_test.py index e030c8923f9..4aa3ac364c7 100644 --- a/tensorflow_datasets/testing/mocking_test.py +++ b/tensorflow_datasets/testing/mocking_test.py @@ -37,7 +37,7 @@ class MockingTest(test_case.TestCase): - + """A class for testing tensorflow_datasets.testing.mocking""" def test_mocking_imagenet(self): with mocking.mock_data(): ds = registered.load('imagenet2012', split='train') @@ -53,7 +53,7 @@ def test_mocking_lm1b(self): self.assertEqual(ex['text'].dtype, tf.int64) ex['text'].shape.assert_is_compatible_with((None,)) - def test_custom_as_dataset(self): + def test_custom_as_dataset(self): # pylint: disable = missing-function-docstring def _as_dataset(self, *args, **kwargs): # pylint: disable=unused-argument return tf.data.Dataset.from_generator( lambda: ({ # pylint: disable=g-long-lambda diff --git a/tensorflow_datasets/testing/test_case.py b/tensorflow_datasets/testing/test_case.py index 33c832d2091..2f9ca40596e 100644 --- a/tensorflow_datasets/testing/test_case.py +++ b/tensorflow_datasets/testing/test_case.py @@ -48,7 +48,7 @@ class TestCase(tf.test.TestCase): """ @classmethod - def setUpClass(cls): + def setUpClass(cls): # pylint: disable = invalid-name super(TestCase, cls).setUpClass() cls.test_data = os.path.join(os.path.dirname(__file__), "test_data") # Test must not communicate with GCS. @@ -65,12 +65,12 @@ def gcs_access(self): gcs_utils.gcs_dataset_info_files = GCS_ACCESS_FNS["dummy_info"] gcs_utils.is_dataset_on_gcs = GCS_ACCESS_FNS["dummy_datasets"] - def setUp(self): + def setUp(self): # pylint: disable = invalid-name super(TestCase, self).setUp() # get_temp_dir is actually the same for all tests, so create a temp sub-dir. self.tmp_dir = tempfile.mkdtemp(dir=tf.compat.v1.test.get_temp_dir()) - def assertRaisesWithPredicateMatch(self, err_type, predicate): + def assertRaisesWithPredicateMatch(self, err_type, predicate): # pylint: disable = invalid-name if isinstance(predicate, six.string_types): predicate_fct = lambda err: predicate in str(err) else: @@ -79,7 +79,8 @@ def assertRaisesWithPredicateMatch(self, err_type, predicate): err_type, predicate_fct) @contextlib.contextmanager - def assertLogs(self, text, level="info"): + def assertLogs(self, text, level="info"): # pylint: disable = invalid-name + """Log text messages with their level information""" with absltest.mock.patch.object(logging, level) as mock_log: yield concat_logs = "" diff --git a/tensorflow_datasets/testing/test_utils.py b/tensorflow_datasets/testing/test_utils.py index 29a1f99f371..daacef15ef7 100644 --- a/tensorflow_datasets/testing/test_utils.py +++ b/tensorflow_datasets/testing/test_utils.py @@ -73,7 +73,7 @@ def fake_examples_dir(): return os.path.join(os.path.dirname(__file__), "test_data", "fake_examples") -class FeatureExpectationItem(object): +class FeatureExpectationItem(): """Test item of a FeatureExpectation.""" def __init__( @@ -111,7 +111,7 @@ def setUpClass(cls): cls._sub_test_stack = [] @contextlib.contextmanager - def _subTest(self, test_str): + def _subTest(self, test_str): # pylint: disable = invalid-name, missing-function-docstring sub_test_not_implemented = True if sub_test_not_implemented: yield @@ -122,7 +122,7 @@ def _subTest(self, test_str): yield self._sub_test_stack.pop() - def assertAllEqualNested(self, d1, d2): + def assertAllEqualNested(self, d1, d2): # pylint: disable = invalid-name """Same as assertAllEqual but compatible with nested dict.""" if isinstance(d1, dict): # assertAllEqual do not works well with dictionaries so assert @@ -213,7 +213,7 @@ def decorated(self, *args, **kwargs): return decorator -class RaggedConstant(object): +class RaggedConstant(): """Container of tf.ragged.constant values. This simple wrapper forward the arguments to delay the RaggedTensor @@ -233,7 +233,7 @@ class FeatureExpectationsTestCase(SubTestCase): """Tests FeatureExpectations with full encode-decode.""" @run_in_graph_and_eager_modes() - def assertFeature(self, feature, shape, dtype, tests, serialized_info=None): + def assertFeature(self, feature, shape, dtype, tests, serialized_info=None): # pylint: disable = invalid-name """Test the given feature against the predicates.""" # Check the shape/dtype @@ -264,7 +264,7 @@ def assertFeature(self, feature, shape, dtype, tests, serialized_info=None): dtype=dtype, ) - def assertFeatureTest(self, fdict, test, feature, shape, dtype): + def assertFeatureTest(self, fdict, test, feature, shape, dtype): # pylint: disable = invalid-name """Test that encode=>decoding of a value works correctly.""" # test feature.encode_example can be pickled and unpickled for beam. dill.loads(dill.dumps(feature.encode_example)) @@ -385,7 +385,7 @@ def _split_generators(self, dl_manager): gen_kwargs={"range_": range(20, 30)}), ] - def _generate_examples(self, range_): + def _generate_examples(self, range_): # pylint: disable = arguments-differ for i in range_: yield i, {"x": i} @@ -415,7 +415,7 @@ def _split_generators(self, dl_manager): gen_kwargs=dict()), ] - def _generate_examples(self): + def _generate_examples(self): # pylint: disable = arguments-differ for i in range(20): yield i, { "image": np.ones((28, 28, 1), dtype=np.uint8), @@ -479,9 +479,8 @@ def make_mock_check_output(filenames, err_msg): def check_output(command_args): if command_args[2] == "files": return files_call(command_args) - else: - assert command_args[2] == "download" - return dl_call(command_args) + assert command_args[2] == "download" + return dl_call(command_args) return check_output @@ -490,7 +489,7 @@ def check_output(command_args): yield -class DummySerializer(object): +class DummySerializer(): """To mock example_serializer.ExampleSerializer.""" def __init__(self, specs): @@ -500,7 +499,7 @@ def serialize_example(self, example): return bytes(example) -class DummyParser(object): +class DummyParser(): """To mock example_parser.ExampleParser.""" def __init__(self, specs): @@ -508,4 +507,3 @@ def __init__(self, specs): def parse_example(self, ex): return ex - diff --git a/tensorflow_datasets/testing/test_utils_test.py b/tensorflow_datasets/testing/test_utils_test.py index 88e94882dec..7f45b1fe435 100644 --- a/tensorflow_datasets/testing/test_utils_test.py +++ b/tensorflow_datasets/testing/test_utils_test.py @@ -14,6 +14,7 @@ # limitations under the License. # Lint as: python3 +# pylint: disable = missing-class-docstring, missing-function-docstring """Tests for tensorflow_datasets.core.test_utils.""" from __future__ import absolute_import @@ -57,14 +58,14 @@ def test_run_in_graph_and_eager_modes_setup_in_same_mode(self): class ExampleTest(test_case.TestCase): - def runTest(self): + def runTest(self): # pylint: disable = invalid-name pass - def setUp(self): + def setUp(self): # pylint: disable = invalid-name modes.append("setup_" + mode_name()) @test_utils.run_in_graph_and_eager_modes - def testBody(self): + def testBody(self): # pylint: disable = invalid-name modes.append("run_" + mode_name()) e = ExampleTest()