Merge pull request #57 from Milesjpool/master

Move away from deprecated scrapy log module

Author: vshlapakov

sh_scrapy/extension.py

@@ -1,7 +1,7 @@
 import logging
 from weakref import WeakKeyDictionary
 
-from scrapy import signals, log
+from scrapy import signals
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.exporters import PythonItemExporter
 from scrapy.http import Request
@@ -24,6 +24,7 @@ def __init__(self, crawler):
         self.hsref = hsref.hsref
         self.pipe_writer = pipe_writer
         self.crawler = crawler
+        self.logger = logging.getLogger(__name__)
         self._write_item = self.pipe_writer.write_item
         # https://github.com/scrapy/scrapy/commit/c76190d491fca9f35b6758bdc06c34d77f5d9be9
         exporter_kwargs = {'binary': False} if not IS_PYTHON2 else {}
@@ -39,7 +40,7 @@ def from_crawler(cls, crawler):
 
     def item_scraped(self, item, spider):
         if not isinstance(item, (dict, BaseItem)):
-            log.msg("Wrong item type: %s" % item, level=logging.ERROR)
+            self.logger.error("Wrong item type: %s" % item)
             return
         type_ = type(item).__name__
         item = self.exporter.export_item(item)

sh_scrapy/log.py

@@ -4,7 +4,7 @@
 import warnings
 
 from twisted.python import log as txlog
-from scrapy import log, __version__
+from scrapy import __version__
 
 from sh_scrapy.compat import to_native_str
 from sh_scrapy.writer import pipe_writer
@@ -69,17 +69,12 @@ def initialize_logging():
 
     # Scrapy specifics
     if 'SCRAPY_JOB' in os.environ:
-        log.msg("Scrapy %s started" % __version__)
-        log.start = _dummy  # ugly but needed to prevent scrapy re-opening the log
+        logger = logging.getLogger(__name__)
+        logger.info("Scrapy %s started" % __version__)
 
     return hdlr
 
 
-def _dummy(*a, **kw):
-    """Scrapy log.start dummy monkeypatch"""
-    pass
-
-
 class HubstorageLogHandler(logging.Handler):
     """Python logging handler that writes to HubStorage"""
 

tests/test_log.py

@@ -5,8 +5,7 @@
 import sys
 import zlib
 
-import scrapy.log
-from sh_scrapy.log import _dummy, _stdout, _stderr
+from sh_scrapy.log import _stdout, _stderr
 from sh_scrapy.log import initialize_logging
 from sh_scrapy.log import HubstorageLogHandler
 from sh_scrapy.log import HubstorageLogObserver
@@ -52,16 +51,6 @@ def test_initialize_logging_dont_fail(observer, txlog_start):
     assert loghandler.level == logging.INFO
     assert loghandler.formatter._fmt == '[%(name)s] %(message)s'
 
-
-@mock.patch('sh_scrapy.log.pipe_writer')
-def test_initialize_logging_test_scrapy_specific(pipe_writer):
-    """Make sure we reset scrapy.log.start"""
-    loghandler = initialize_logging()
-    assert scrapy.log.start == _dummy
-    # test it doesn't fail
-    scrapy.log.start()
-
-
 @mock.patch('sh_scrapy.log.pipe_writer')
 def test_hs_loghandler_emit_ok(pipe_writer):
     hdlr = HubstorageLogHandler()