Skip to content

Commit 3b1a5f2

Browse files
authored
Merge pull request #57 from Milesjpool/master
Move away from deprecated scrapy log module
2 parents 142ac95 + e20ba31 commit 3b1a5f2

File tree

3 files changed

+7
-22
lines changed

3 files changed

+7
-22
lines changed

sh_scrapy/extension.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import logging
22
from weakref import WeakKeyDictionary
33

4-
from scrapy import signals, log
4+
from scrapy import signals
55
from scrapy.exceptions import ScrapyDeprecationWarning
66
from scrapy.exporters import PythonItemExporter
77
from scrapy.http import Request
@@ -24,6 +24,7 @@ def __init__(self, crawler):
2424
self.hsref = hsref.hsref
2525
self.pipe_writer = pipe_writer
2626
self.crawler = crawler
27+
self.logger = logging.getLogger(__name__)
2728
self._write_item = self.pipe_writer.write_item
2829
# https://github.com/scrapy/scrapy/commit/c76190d491fca9f35b6758bdc06c34d77f5d9be9
2930
exporter_kwargs = {'binary': False} if not IS_PYTHON2 else {}
@@ -39,7 +40,7 @@ def from_crawler(cls, crawler):
3940

4041
def item_scraped(self, item, spider):
4142
if not isinstance(item, (dict, BaseItem)):
42-
log.msg("Wrong item type: %s" % item, level=logging.ERROR)
43+
self.logger.error("Wrong item type: %s" % item)
4344
return
4445
type_ = type(item).__name__
4546
item = self.exporter.export_item(item)

sh_scrapy/log.py

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import warnings
55

66
from twisted.python import log as txlog
7-
from scrapy import log, __version__
7+
from scrapy import __version__
88

99
from sh_scrapy.compat import to_native_str
1010
from sh_scrapy.writer import pipe_writer
@@ -69,17 +69,12 @@ def initialize_logging():
6969

7070
# Scrapy specifics
7171
if 'SCRAPY_JOB' in os.environ:
72-
log.msg("Scrapy %s started" % __version__)
73-
log.start = _dummy # ugly but needed to prevent scrapy re-opening the log
72+
logger = logging.getLogger(__name__)
73+
logger.info("Scrapy %s started" % __version__)
7474

7575
return hdlr
7676

7777

78-
def _dummy(*a, **kw):
79-
"""Scrapy log.start dummy monkeypatch"""
80-
pass
81-
82-
8378
class HubstorageLogHandler(logging.Handler):
8479
"""Python logging handler that writes to HubStorage"""
8580

tests/test_log.py

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@
55
import sys
66
import zlib
77

8-
import scrapy.log
9-
from sh_scrapy.log import _dummy, _stdout, _stderr
8+
from sh_scrapy.log import _stdout, _stderr
109
from sh_scrapy.log import initialize_logging
1110
from sh_scrapy.log import HubstorageLogHandler
1211
from sh_scrapy.log import HubstorageLogObserver
@@ -52,16 +51,6 @@ def test_initialize_logging_dont_fail(observer, txlog_start):
5251
assert loghandler.level == logging.INFO
5352
assert loghandler.formatter._fmt == '[%(name)s] %(message)s'
5453

55-
56-
@mock.patch('sh_scrapy.log.pipe_writer')
57-
def test_initialize_logging_test_scrapy_specific(pipe_writer):
58-
"""Make sure we reset scrapy.log.start"""
59-
loghandler = initialize_logging()
60-
assert scrapy.log.start == _dummy
61-
# test it doesn't fail
62-
scrapy.log.start()
63-
64-
6554
@mock.patch('sh_scrapy.log.pipe_writer')
6655
def test_hs_loghandler_emit_ok(pipe_writer):
6756
hdlr = HubstorageLogHandler()

0 commit comments

Comments
 (0)