File tree Expand file tree Collapse file tree 3 files changed +7
-22
lines changed Expand file tree Collapse file tree 3 files changed +7
-22
lines changed Original file line number Diff line number Diff line change 1
1
import logging
2
2
from weakref import WeakKeyDictionary
3
3
4
- from scrapy import signals , log
4
+ from scrapy import signals
5
5
from scrapy .exceptions import ScrapyDeprecationWarning
6
6
from scrapy .exporters import PythonItemExporter
7
7
from scrapy .http import Request
@@ -24,6 +24,7 @@ def __init__(self, crawler):
24
24
self .hsref = hsref .hsref
25
25
self .pipe_writer = pipe_writer
26
26
self .crawler = crawler
27
+ self .logger = logging .getLogger (__name__ )
27
28
self ._write_item = self .pipe_writer .write_item
28
29
# https://github.com/scrapy/scrapy/commit/c76190d491fca9f35b6758bdc06c34d77f5d9be9
29
30
exporter_kwargs = {'binary' : False } if not IS_PYTHON2 else {}
@@ -39,7 +40,7 @@ def from_crawler(cls, crawler):
39
40
40
41
def item_scraped (self , item , spider ):
41
42
if not isinstance (item , (dict , BaseItem )):
42
- log . msg ("Wrong item type: %s" % item , level = logging . ERROR )
43
+ self . logger . error ("Wrong item type: %s" % item )
43
44
return
44
45
type_ = type (item ).__name__
45
46
item = self .exporter .export_item (item )
Original file line number Diff line number Diff line change 4
4
import warnings
5
5
6
6
from twisted .python import log as txlog
7
- from scrapy import log , __version__
7
+ from scrapy import __version__
8
8
9
9
from sh_scrapy .compat import to_native_str
10
10
from sh_scrapy .writer import pipe_writer
@@ -69,17 +69,12 @@ def initialize_logging():
69
69
70
70
# Scrapy specifics
71
71
if 'SCRAPY_JOB' in os .environ :
72
- log . msg ( "Scrapy %s started" % __version__ )
73
- log . start = _dummy # ugly but needed to prevent scrapy re-opening the log
72
+ logger = logging . getLogger ( __name__ )
73
+ logger . info ( "Scrapy %s started" % __version__ )
74
74
75
75
return hdlr
76
76
77
77
78
- def _dummy (* a , ** kw ):
79
- """Scrapy log.start dummy monkeypatch"""
80
- pass
81
-
82
-
83
78
class HubstorageLogHandler (logging .Handler ):
84
79
"""Python logging handler that writes to HubStorage"""
85
80
Original file line number Diff line number Diff line change 5
5
import sys
6
6
import zlib
7
7
8
- import scrapy .log
9
- from sh_scrapy .log import _dummy , _stdout , _stderr
8
+ from sh_scrapy .log import _stdout , _stderr
10
9
from sh_scrapy .log import initialize_logging
11
10
from sh_scrapy .log import HubstorageLogHandler
12
11
from sh_scrapy .log import HubstorageLogObserver
@@ -52,16 +51,6 @@ def test_initialize_logging_dont_fail(observer, txlog_start):
52
51
assert loghandler .level == logging .INFO
53
52
assert loghandler .formatter ._fmt == '[%(name)s] %(message)s'
54
53
55
-
56
- @mock .patch ('sh_scrapy.log.pipe_writer' )
57
- def test_initialize_logging_test_scrapy_specific (pipe_writer ):
58
- """Make sure we reset scrapy.log.start"""
59
- loghandler = initialize_logging ()
60
- assert scrapy .log .start == _dummy
61
- # test it doesn't fail
62
- scrapy .log .start ()
63
-
64
-
65
54
@mock .patch ('sh_scrapy.log.pipe_writer' )
66
55
def test_hs_loghandler_emit_ok (pipe_writer ):
67
56
hdlr = HubstorageLogHandler ()
You can’t perform that action at this time.
0 commit comments