1
1
import json
2
2
import os
3
3
import re
4
+ import time
4
5
from unittest .mock import patch
5
6
6
7
import pytest
@@ -23,48 +24,54 @@ def test_get_settings(environ):
23
24
assert re .search (r"^\S+j1\.log$" , settings ["LOG_FILE" ])
24
25
25
26
if environ .items_dir :
26
- feeds = json .loads (settings [ "FEEDS" ] )
27
+ feeds = json .loads (settings . pop ( "FEEDS" ) )
27
28
path , value = feeds .popitem ()
28
29
30
+ assert list (settings ) == ["LOG_FILE" ]
29
31
assert feeds == {}
30
32
assert re .search (r"^file:///\S+j1\.jl$" , path )
31
33
assert value == {"format" : "jsonlines" }
32
34
33
35
34
36
@pytest .mark .parametrize (
35
- ("items_dir" , "expected " ),
37
+ ("items_dir" , "pattern " ),
36
38
[
37
39
(
38
40
"https://host.example/path?query=value#fragment" ,
39
- "https://host.example/path/p1/s1/j1.jl?query=value#fragment" ,
41
+ r "https://host\ .example/path/p1/s1/j1\ .jl\ ?query=value#fragment" ,
40
42
),
41
43
(
42
44
"https://host.example/path/" ,
43
45
"https://host.example/path/p1/s1/j1.jl" , # no double slashes
44
46
),
45
47
(
46
48
"file:/root.dir/path?ignored#ignored" ,
47
- "file:///root.dir/path/p1/s1/j1.jl" ,
49
+ r "file:///([A-Z]:/)? root\ .dir/path/p1/s1/j1\ .jl" ,
48
50
),
49
51
(
50
52
"file://hostname/root.dir/path?ignored#ignored" ,
51
- "file:///root.dir/path/p1/s1/j1.jl" ,
53
+ r "file:///([A-Z]:/)? root.dir/path/p1/s1/j1.jl" ,
52
54
),
53
55
(
54
56
"file:///root.dir/path?ignored#ignored" ,
55
- "file:///root.dir/path/p1/s1/j1.jl" ,
57
+ r "file:///([A-Z]:/)? root.dir/path/p1/s1/j1.jl" ,
56
58
),
57
59
],
58
60
)
59
61
@patch ("os.listdir" , lambda _ : [])
60
62
@patch ("os.makedirs" , lambda _ : _ )
61
- def test_get_settings_url (items_dir , expected ):
63
+ def test_get_settings_url (items_dir , pattern ):
62
64
config = Config (values = {"logs_dir" : "" , "items_dir" : items_dir })
63
65
environ = Environment (config , initenv = {})
64
66
65
67
settings = environ .get_settings ({"_project" : "p1" , "_spider" : "s1" , "_job" : "j1" })
68
+ feeds = json .loads (settings .pop ("FEEDS" ))
69
+ path , value = feeds .popitem ()
66
70
67
- assert settings == {"FEEDS" : f'{{"{ expected } ": {{"format": "jsonlines"}}}}' }
71
+ assert settings == {}
72
+ assert feeds == {}
73
+ assert re .search (pattern , path )
74
+ assert value == {"format" : "jsonlines" }
68
75
69
76
70
77
@pytest .mark .parametrize ("values" , [{"items_dir" : "../items" }, {"logs_dir" : "../logs" }])
@@ -96,6 +103,7 @@ def test_jobs_to_keep(chdir):
96
103
97
104
(directory / "j1.a" ).touch ()
98
105
(directory / "j2.b" ).touch ()
106
+ time .sleep (0.000001 )
99
107
(directory / "j3.c" ).touch ()
100
108
(directory / "j4.d" ).touch ()
101
109
0 commit comments