Skip to content

Commit a8d0a73

Browse files
jpmckinneyjpmckinney
committed
feat: Add "Last modified" column to the directory listings of log files and item feeds, closes #509 
1 parent 9226cb6 commit a8d0a73

File tree

2 files changed

+116
-3
lines changed

2 files changed

+116
-3
lines changed

‎docs/news.rst

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ Unreleased
1111
Added
1212
~~~~~
1313

14+
- Add "Last modified" column to the directory listings of log files and item feeds.
1415
- Add environment variables to override common options. See :doc:`config`.
1516
- Add documentation on how to add webservices (endpoints). See :ref:`config-services`.
1617

‎scrapyd/website.py

+115-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import socket
22
from datetime import datetime, timedelta
3-
from urllib.parse import urlparse
3+
from html import escape
4+
from urllib.parse import quote, urlparse
45

56
from scrapy.utils.misc import load_object
67
from twisted.application.service import IServiceCollection
8+
from twisted.python import filepath
79
from twisted.web import resource, static
810

911
from scrapyd.interfaces import IEggStorage, IPoller, ISpiderScheduler
@@ -15,6 +17,116 @@ def get_base_path(self, txrequest):
1517
return txrequest.getHeader(self.prefix_header) or ''
1618

1719

20+
# Use local DirectoryLister class.
21+
class File(static.File):
22+
def directoryListing(self):
23+
path = self.path
24+
names = self.listNames()
25+
return DirectoryLister(
26+
path, names, self.contentTypes, self.contentEncodings, self.defaultType
27+
)
28+
29+
30+
# Add "Last modified" column.
31+
class DirectoryLister(static.DirectoryLister):
32+
template = """<html>
33+
<head>
34+
<title>%(header)s</title>
35+
<style>
36+
.even-dir { background-color: #efe0ef }
37+
.even { background-color: #eee }
38+
.odd-dir {background-color: #f0d0ef }
39+
.odd { background-color: #dedede }
40+
.icon { text-align: center }
41+
.listing {
42+
margin-left: auto;
43+
margin-right: auto;
44+
width: 50%%;
45+
padding: 0.1em;
46+
}
47+
48+
body { border: 0; padding: 0; margin: 0; background-color: #efefef; }
49+
h1 {padding: 0.1em; background-color: #777; color: white; border-bottom: thin white dashed;}
50+
51+
</style>
52+
</head>
53+
54+
<body>
55+
<h1>%(header)s</h1>
56+
57+
<table>
58+
<thead>
59+
<tr>
60+
<th>Filename</th>
61+
<th>Size</th>
62+
<th>Last modified</th>
63+
<th>Content type</th>
64+
<th>Content encoding</th>
65+
</tr>
66+
</thead>
67+
<tbody>
68+
%(tableContent)s
69+
</tbody>
70+
</table>
71+
72+
</body>
73+
</html>
74+
"""
75+
76+
linePattern = """<tr class="%(class)s">
77+
<td><a href="%(href)s">%(text)s</a></td>
78+
<td>%(size)s</td>
79+
<td>%(modified)s</td>
80+
<td>%(type)s</td>
81+
<td>%(encoding)s</td>
82+
</tr>
83+
"""
84+
85+
def _getFilesAndDirectories(self, directory):
86+
files = []
87+
dirs = []
88+
89+
for path in directory:
90+
if isinstance(path, bytes):
91+
path = path.decode("utf8")
92+
93+
url = quote(path, "/")
94+
escapedPath = escape(path)
95+
childPath = filepath.FilePath(self.path).child(path)
96+
modified = datetime.fromtimestamp(childPath.getModificationTime()).strftime("%Y-%m-%d %H:%M") # NEW
97+
98+
if childPath.isdir():
99+
dirs.append(
100+
{
101+
"text": escapedPath + "/",
102+
"href": url + "/",
103+
"size": "",
104+
"type": "[Directory]",
105+
"encoding": "",
106+
"modified": modified, # NEW
107+
}
108+
)
109+
else:
110+
mimetype, encoding = static.getTypeAndEncoding(
111+
path, self.contentTypes, self.contentEncodings, self.defaultType
112+
)
113+
try:
114+
size = childPath.getsize()
115+
except OSError:
116+
continue
117+
files.append(
118+
{
119+
"text": escapedPath,
120+
"href": url,
121+
"type": "[%s]" % mimetype,
122+
"encoding": (encoding and "[%s]" % encoding or ""),
123+
"size": static.formatFileSize(size),
124+
"modified": modified, # NEW
125+
}
126+
)
127+
return dirs, files
128+
129+
18130
class Root(resource.Resource):
19131

20132
def __init__(self, config, app):
@@ -29,9 +141,9 @@ def __init__(self, config, app):
29141
self.nodename = config.get('node_name', socket.gethostname())
30142
self.putChild(b'', Home(self, self.local_items))
31143
if logsdir:
32-
self.putChild(b'logs', static.File(logsdir.encode('ascii', 'ignore'), 'text/plain'))
144+
self.putChild(b'logs', File(logsdir.encode('ascii', 'ignore'), 'text/plain'))
33145
if self.local_items:
34-
self.putChild(b'items', static.File(itemsdir, 'text/plain'))
146+
self.putChild(b'items', File(itemsdir, 'text/plain'))
35147
self.putChild(b'jobs', Jobs(self, self.local_items))
36148
services = config.items('services', ())
37149
for servName, servClsName in services:

0 commit comments

Comments
 (0)