-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathconfig.js.sample
55 lines (45 loc) · 1.78 KB
/
config.js.sample
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
var config = {};
// base directory for saving scrape artifacts
config.baseDir = 'downloads/';
config.log = {
dir: 'logs/', // directory to save file logs; set to null to not save file logs
consoleLevel: 'info', // minimum log level to print to console. debug < info < warn < error
fileLevel: 'info' // minimum log level to save to file. debug < info < warn < error
};
config.retryDelay = 10; // default time to wait between page requests in seconds
config.retryMax = 5; // default number of times to retry getting a page before giving up
// name : common name of the site to scrape; used for save location
// url : URL of the site to scrape
// action: disabled -> do nothing, update -> only save new posts, refresh -> overwrite all
// save : if set, writes output files to the given directory
config.sites = [
// { name: '', url: '', action: ''}
// { name: '', url: '', action: '', save: ''}
// Examples:
// { name: 'imgur Gallery', url: 'http://imgur.com/', action: 'refresh' }
];
// RIPPER-SPECIFIC CONFIGS //
config.rip = {};
config.rip.blogger = {
pageDelay: 10, // time in seconds to wait between page requests for rate limiting
retryDelay: 15, // in the event of an HTTP error, time in seconds before repeating the request
retryMax: 7 // blogger has a thing about trying to time out requests repeatedly
};
config.rip.imgur = {};
config.rip.imgur.album = {
retryDelay: 5,
ordered: true // if true, prepend index of image in album onto filename '1 - image.png'
};
config.rip.imgur.gallery = {
retryDelay: 5,
distance: 5 // number of pages to rip. 1 page = 60 images
};
config.rip.gfycat = {
retryDelay: 5,
formats: {
webm: false, // if true, saves the requested image in <format>
gif: true, // will save once for each set format.
mp4: false
}
};
module.exports = config;