-
Notifications
You must be signed in to change notification settings - Fork 0
/
Scraper.js
96 lines (81 loc) · 2.24 KB
/
Scraper.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
const cheerio = require("cheerio");
const axios = require("axios");
async function startScraper(url) {
let res = await fetchData(url);
if (!res.data) {
Console.error("Invalid data object");
return null;
}
const html = res.data;
const stockdata = extractStockData(html);
return stockdata;
}
async function fetchData(url) {
console.log(`Crawling: ${url}`);
//make a http call to url
let response = await axios(url).catch((err) => console.log(err));
if (response.status !== 200) {
console.log("Error fetching data");
return;
}
return response;
}
function extractStockData(html) {
const $ = cheerio.load(html);
var StockData = {
companyName: "",
exchange: "",
price: "",
symbol: "",
percentChange: "",
high52: "",
low52: "",
};
if ($(html).find(".quote-company-name ").length < 1) {
return null;
}
$(".quote-company-name ", html).each(function (i, item) {
StockData.companyName = $("H4", item).text().trim();
});
$(".blurb.text-darkgrey", html).each(function (i, item) {
StockData.exchange = $(".text-darkgrey", item).text().trim();
});
$(".price", html).each(function (i, item) {
StockData.price = $("span", item).text().trim();
});
StockData.symbol = $(".labs-symbol", html)
.first()
.contents()
.filter(function () {
return this.type === "text";
})
.text()
.trim();
StockData.high52 = $(".week-high", html)
.first()
.contents()
.filter(function () {
return this.type === "text";
})
.text()
.trim();
StockData.low52 = $(".week-low", html)
.first()
.contents()
.filter(function () {
return this.type === "text";
})
.text()
.trim();
$(".text-red, .text-green, .change-off", html).each(function (i, item) {
if (item.name === "strong") {
var content = item.children[0].data.trim();
var arr = content.split("\n");
var finalResult = arr[1].replace(/\t/g, "").toString();
finalResult = finalResult.substring(1, finalResult.length - 2);
StockData.percentChange = finalResult;
}
});
return StockData;
}
module.exports.startScraper = startScraper;