-
Notifications
You must be signed in to change notification settings - Fork 69
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
325a03e
commit ed7e76e
Showing
9 changed files
with
350 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
from .core.scrapers.redfin import RedfinScraper | ||
from .core.scrapers.types import ListingType, Home | ||
from .core.scrapers import ScraperInput | ||
from .exceptions import InvalidSite, InvalidListingType | ||
|
||
|
||
_scrapers = { | ||
"redfin": RedfinScraper, | ||
} | ||
|
||
|
||
def scrape_property( | ||
location: str, | ||
listing_type: str = "for_sale", #: for_sale, for_rent, sold | ||
site_name: str = "redfin", | ||
) -> list[Home]: #: eventually, return pandas dataframe | ||
if site_name not in _scrapers: | ||
raise InvalidSite(f"Provided site, '{site_name}', does not exist.") | ||
|
||
if listing_type.upper() not in ListingType.__members__: | ||
raise InvalidListingType(f"Provided listing type, '{listing_type}', does not exist.") | ||
|
||
scraper_input = ScraperInput( | ||
location=location, | ||
listing_type=ListingType[listing_type.upper()], | ||
) | ||
|
||
site = _scrapers[site_name](scraper_input) | ||
|
||
return site.search() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
from dataclasses import dataclass | ||
import requests | ||
from .types import Home, ListingType | ||
|
||
|
||
@dataclass | ||
class ScraperInput: | ||
location: str | ||
listing_type: ListingType | ||
proxy_url: str | None = None | ||
|
||
|
||
class Scraper: | ||
def __init__(self, scraper_input: ScraperInput): | ||
self.location = scraper_input.location | ||
self.session = requests.Session() | ||
|
||
if scraper_input.proxy_url: | ||
self.session.proxies = { | ||
"http": scraper_input.proxy_url, | ||
"https": scraper_input.proxy_url, | ||
} | ||
|
||
def search(self) -> list[Home]: | ||
... |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
from .. import Scraper | ||
|
||
|
||
class RedfinScraper(Scraper): | ||
def __init__(self, scraper_input): | ||
super().__init__(scraper_input) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
from dataclasses import dataclass | ||
from enum import Enum | ||
|
||
|
||
class ListingType(Enum): | ||
FOR_SALE = "for_sale" | ||
FOR_RENT = "for_rent" | ||
SOLD = "sold" | ||
|
||
|
||
@dataclass | ||
class Address: | ||
address_one: str | ||
city: str | ||
state: str | ||
zip_code: str | ||
|
||
address_two: str | None = None | ||
|
||
|
||
@dataclass | ||
class Home: | ||
address: Address |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
class InvalidSite(Exception): | ||
"""Raised when a provided site is does not exist.""" | ||
pass | ||
|
||
|
||
class InvalidListingType(Exception): | ||
"""Raised when a provided listing type is does not exist.""" | ||
pass |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
from homeharvest import scrape_property | ||
|
||
|
||
def test_redfin(): | ||
result = scrape_property( | ||
location="85001" | ||
) | ||
|
||
assert result is not None |