Skip to content

Commit 56f0146

Browse files
committed
Revert "Drafting infer_scraper_function_is_singular"
This reverts commit e795223.
1 parent e795223 commit 56f0146

File tree

3 files changed

+5
-37
lines changed

3 files changed

+5
-37
lines changed

minet/scrape/analysis.py

+2-28
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
#
55
# Functions performing analysis of scraper definitions.
66
#
7-
from typing import Optional, List, Callable, Iterable, Iterator, Generator
8-
from minet.types import Literal, get_type_hints
7+
from typing import Optional, List
8+
from minet.types import Literal
99

1010
import ast
1111
import soupsieve
@@ -113,32 +113,6 @@ def analyse(scraper):
113113
return analysis
114114

115115

116-
NoneType = type(None)
117-
118-
# TODO: what about singular tuples and lists?
119-
PluralTypes = (list, frozenset, set, tuple, Iterator, Iterable, Generator)
120-
121-
122-
def infer_scraper_return_type_is_singular(t) -> bool:
123-
# NOTE: we default to plural
124-
return False
125-
126-
127-
def infer_scraper_function_is_singular(fn: Callable) -> bool:
128-
if not callable(fn):
129-
raise TypeError("expecting a function")
130-
131-
hints = get_type_hints(fn)
132-
return_type = hints.get("return")
133-
134-
if return_type is None:
135-
return False
136-
137-
# TODO: handle unions
138-
139-
return infer_scraper_return_type_is_singular(return_type)
140-
141-
142116
ERRORS_PRIORITY = {
143117
InvalidCSSSelectorError: 0,
144118
ScraperValidationUnknownKeyError: 1,

minet/types.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,15 @@
22
from typing import Union, List, Optional
33
from os import PathLike
44
from io import FileIO
5+
from bs4 import BeautifulSoup
56
from urllib3 import Timeout
67
from ebbe import format_repr
78

89
# NOTE: yes this is repetitive, but mypy only understands this, don't ask questions...
910
if sys.version_info >= (3, 8):
10-
from typing import Literal, get_type_hints, get_origin, get_args
11+
from typing import Literal
1112
else:
12-
from typing_extensions import Literal, get_type_hints, get_origin, get_args
13+
from typing_extensions import Literal
1314

1415
if sys.version_info >= (3, 10):
1516
from typing import TypeGuard, Concatenate, ParamSpec

test/scraper_test.py

-7
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
from minet.scrape import scrape, Scraper
99
from minet.scrape.analysis import (
1010
fieldnames_from_definition,
11-
infer_scraper_function_is_singular,
1211
validate,
1312
analyse,
1413
ScraperAnalysis,
@@ -549,12 +548,6 @@ def test_analysis(self):
549548
plural=True, fieldnames=["url", "title"], output_type="collection"
550549
)
551550

552-
def test_singular_inference(self):
553-
def scraper(row, soup) -> str:
554-
return ""
555-
556-
infer_scraper_function_is_singular(scraper)
557-
558551
def test_absent_tail_call(self):
559552
item = scrape({"sel": "quote", "fields": {"url": "href"}}, BASIC_HTML)
560553

0 commit comments

Comments
 (0)