skip publishers without sources

flairNLP · May 16, 2024 · d8e6c1a · d8e6c1a
1 parent 9c8d656
commit d8e6c1a
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/scripts/publisher_coverage.py b/scripts/publisher_coverage.py
@@ -32,6 +32,11 @@ def main() -> None:
         ):
             publisher_name: str = publisher.name  # type: ignore[attr-defined]
 
+            if not any(publisher.source_mapping.values()):  # type: ignore[attr-defined]
+                # skip publishers providing no sources for forward crawling
+                print(f"⏩  SKIPPED: {publisher_name!r} - No sources defined")
+                continue
+
             crawler: Crawler = Crawler(publisher, delay=0.4)
 
             timed_next = timeout(next, time=20, silent=True)

diff --git a/scripts/utility.py b/scripts/utility.py
@@ -14,16 +14,16 @@ def _interrupt_handler() -> None:
 
 
 @overload
-def timeout(func: Callable[P, T], time: int, silent: Literal[False] = ...) -> Callable[P, T]:
+def timeout(func: Callable[P, T], time: float, silent: Literal[False] = ...) -> Callable[P, T]:
     ...
 
 
 @overload
-def timeout(func: Callable[P, T], time: int, silent: Literal[True]) -> Callable[P, Optional[T]]:
+def timeout(func: Callable[P, T], time: float, silent: Literal[True]) -> Callable[P, Optional[T]]:
     ...
 
 
-def timeout(func: Callable[P, T], time: int, silent: bool = False) -> Callable[P, Optional[T]]:
+def timeout(func: Callable[P, T], time: float, silent: bool = False) -> Callable[P, Optional[T]]:
     @wraps(func)
     def wrapper(*args: P.args, **kwargs: P.kwargs) -> Optional[T]:
         # register interrupt handler