Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Dropping support for Python2 from fetchcode. #88

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Dropping support for Python2 from fetchcode. #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9c8d9ee
Removed all __future__ imports.
35C4n0r Jan 3, 2023
ca27443
Reformatted files in src using black.
35C4n0r Jan 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
18 changes: 9 additions & 9 deletions src/fetchcode/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,14 @@ def __init__(self, location, content_type, size, url):
def fetch_http(url, location):
"""
Return a `Response` object built from fetching the content at a HTTP/HTTPS based `url` URL string
saving the content in a file at `location`
saving the content in a file at `location`
"""
r = requests.get(url)
with open(location, 'wb') as f:
with open(location, "wb") as f:
f.write(r.content)

content_type = r.headers.get('content-type')
size = r.headers.get('content-length')
content_type = r.headers.get("content-type")
size = r.headers.get("content-length")
size = int(size) if size else None

resp = Response(location=location, content_type=content_type, size=size, url=url)
Expand All @@ -59,7 +59,7 @@ def fetch_http(url, location):
def fetch_ftp(url, location):
"""
Return a `Response` object built from fetching the content at a FTP based `url` URL string
saving the content in a file at `location`
saving the content in a file at `location`
"""
url_parts = urlparse(url)

Expand All @@ -79,8 +79,8 @@ def fetch_ftp(url, location):
content_type = None

ftp.cwd(dir)
file = 'RETR {}'.format(file)
with open(location, 'wb') as f:
file = "RETR {}".format(file)
with open(location, "wb") as f:
ftp.retrbinary(file, f.write)
ftp.close()

Expand All @@ -99,9 +99,9 @@ def fetch(url):
url_parts = urlparse(url)
scheme = url_parts.scheme

fetchers = {'ftp': fetch_ftp, 'http': fetch_http, 'https': fetch_http}
fetchers = {"ftp": fetch_ftp, "http": fetch_http, "https": fetch_http}

if scheme in fetchers:
return fetchers.get(scheme)(url, location)

raise Exception('Not a supported/known scheme.')
raise Exception("Not a supported/known scheme.")
129 changes: 100 additions & 29 deletions src/fetchcode/commoncode_datautils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,6 @@
# ScanCode is a free software code scanning tool from nexB Inc. and others.
# Visit https://github.com/nexB/scancode-toolkit/ for support and download.

from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

from collections import OrderedDict

Expand All @@ -39,14 +36,23 @@
"""


HELP_METADATA = '__field_help'
LABEL_METADATA = '__field_label'
HELP_METADATA = "__field_help"
LABEL_METADATA = "__field_label"


def attribute(default=attr.NOTHING, validator=None,
repr=False, eq=True, order=True, # NOQA
init=True, type=None, converter=None, # NOQA
help=None, label=None, metadata=None,): # NOQA
def attribute(
default=attr.NOTHING,
validator=None,
repr=False,
eq=True,
order=True, # NOQA
init=True,
type=None,
converter=None, # NOQA
help=None,
label=None,
metadata=None,
): # NOQA
"""
A generic attribute with help metadata and that is not included in the
representation by default.
Expand All @@ -67,12 +73,20 @@ def attribute(default=attr.NOTHING, validator=None,
init=init,
metadata=metadata,
type=type,
converter=converter
converter=converter,
)


def Boolean(default=False, validator=None, repr=False, eq=True, order=True, # NOQA
converter=None, label=None, help=None,): # NOQA
def Boolean(
default=False,
validator=None,
repr=False,
eq=True,
order=True, # NOQA
converter=None,
label=None,
help=None,
): # NOQA
"""
A boolean attribute.
"""
Expand All @@ -90,8 +104,16 @@ def Boolean(default=False, validator=None, repr=False, eq=True, order=True, # N
)


def TriBoolean(default=None, validator=None, repr=False, eq=True, order=True, # NOQA
converter=None, label=None, help=None,): # NOQA
def TriBoolean(
default=None,
validator=None,
repr=False,
eq=True,
order=True, # NOQA
converter=None,
label=None,
help=None,
): # NOQA
"""
A tri-boolean attribute with possible values of None, True and False.
"""
Expand All @@ -109,8 +131,16 @@ def TriBoolean(default=None, validator=None, repr=False, eq=True, order=True, #
)


def String(default=None, validator=None, repr=False, eq=True, order=True, # NOQA
converter=None, label=None, help=None,): # NOQA
def String(
default=None,
validator=None,
repr=False,
eq=True,
order=True, # NOQA
converter=None,
label=None,
help=None,
): # NOQA
"""
A string attribute.
"""
Expand All @@ -128,8 +158,16 @@ def String(default=None, validator=None, repr=False, eq=True, order=True, # NOQ
)


def Integer(default=0, validator=None, repr=False, eq=True, order=True, # NOQA
converter=None, label=None, help=None,): # NOQA
def Integer(
default=0,
validator=None,
repr=False,
eq=True,
order=True, # NOQA
converter=None,
label=None,
help=None,
): # NOQA
"""
An integer attribute.
"""
Expand All @@ -148,8 +186,16 @@ def Integer(default=0, validator=None, repr=False, eq=True, order=True, # NOQA
)


def Float(default=0.0, validator=None, repr=False, eq=True, order=True, # NOQA
converter=None, label=None, help=None,): # NOQA
def Float(
default=0.0,
validator=None,
repr=False,
eq=True,
order=True, # NOQA
converter=None,
label=None,
help=None,
): # NOQA
"""
A float attribute.
"""
Expand All @@ -167,9 +213,17 @@ def Float(default=0.0, validator=None, repr=False, eq=True, order=True, # NOQA
)


def List(item_type=typing.Any, default=attr.NOTHING, validator=None,
repr=False, eq=True, order=True, # NOQA
converter=None, label=None, help=None,): # NOQA
def List(
item_type=typing.Any,
default=attr.NOTHING,
validator=None,
repr=False,
eq=True,
order=True, # NOQA
converter=None,
label=None,
help=None,
): # NOQA
"""
A list attribute: the optional item_type defines the type of items it stores.
"""
Expand All @@ -190,9 +244,17 @@ def List(item_type=typing.Any, default=attr.NOTHING, validator=None,
)


def Mapping(value_type=typing.Any, default=attr.NOTHING, validator=None,
repr=False, eq=True, order=True, # NOQA
converter=None, help=None, label=None): # NOQA
def Mapping(
value_type=typing.Any,
default=attr.NOTHING,
validator=None,
repr=False,
eq=True,
order=True, # NOQA
converter=None,
help=None,
label=None,
): # NOQA
"""
A mapping attribute: the optional value_type defines the type of values it
stores. The key is always a string.
Expand Down Expand Up @@ -221,8 +283,17 @@ def Mapping(value_type=typing.Any, default=attr.NOTHING, validator=None,
# FIXME: add proper support for dates!!!
##################################################

def Date(default=None, validator=None, repr=False, eq=True, order=True, # NOQA
converter=None, label=None, help=None,): # NOQA

def Date(
default=None,
validator=None,
repr=False,
eq=True,
order=True, # NOQA
converter=None,
label=None,
help=None,
): # NOQA
"""
A date attribute. It always serializes to an ISO date string.
Behavior is TBD and for now this is exactly a string.
Expand All @@ -236,4 +307,4 @@ def Date(default=None, validator=None, repr=False, eq=True, order=True, # NOQA
converter=converter,
help=help,
label=label,
)
)
Loading