Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions connectors/sources/dropbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,8 @@ async def api_call(self, base_url, url_name, data=None, file_type=None, **kwargs
headers = self._get_request_headers(
file_type=file_type, url_name=url_name, kwargs=kwargs
)

self._logger.debug(f"Calling Dropbox Endpoint: {url}")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suggest adding the headers, since the dropbox API is weird and uses headers for their params

async with self._get_session.post(
url=url, headers=headers, data=data
) as response:
Expand Down Expand Up @@ -864,8 +866,18 @@ async def get_content(
Returns:
dictionary: Content document with _id, _timestamp and attachment content
"""
if not doit:
self._logger.debug(
f"Skipping attachment downloading for {attachment['name']}"
)
return

file_size = int(attachment["size"])
if not (doit and file_size > 0):

if file_size <= 0:
self._logger.warning(
f"Skipping file '{attachment['name']}' as file size is {file_size}"
)
return

filename = attachment["name"]
Expand All @@ -880,7 +892,7 @@ async def get_content(
download_func = self.download_func(is_shared, attachment, filename, folder_id)
if not download_func:
self._logger.warning(
f"Skipping the file: {filename} since it is not in the downloadable format."
f"Skipping file '{filename}' since it is not downloadable."
)
return

Expand Down Expand Up @@ -1147,6 +1159,7 @@ async def fetch_file_folders_with_dls(self):
account_id, member_id = await self.get_account_details()
self.dropbox_client.member_id = member_id
async for folder_id in self.get_team_folder_id():
self._logger.info(f"Iterating through folder with id '{folder_id}'")
async for mapped_document in self.add_document_to_list(
func=self._fetch_files_folders,
account_id=account_id,
Expand Down Expand Up @@ -1183,10 +1196,12 @@ async def get_docs(self, filtering=None):
async for document, attachment in self.advanced_sync(rule=rule):
yield self.document_tuple(document=document, attachment=attachment)
else:
self._logger.info("Fetching files and folders")
async for document, attachment in self._fetch_files_folders(
path=self.dropbox_client.path
):
yield self.document_tuple(document=document, attachment=attachment)

self._logger.info("Fetching shared files")
async for document, attachment in self._fetch_shared_files():
yield self.document_tuple(document=document, attachment=attachment)