Skip to content

only for testing - fix test failure #1053

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kongtiaowang wants to merge 2 commits into from
Closed

only for testing - fix test failure #1053

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5a8bcac
fix
kongtiaowang Mar 9, 2026
c21484f
fix
kongtiaowang Mar 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 30 additions & 13 deletions scripts/Crawlers/OSFCrawler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,19 +130,36 @@ def _download_files(
else:
filename = file["attributes"]["name"]
url = file["links"]["download"]

# Handle zip files: only register URL, don't download
if filename.endswith(".zip"):
target_path = os.path.join(inner_path, filename)
if self.verbose:
print("Registering zip (no download):", target_path)
annex("addurl", "--fast", url, "--file", target_path)
file_size = file["attributes"]["size"] # in bytes
threshold = 1048576 # 1 MB

if file_size < threshold:
# Original behaviour: download zip with archive=True, others normally
if filename.endswith(".zip"):
d.download_url(
url,
path=os.path.join(inner_path, ""),
archive=True,
)
else:
d.download_url(
url,
path=os.path.join(inner_path, ""),
archive=False,
)
else:
d.download_url(
url,
path=os.path.join(inner_path, ""),
archive=False,
)
# New behaviour: for zip files only register the URL, others download normally
if filename.endswith(".zip"):
target_path = os.path.join(inner_path, filename)
if self.verbose:
print("Registering zip (no download):", target_path)
annex("addurl", "--fast", url, "--file", target_path)
else:
d.download_url(
url,
path=os.path.join(inner_path, ""),
archive=False,
)

# append the size of the downloaded file to the sizes array
file_size = file["attributes"]["size"]
Expand Down Expand Up @@ -598,4 +615,4 @@ def _setup_private_dataset(
return False

def _is_private_dataset(self, files_url) -> bool:
return True if requests.get(files_url).status_code == 401 else False
return True if requests.get(files_url).status_code == 401 else False
Loading