jonny/scrape-dandi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix click probs

Browse source
This commit is contained in:
jonny 2023-10-26 20:11:01 -07:00
parent 76c8d086f9
commit 05240668a8
1 changed files with 12 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
scrape_dandi/__init__.py
View file

@ -1,7 +1,8 @@
from pathlib import Path
from tqdm import trange
from datetime import datetime
import sys
import click
from dandi.consts import ZARR_EXTENSIONS, metadata_all_fields
from dandi.dandiarchive import DandisetURL, _dandi_url_parser, parse_dandi_url
from dandi.cli.cmd_download import download
@ -36,28 +37,31 @@ def check_nwb(dandiset:int) -> bool:
def main():
for i in trange(MAX_DANDISET):
if not check_nwb(i) or i in SKIP_DANDISETS:
with open(LOG_TXT, 'a') as lfile:
lfile.write(f"{datetime.now().isoformat()} - {i:03d} - SKIP\n")
continue
id = DANDI_ID.format(i)
try:
download(
[
download.main([
id,
'-o', str(OUT_DIR),
'--existing', 'refresh',
'--jobs', str(JOBS)
]
], standalone_mode=False
)
print('\nafterdl')
with open(LOG_TXT, 'a') as lfile:
lfile.write(f"{datetime.now().isoformat()} - {i:03d} - GET\n")
except KeyboardInterrupt:
sys.exit(1)
except Exception as e:
print('\nexception')
with open(LOG_TXT, 'a') as lfile:
lfile.write(f"{datetime.now().isoformat()} - {i:03d} - ERROR\n")
lfile.write(str(e))
continue
with open(LOG_TXT, 'a') as lfile:
lfile.write(f"{datetime.now().isoformat()} - {i:03d} - GET\n")
# continue