jonny/scrape-dandi
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

some updates

Browse source
This commit is contained in:
jonny 2023-10-24 00:08:45 -07:00
parent debbadc939
commit 76c8d086f9
1 changed files with 18 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
scrape_dandi/__init__.py
View file

@ -11,11 +11,11 @@ OUT_DIR = Path('/mnt/seedbank/p2p/dandi/')
#OUT_DIR = '.' #OUT_DIR = '.'
LOG_TXT = 'log.txt' LOG_TXT = 'log.txt'
SKIP_DANDISETS = [ SKIP_DANDISETS = [
'000108' # humongous 372 human light sheet imaging 108 # humongous 372 human light sheet imaging
] ]
DANDI_ID = 'DANDI:{:06d}' DANDI_ID = 'DANDI:{:06d}'
MAX_DANDISET = 683 MAX_DANDISET = 683
JOBS = 64
def check_nwb(dandiset:int) -> bool: def check_nwb(dandiset:int) -> bool:
@ -36,20 +36,26 @@ def check_nwb(dandiset:int) -> bool:
def main(): def main():
for i in trange(MAX_DANDISET): for i in trange(MAX_DANDISET):
if not check_nwb(i): if not check_nwb(i) or i in SKIP_DANDISETS:
with open(LOG_TXT, 'a') as lfile: with open(LOG_TXT, 'a') as lfile:
lfile.write(f"{datetime.now().isoformat()} - {i:03d} - SKIP\n") lfile.write(f"{datetime.now().isoformat()} - {i:03d} - SKIP\n")
continue continue
id = DANDI_ID.format(i) id = DANDI_ID.format(i)
download( try:
[ download(
id, [
'-o', str(OUT_DIR), id,
'--existing', 'refresh', '-o', str(OUT_DIR),
'--jobs', '24' '--existing', 'refresh',
] '--jobs', str(JOBS)
) ]
)
except Exception as e:
with open(LOG_TXT, 'a') as lfile:
lfile.write(f"{datetime.now().isoformat()} - {i:03d} - ERROR\n")
lfile.write(str(e))
continue
with open(LOG_TXT, 'a') as lfile: with open(LOG_TXT, 'a') as lfile:
lfile.write(f"{datetime.now().isoformat()} - {i:03d} - GET\n") lfile.write(f"{datetime.now().isoformat()} - {i:03d} - GET\n")