68 lines
1.9 KiB
Python
68 lines
1.9 KiB
Python
from pathlib import Path
|
|
from tqdm import trange
|
|
from datetime import datetime
|
|
import sys
|
|
import click
|
|
from dandi.consts import ZARR_EXTENSIONS, metadata_all_fields
|
|
from dandi.dandiarchive import DandisetURL, _dandi_url_parser, parse_dandi_url
|
|
from dandi.cli.cmd_download import download
|
|
|
|
|
|
OUT_DIR = Path('/mnt/seedbank/p2p/dandi/')
|
|
#OUT_DIR = '.'
|
|
LOG_TXT = 'log.txt'
|
|
SKIP_DANDISETS = [
|
|
108 # humongous 372 human light sheet imaging
|
|
]
|
|
DANDI_ID = 'DANDI:{:06d}'
|
|
MAX_DANDISET = 683
|
|
JOBS = 64
|
|
|
|
|
|
def check_nwb(dandiset:int) -> bool:
|
|
if dandiset == 108:
|
|
return False
|
|
|
|
id = DANDI_ID.format(dandiset)
|
|
try:
|
|
url = parse_dandi_url(id)
|
|
with url.navigate(strict=True) as (c, dandiset,assets):
|
|
is_nwb = any([a.path.endswith('nwb') for a in assets])
|
|
is_not_draft = dandiset.version.identifier != 'draft'
|
|
return is_nwb and is_not_draft
|
|
except:
|
|
return False
|
|
|
|
|
|
def main():
|
|
|
|
for i in trange(MAX_DANDISET):
|
|
if not check_nwb(i) or i in SKIP_DANDISETS:
|
|
with open(LOG_TXT, 'a') as lfile:
|
|
lfile.write(f"{datetime.now().isoformat()} - {i:03d} - SKIP\n")
|
|
continue
|
|
id = DANDI_ID.format(i)
|
|
try:
|
|
download.main([
|
|
|
|
id,
|
|
'-o', str(OUT_DIR),
|
|
'--existing', 'refresh',
|
|
'--jobs', str(JOBS)
|
|
], standalone_mode=False
|
|
)
|
|
print('\nafterdl')
|
|
with open(LOG_TXT, 'a') as lfile:
|
|
lfile.write(f"{datetime.now().isoformat()} - {i:03d} - GET\n")
|
|
except KeyboardInterrupt:
|
|
sys.exit(1)
|
|
except Exception as e:
|
|
print('\nexception')
|
|
with open(LOG_TXT, 'a') as lfile:
|
|
lfile.write(f"{datetime.now().isoformat()} - {i:03d} - ERROR\n")
|
|
lfile.write(str(e))
|
|
# continue
|
|
|
|
|
|
|
|
|