44 lines
1 KiB
Python
44 lines
1 KiB
Python
|
from dandi.cli.cmd_ls import ls
|
||
|
import io
|
||
|
import sys
|
||
|
import json
|
||
|
from click.exceptions import ClickException
|
||
|
from scrape_dandi import DANDI_ID, MAX_DANDISET
|
||
|
from tqdm import trange
|
||
|
from multiprocessing import Pool
|
||
|
|
||
|
def dset_ls(dset: int) -> dict:
|
||
|
orig = sys.stdout
|
||
|
capturer = io.StringIO()
|
||
|
metadata = {}
|
||
|
try:
|
||
|
sys.stdout = capturer
|
||
|
ls.main([DANDI_ID.format(dset), '--format', 'json'], standalone_mode=False)
|
||
|
metadata = json.loads(capturer.getvalue())[0]
|
||
|
except ClickException:
|
||
|
pass
|
||
|
finally:
|
||
|
sys.stdout = orig
|
||
|
return metadata
|
||
|
|
||
|
def get_all_meta() -> dict:
|
||
|
meta = {}
|
||
|
|
||
|
try:
|
||
|
for i in trange(MAX_DANDISET):
|
||
|
this_meta = dset_ls(i)
|
||
|
if this_meta:
|
||
|
meta[i] = this_meta
|
||
|
finally:
|
||
|
return meta
|
||
|
|
||
|
def get_and_save_meta(file:str='metadata.json'):
|
||
|
|
||
|
#try:
|
||
|
meta = get_all_meta()
|
||
|
#finally:
|
||
|
with open(file, 'w') as jfile:
|
||
|
json.dump(meta, jfile)
|
||
|
|
||
|
if __name__ == '__main__':
|
||
|
get_and_save_meta()
|