Skip to content

Latest commit

 

History

History
90 lines (65 loc) · 2.77 KB

datasets.md

File metadata and controls

90 lines (65 loc) · 2.77 KB

Instructions for downloading datasets

Conceptual Captions

First, download Validation_GCC-1.1.0-Validation.tsv from the Conceptual Captions dataset here.
Then, use gather_cc.py to download the images. After running the script, you should get both a folder of images and a Validation_GCC-1.1.0-Validation_output.csv file.
The process of Conceptual Captions is the same with this repo.

python gather_cc.py Validation_GCC-1.1.0-Validation.tsv

ImageNet distribution shifts

export DATA_LOCATION=~/data
cd $DATA_LOCATION

ImageNet-A

wget https://people.eecs.berkeley.edu/~hendrycks/imagenet-a.tar
tar -xvzf imagenet-a.tar
rm imagenet-a.tar

ImageNet-R

wget https://people.eecs.berkeley.edu/~hendrycks/imagenet-r.tar
tar -xvzf imagenet-r.tar
rm imagenet-r.tar

ImageNet Sketch

Download links:

ImageNet V2

wget https://s3-us-west-2.amazonaws.com/imagenetv2public/imagenetv2-matched-frequency.tar.gz
tar -xvf imagenetv2-matched-frequency.tar.gz
rm imagenetv2-matched-frequency.tar.gz

ObjectNet

wget https://objectnet.dev/downloads/objectnet-1.0.zip
unzip objectnet-1.0.zip
rm objectnet-1.0.zip

YTBB Robust and ImageNet Vid-Robust

wget https://do-imagenet-classifiers-generalize-across-time.s3-us-west-2.amazonaws.com/imagenet_vid_ytbb_robust.tar.gz
tar -xvf imagenet_vid_ytbb_robust.tar.gz
mv imagenet_vid_ytbb_robust/* .
rm -rf imagenet_vid_ytbb_robust*

WILDS distribution shifts

export DATA_LOCATION=~/data
python utils/download_wilds_datasets.py

CIFAR distribution shifts

export DATA_LOCATION=~/data

# CIFAR10.1
mkdir -p $DATA_LOCATION/CIFAR-10.1
wget https://github.com/modestyachts/CIFAR-10.1/raw/master/datasets/cifar10.1_v6_data.npy -P $DATA_LOCATION/CIFAR-10.1
wget https://github.com/modestyachts/CIFAR-10.1/raw/master/datasets/cifar10.1_v6_labels.npy -P $DATA_LOCATION/CIFAR-10.1

# CIFAR10.2
mkdir -p $DATA_LOCATION/CIFAR-10.2
wget https://github.com/modestyachts/cifar-10.2/raw/61b0e3ac09809a2351379fb54331668cc9c975c4/cifar102_test.npy -P $DATA_LOCATION/CIFAR-10.2
wget https://github.com/modestyachts/cifar-10.2/raw/61b0e3ac09809a2351379fb54331668cc9c975c4/cifar102_train.npy -P $DATA_LOCATION/CIFAR-10.2