HyperOpt-ASHA for Contrastive Learning Models

Author: Praroop Chanda

.DS_Store

@@ -0,0 +1,2 @@
+*.pth filter=lfs diff=lfs merge=lfs -text
+SogCLR_AdamW_checkpoint.pth filter=lfs diff=lfs merge=lfs -text

.gitattributes

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+##NECESSARY JOB SPECIFICATIONS
+#SBATCH --time=1:00:00
+#SBATCH --mem=40G
+#SBATCH --ntasks=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=8
+#SBATCH --gpus-per-task=1
+#SBATCH --partition=gpu
+#SBATCH --output=final_eval_clip_adamW.out
+
+#First Executable Line
+conda init  
+export HF_DATASETS_OFFLINE=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+source ~/.bashrc
+conda activate myenv
+cd ./iSogCLR
+echo "Running file"
+
+#export PYTHONPATH="$PYTHONPATH:./bimodal_exps"
+#export HUGGINGFACE_HUB_CACHE='./checkpoints/huggingface'
+
+data_path=../datasets
+ann_path=../clip_train
+train_image_root=cc3m_subset_100k/
+data=cc3m
+train_file=${data}_train_subset_90.json
+gamma=0.8
+epochs=30
+ita_type=clip
+
+CUDA_VISIBLE_DEVICES=0 python ./iSogCLR/bimodal_exps/clip2.py \
+    --data_path ${data_path} \
+    --ann_path ${ann_path} \
+    --train_file ${train_file} \
+    --train_image_root ${train_image_root} \
+    --output_dir ../datasets/Besteval/clip_adamW \
+    --init_model \
+    --use_amp \
+    --ita_type ${ita_type} \
+    --tau_init 0.01 \
+    --sogclr_gamma ${gamma} \
+    --eta_init 0.03 --sched cosine \
+    --no-distributed \
+    --lr 6.732199259338269e-05 \
+    --temp 0.06547641559385874 \
+    --epochs ${epochs} \
+    --evaluate --checkpoint ../datasets/Bestoutput/clip_adamw/checkpoint_26.pth \
+    --zs_dataset imagenet --zs_datafolder ../datasets/imagenet/val
+
+

DL_Final_Report.pdf

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+##NECESSARY JOB SPECIFICATIONS
+#SBATCH --time=1:00:00
+#SBATCH --mem=40G
+#SBATCH --ntasks=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=8
+#SBATCH --gpus-per-task=1
+#SBATCH --partition=gpu
+#SBATCH --output=final_eval_clip_nadam.out
+
+#First Executable Line
+conda init  
+export HF_DATASETS_OFFLINE=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+source ~/.bashrc
+conda activate myenv
+cd ./iSogCLR
+echo "Running file"
+
+#export PYTHONPATH="$PYTHONPATH:./bimodal_exps"
+#export HUGGINGFACE_HUB_CACHE='./checkpoints/huggingface'
+
+data_path=../datasets
+ann_path=../clip_train
+train_image_root=cc3m_subset_100k/
+data=cc3m
+train_file=${data}_train_subset_90.json
+gamma=0.8
+epochs=30
+ita_type=clip
+
+CUDA_VISIBLE_DEVICES=0 python ./iSogCLR/bimodal_exps/clip2.py \
+    --data_path ${data_path} \
+    --ann_path ${ann_path} \
+    --train_file ${train_file} \
+    --train_image_root ${train_image_root} \
+    --output_dir ../datasets/Besteval/clip_nadam \
+    --init_model \
+    --opt nadam \
+    --use_amp \
+    --ita_type ${ita_type} \
+    --tau_init 0.01 \
+    --sogclr_gamma ${gamma} \
+    --eta_init 0.03 --sched cosine \
+    --no-distributed \
+    --lr 0.0001001267832975453 \
+    --temp 0.0016226846798753232 \
+    --epochs ${epochs} \
+    --evaluate --checkpoint ../datasets/Bestoutput/clip_nadam/checkpoint_30.pth \
+    --zs_dataset imagenet --zs_datafolder ../datasets/imagenet/val
+
+

Eval_Job_Runs/EvalJob_clip_adamW.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+##NECESSARY JOB SPECIFICATIONS
+#SBATCH --time=1:00:00
+#SBATCH --mem=40G
+#SBATCH --ntasks=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=8
+#SBATCH --gpus-per-task=1
+#SBATCH --partition=gpu
+#SBATCH --output=final_eval_cyclip_adamW.out
+
+#First Executable Line
+conda init  
+export HF_DATASETS_OFFLINE=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+source ~/.bashrc
+conda activate myenv
+cd ./iSogCLR
+echo "Running file"
+
+#export PYTHONPATH="$PYTHONPATH:./bimodal_exps"
+#export HUGGINGFACE_HUB_CACHE='./checkpoints/huggingface'
+
+data_path=../datasets
+ann_path=../clip_train
+train_image_root=cc3m_subset_100k/
+data=cc3m
+train_file=${data}_train_subset_90.json
+gamma=0.5390333896985938
+epochs=30
+ita_type=cyclip
+
+CUDA_VISIBLE_DEVICES=0 python ./iSogCLR/bimodal_exps/clip2.py \
+    --data_path ${data_path} \
+    --ann_path ${ann_path} \
+    --train_file ${train_file} \
+    --train_image_root ${train_image_root} \
+    --output_dir ../datasets/Besteval/cyclip_adamW \
+    --init_model \
+    --use_amp \
+    --ita_type ${ita_type} \
+    --tau_init 0.01 \
+    --sogclr_gamma ${gamma} \
+    --eta_init 0.03 --sched cosine \
+    --no-distributed \
+    --lr 0.0002828819771699495 \
+    --temp 0.01621707878411723 \
+    --cylambda_1 0.5940949145191814 \
+    --cylambda_2  0.4976852107138287 \
+    --epochs ${epochs} \
+    --evaluate --checkpoint ../datasets/Bestoutput/cyclip_adamw/checkpoint_30.pth \
+    --zs_dataset imagenet --zs_datafolder ../datasets/imagenet/val

Eval_Job_Runs/EvalJob_clip_nadam.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+##NECESSARY JOB SPECIFICATIONS
+#SBATCH --time=1:00:00
+#SBATCH --mem=40G
+#SBATCH --ntasks=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=8
+#SBATCH --gpus-per-task=1
+#SBATCH --partition=gpu
+#SBATCH --output=final_eval_cyclip_nadam.out
+
+#First Executable Line
+conda init  
+export HF_DATASETS_OFFLINE=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+source ~/.bashrc
+conda activate myenv
+cd ./iSogCLR
+echo "Running file"
+
+#export PYTHONPATH="$PYTHONPATH:./bimodal_exps"
+#export HUGGINGFACE_HUB_CACHE='./checkpoints/huggingface'
+
+data_path=../datasets
+ann_path=../clip_train
+train_image_root=cc3m_subset_100k/
+data=cc3m
+train_file=${data}_train_subset_90.json
+gamma=0.5390333896985938
+epochs=30
+ita_type=cyclip
+
+CUDA_VISIBLE_DEVICES=0 python ./iSogCLR/bimodal_exps/clip2.py \
+    --data_path ${data_path} \
+    --ann_path ${ann_path} \
+    --train_file ${train_file} \
+    --train_image_root ${train_image_root} \
+    --output_dir ../datasets/Besteval/cyclip_nadam \
+    --init_model \
+    --use_amp \
+    --opt nadam \
+    --ita_type ${ita_type} \
+    --tau_init 0.01 \
+    --sogclr_gamma ${gamma} \
+    --eta_init 0.03 --sched cosine \
+    --no-distributed \
+    --lr 0.00012587716005886245 \
+    --temp 0.005232218198443648 \
+    --cylambda_1 0.45384301584103603 \
+    --cylambda_2  0.6589497618647435 \
+    --epochs ${epochs} \
+    --evaluate --checkpoint ../datasets/Bestoutput/cyclip_nadam/checkpoint_30.pth \
+    --zs_dataset imagenet --zs_datafolder ../datasets/imagenet/val

Eval_Job_Runs/EvalJob_cyclip_adamW.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+##NECESSARY JOB SPECIFICATIONS
+#SBATCH --time=1:00:00
+#SBATCH --mem=40G
+#SBATCH --ntasks=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=8
+#SBATCH --gpus-per-task=1
+#SBATCH --partition=gpu
+#SBATCH --output=final_eval_sogclr_adamW.out
+
+#First Executable Line
+conda init  
+export HF_DATASETS_OFFLINE=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+source ~/.bashrc
+conda activate myenv
+cd ./iSogCLR
+echo "Running file"
+
+#export PYTHONPATH="$PYTHONPATH:./bimodal_exps"
+#export HUGGINGFACE_HUB_CACHE='./checkpoints/huggingface'
+
+data_path=../datasets
+ann_path=../clip_train
+train_image_root=cc3m_subset_100k/
+data=cc3m
+train_file=${data}_train_subset_90.json
+gamma=0.5390333896985938
+epochs=30
+ita_type=sogclr
+
+CUDA_VISIBLE_DEVICES=0 python ./iSogCLR/bimodal_exps/clip2.py \
+    --data_path ${data_path} \
+    --ann_path ${ann_path} \
+    --train_file ${train_file} \
+    --train_image_root ${train_image_root} \
+    --output_dir ../datasets/Besteval/sogclr_adamW \
+    --init_model \
+    --use_amp \
+    --ita_type ${ita_type} \
+    --tau_init 0.01 \
+    --sogclr_gamma ${gamma} \
+    --eta_init 0.03 --sched cosine \
+    --no-distributed \
+    --lr 2.110170394664965e-05 \
+    --temp 0.0559451134595352 \
+    --epochs ${epochs} \
+    --evaluate --checkpoint ../datasets/Bestoutput/sogclr_adamw/checkpoint_25.pth \
+    --zs_dataset imagenet --zs_datafolder ../datasets/imagenet/val

Eval_Job_Runs/EvalJob_cyclip_nadam.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+##NECESSARY JOB SPECIFICATIONS
+#SBATCH --time=1:00:00
+#SBATCH --mem=40G
+#SBATCH --ntasks=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=8
+#SBATCH --gpus-per-task=1
+#SBATCH --partition=gpu
+#SBATCH --output=final_eval_sogclr_nadam.out
+
+#First Executable Line
+conda init  
+export HF_DATASETS_OFFLINE=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+source ~/.bashrc
+conda activate myenv
+cd ./iSogCLR
+echo "Running file"
+
+#export PYTHONPATH="$PYTHONPATH:./bimodal_exps"
+#export HUGGINGFACE_HUB_CACHE='./checkpoints/huggingface'
+
+data_path=../datasets
+ann_path=../clip_train
+train_image_root=cc3m_subset_100k/
+data=cc3m
+train_file=${data}_train_subset_90.json
+gamma=0.623548869476773
+epochs=30
+ita_type=sogclr
+
+CUDA_VISIBLE_DEVICES=0 python ./iSogCLR/bimodal_exps/clip2.py \
+    --data_path ${data_path} \
+    --ann_path ${ann_path} \
+    --train_file ${train_file} \
+    --train_image_root ${train_image_root} \
+    --output_dir ../datasets/Besteval/sogclr_nadam \
+    --init_model \
+    --opt nadam \
+    --use_amp \
+    --ita_type ${ita_type} \
+    --tau_init 0.01 \
+    --sogclr_gamma ${gamma} \
+    --eta_init 0.03 --sched cosine \
+    --no-distributed \
+    --lr 0.00041892083091233187 \
+    --temp  0.05353708355932494 \
+    --epochs ${epochs} \
+    --evaluate --checkpoint ../datasets/Bestoutput/sogclr_nadam/checkpoint_30.pth \
+    --zs_dataset imagenet --zs_datafolder ../datasets/imagenet/val

Eval_Job_Runs/EvalJob_sogclr_adamW.sh

@@ -0,0 +1,37 @@
+DL-Final Project -636
+
+Team Details:
+* Ishaan Singh Rawal
+* Praroop Chanda
+For any assistance in running the code, please contact: -
+[email protected]
+[email protected]
+README:
+* To perform hyperparameter tuning using Ray Tune, execute the job files located in the Ray_Tune_Hyperparameter folder. The files are named in the following format:
+
+MyJobRaytune_{Model_Name}_{Optimizer}.sh
+
+* To Train and validate the code on best tuned hyperparameters, execute job files located in best_job_runs folder. The files are named in the following format:
+	
+MyJob{Model_Name}_bestlr_{Optimizer}.sh
+
+* To evaluate/Test the code, execute job files located in Eval_Job_Runs folder. The files are named in the following format:
+
+EvalJob_{Model_Name}_{Optimizer}.sh
+
+HereÕs how to use them:
+1. Replace {Model_Name} with the desired model name.
+2. Replace {Optimizer} with the optimizer of your choice.
+
+*** The Training code is available in the Clip1.py file present inside in the iSogCLR>bimodal_exps folder.
+*** The code used for hyper-parameter tuning is raytune_eval_search.py present inside iSogCLR>bimodal_exps folder.
+*** The clip_train folder contains the json files for training and validation sets.
+
+Link to Pickle file - https://drive.google.com/file/d/1PRy0jNMSTYXJRBwl47RXU-5ijjZoRV1L/view?usp=sharing
+
+
+
+
+
+
+