This is code for our paper Exploring Demonstration Ensembling for In-context Learning published at the workshop on understanding foundation models at ICLR 2023.
cd preprocess/
python _build_gym.py --build --n_proc=40 --do_test --test_k 6
python _build_gym.py --build --n_proc=40 --do_test --test_k 10
DATASET_NAME=glue-sst2
N_DEMOS=10 # number of demos
NBUCKETS=5 # number of demo buckets
ENS_METHOD=max # max, MoE, or PoE
WEIGHTED=true # whether to weigh buckets by similarity with a test example
CLUSTER=false # whether to use cluster the demos into bins
BSZ=64 # batch size
python test.py --max_length 1024 --model $MODEL --use_demonstrations \
--out_results_dir $OUTDIR \
--test_batch_size $BSZ \
--k $N_DEMOS \
--n_ensemble $NBINS \
--ensemble_method $ENS_METHOD \
--dist_ensemble $WEIGHTED \
--cluster_demos true \DATASET_NAME=glue-sst2
N_DEMOS=10 # number of demos
NBUCKETS=5 # number of demo buckets
ENS_METHOD=max # max, MoE, or PoE
WEIGHTED=true # whether to weigh buckets by similarity with a test example
CLUSTER=true # whether to use cluster the demos into bins
BSZ=64 # batch size
python test.py --max_length 1024 --model $MODEL --use_demonstrations \
--out_results_dir $OUTDIR \
--test_batch_size $BSZ \
--k $N_DEMOS \
--n_ensemble $NBINS \
--ensemble_method $ENS_METHOD \
--dist_ensemble $WEIGHTED \
--cluster_demos true \
--dissimilar_together falseFor diverse clustering, pass --dissimilar_together true. For similar-together clustering, pass --dissimilar_together false
This code here is largely based on Rethinking Demonstrations, so many thanks to the authors of that amazing repo!
If you use this code, please consider citing our paper:
@inproceedings{khalifa2023exploring,
title={Exploring Demonstration Ensembling for In-context Learning},
author={Khalifa, Muhammad and Logeswaran, Lajanugen and Lee, Moontae and Lee, Honglak and Wang, Lu},
booktitle={ICLR 2023 Workshop on Mathematical and Empirical Understanding of Foundation Models},
year={2023}
}