sudo apt-get install git-lfs python3.10-dev

git lfs install --skip-repo

git lfs pull origin

pip3 install transformers faiss-gpu accelerate k-means-constrained pandas gin-config tensorboard
pip3 install torch torchvision torchaudio
pip3 install fbgemm-gpu --index-url https://download.pytorch.org/whl/cu121/

mkdir -p tmp/ && python3 preprocess_public_data.py

python3 eval_batch.py

@misc{ding2024retrievallearnedsimilarities,
      title={Efficient Retrieval with Learned Similarities}, 
      author={Bailu Ding and Jiaqi Zhai},
      year={2024},
      eprint={2407.15462},
      archivePrefix={arXiv},
      primaryClass={cs.IR},
      url={https://arxiv.org/abs/2407.15462}, 
}

@inproceedings{10.1145/3580305.3599897,
      author = {Zhai, Jiaqi and Gong, Zhaojie and Wang, Yueming and Sun, Xiao and Yan, Zheng and Li, Fu and Liu, Xing},
      title = {Revisiting Neural Retrieval on Accelerators},
      year = {2023},
      isbn = {9798400701030},
      publisher = {Association for Computing Machinery},
      address = {New York, NY, USA},
      url = {https://doi.org/10.1145/3580305.3599897},
      doi = {10.1145/3580305.3599897},
      abstract = {Retrieval finds a small number of relevant candidates from a large corpus for information retrieval and recommendation applications. A key component of retrieval is to model (user, item) similarity, which is commonly represented as the dot product of two learned embeddings. This formulation permits efficient inference, commonly known as Maximum Inner Product Search (MIPS). Despite its popularity, dot products cannot capture complex user-item interactions, which are multifaceted and likely high rank. We hence examine non-dot-product retrieval settings on accelerators, and propose mixture of logits (MoL), which models (user, item) similarity as an adaptive composition of elementary similarity functions. This new formulation is expressive, capable of modeling high rank (user, item) interactions, and further generalizes to the long tail. When combined with a hierarchical retrieval strategy, h-indexer, we are able to scale up MoL to 100M corpus on a single GPU with latency comparable to MIPS baselines. On public datasets, our approach leads to uplifts of up to 77.3\% in hit rate (HR). Experiments on a large recommendation surface at Meta showed strong metric gains and reduced popularity bias, validating the proposed approach's performance and improved generalization.},
      booktitle = {Proceedings of the 29th ACM SIGKDD Conference on Knowledge Discovery and Data Mining},
      pages = {5520–5531},
      numpages = {12},
      keywords = {candidate generation, hierarchical retrieval, information retrieval, nearest neighbor search, non-mips retrieval, recommender systems},
      location = {Long Beach, CA, USA},
      series = {KDD '23}
}