Retrieval-augmented language models (LMs) use non-parametric memory to
substantially outperform their non-retrieval counterparts on perplexity-based
evaluations, but it is an open question whether they achieve similar gains in
few- and zero-shot end-task accuracy. We extensively study one such model, the
k-nearest neighbor LM (kNN-LM), showing that the gains marginally transfer. The
main challenge is to achieve coverage of the verbalizer tokens that define the
different end-task class labels. To address this challenge, we also introduce
kNN-Prompt, a simple and effective kNN-LM with automatically expanded fuzzy
verbalizers (e.g. to expand terrible to also include silly and other
task-specific synonyms for sentiment classification). Across nine diverse
end-tasks, using kNN-Prompt with GPT-2 large yields significant performance
boosts over strong zero-shot baselines (13.4% absolute improvement over the
base LM on average). We also show that other advantages of non-parametric
augmentation hold for end tasks; kNN-Prompt is effective for domain adaptation
with no further training, and gains increase with the size of the retrieval
model.
%0 Generic
%1 shi2022knnprompt
%A Shi, Weijia
%A Michael, Julian
%A Gururangan, Suchin
%A Zettlemoyer, Luke
%D 2022
%K llm machinelearning
%T kNN-Prompt: Nearest Neighbor Zero-Shot Inference
%U http://arxiv.org/abs/2205.13792
%X Retrieval-augmented language models (LMs) use non-parametric memory to
substantially outperform their non-retrieval counterparts on perplexity-based
evaluations, but it is an open question whether they achieve similar gains in
few- and zero-shot end-task accuracy. We extensively study one such model, the
k-nearest neighbor LM (kNN-LM), showing that the gains marginally transfer. The
main challenge is to achieve coverage of the verbalizer tokens that define the
different end-task class labels. To address this challenge, we also introduce
kNN-Prompt, a simple and effective kNN-LM with automatically expanded fuzzy
verbalizers (e.g. to expand terrible to also include silly and other
task-specific synonyms for sentiment classification). Across nine diverse
end-tasks, using kNN-Prompt with GPT-2 large yields significant performance
boosts over strong zero-shot baselines (13.4% absolute improvement over the
base LM on average). We also show that other advantages of non-parametric
augmentation hold for end tasks; kNN-Prompt is effective for domain adaptation
with no further training, and gains increase with the size of the retrieval
model.
@misc{shi2022knnprompt,
abstract = {Retrieval-augmented language models (LMs) use non-parametric memory to
substantially outperform their non-retrieval counterparts on perplexity-based
evaluations, but it is an open question whether they achieve similar gains in
few- and zero-shot end-task accuracy. We extensively study one such model, the
k-nearest neighbor LM (kNN-LM), showing that the gains marginally transfer. The
main challenge is to achieve coverage of the verbalizer tokens that define the
different end-task class labels. To address this challenge, we also introduce
kNN-Prompt, a simple and effective kNN-LM with automatically expanded fuzzy
verbalizers (e.g. to expand terrible to also include silly and other
task-specific synonyms for sentiment classification). Across nine diverse
end-tasks, using kNN-Prompt with GPT-2 large yields significant performance
boosts over strong zero-shot baselines (13.4% absolute improvement over the
base LM on average). We also show that other advantages of non-parametric
augmentation hold for end tasks; kNN-Prompt is effective for domain adaptation
with no further training, and gains increase with the size of the retrieval
model.},
added-at = {2023-03-02T05:34:46.000+0100},
author = {Shi, Weijia and Michael, Julian and Gururangan, Suchin and Zettlemoyer, Luke},
biburl = {https://www.bibsonomy.org/bibtex/2d5cf44a2e205b76581267bf7c928cf1a/sairahul},
description = {kNN-Prompt: Nearest Neighbor Zero-Shot Inference},
interhash = {92f5ca87ef75a50fac32eefef4235656},
intrahash = {d5cf44a2e205b76581267bf7c928cf1a},
keywords = {llm machinelearning},
note = {cite arxiv:2205.13792},
timestamp = {2023-03-02T05:34:46.000+0100},
title = {kNN-Prompt: Nearest Neighbor Zero-Shot Inference},
url = {http://arxiv.org/abs/2205.13792},
year = 2022
}