@misc{indiciae0f37f86edf77,
  title = {Policy Optimization in RLHF: The Impact of Out-of-preference Data},
  author = {Ziniu Li and Tian Xu and Yang Yu},
  year = {2024},
  url = {https://arxiv.org/abs/2312.10584},
  note = {Source identifier: 2312.10584}
}