AppWorld-UL: Benchmarking Diverse Agent-User Interactions for Tool-Use
ICML 2026
Conference
Junzhi Chen*, Harsh Trivedi*, Jane Pan, Michael JQ Zhang, Tejas Srinivasan, Niranjan Balasubramanian, Ashish Sabharwal
Paper
Website
Code
Copy Bib
@inproceedings{appworld-ul-icml26,
title={App{W}orld-{UL}: Benchmarking Diverse Agent-User Interactions for Tool-Use},
author={Junzhi Chen and Harsh Trivedi and Jane Pan and Michael JQ Zhang and Tejas Srinivasan and Niranjan Balasubramanian and Ashish Sabharwal},
booktitle={International Conference on Machine Learning},
year={2026}
}
Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces
ICLR 2026
Conference
Mike A. Merrill, Alexander G. Shaw, et al. (including Harsh Trivedi)
Paper
Website
Code
Copy Bib
@inproceedings{terminal-bench-iclr26,
title={Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces},
author={Mike A. Merrill and Alexander G. Shaw and Nicholas Carlini and Boxuan Li and others},
booktitle={International Conference on Learning Representations},
year={2026},
url={https://arxiv.org/abs/2601.11868}
}
Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation
ICLR 2026
Conference
Sayash Kapoor, Benedikt Stroebl, et al. (including Harsh Trivedi)
Paper
Website
Copy Bib
@inproceedings{hal-iclr26,
title={Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation},
author={Sayash Kapoor and Benedikt Stroebl and Peter Kirgis and Nitya Nadgir and Zachary S. Siegel and Boyi Wei and Tianci Xue and Ziru Chen and Felix Chen and Saiteja Utpala and Franck Ndzomga and Dheeraj Oruganty and Sophie Luskin and Kangheng Liu and Botao Yu and Amit Arora and Dongyoon Hahm and Harsh Trivedi and Huan Sun and Juyong Lee and Tengjun Jin and Yifan Mai and Yifei Zhou and Yuxuan Zhu and Rishi Bommasani and Daniel Kang and Dawn Song and Peter Henderson and Yu Su and Percy Liang and Arvind Narayanan},
booktitle={International Conference on Learning Representations},
year={2026},
url={https://arxiv.org/abs/2510.11977}
}
MolmoWeb: Open Visual Web Agent and Open Data for the Open Web
ECCV 2026
Conference
Tanmay Gupta, Piper Wolters, Zixian Ma, Peter Sushko, et. al (including Harsh Trivedi)
Paper
ECCV
Blog
Code
Copy Bib
@inproceedings{molmoweb-eccv26,
title={{MolmoWeb}: Open Visual Web Agent and Open Data for the Open Web},
author={Tanmay Gupta and Piper Wolters and Zixian Ma and Peter Sushko and Rock Yuren Pang and Diego Llanes and Yue Yang and Taira Anderson and Boyuan Zheng and Zhongzheng Ren and Harsh Trivedi and Taylor Blanton and Caleb Ouellette and Winson Han and Ali Farhadi and Ranjay Krishna},
booktitle={European Conference on Computer Vision},
year={2026}
}
MoNaCo: More Natural and Complex Questions for Reasoning Across Dozens of Documents
TACL 2026
Journal
Tomer Wolfson, Harsh Trivedi, Mor Geva, Yoav Goldberg, Dan Roth, Tushar Khot, Ashish Sabharwal, Reut Tsarfaty
Paper
Website
Copy Bib
@article{wolfson-etal-2026-monaco,
title={{M}o{N}a{C}o: More Natural and Complex Questions for Reasoning Across Dozens of Documents},
author={Tomer Wolfson and Harsh Trivedi and Mor Geva and Yoav Goldberg and Dan Roth and Tushar Khot and Ashish Sabharwal and Reut Tsarfaty},
journal={Transactions of the Association for Computational Linguistics},
volume={14},
pages={23--46},
year={2026},
doi={10.1162/tacl.a.64}
}
Olmo 3
arXiv 2025
Tech report
Team Olmo (including Harsh Trivedi)
Paper
Website
Code
Copy Bib
@article{olmo3-arxiv25,
title={Olmo 3},
author={{Team Olmo}},
journal={arXiv preprint arXiv:2512.13961},
year={2025}
}
AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents
ACL 2024
Conference
🏆 Best Resource Paper Award
Harsh Trivedi, Tushar Khot, Mareike Hartmann, Ruskin Manku, Vinty Dong, Edward Li, Shashank Gupta, Ashish Sabharwal, Niranjan Balasubramanian
Paper
Website
Tweet
Talks
Blog
Poster
Code
Leaderboard
Copy Bib
@inproceedings{appworld-acl24,
title={App{W}orld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents},
author={Harsh Trivedi and Tushar Khot and Mareike Hartmann and Ruskin Manku and Vinty Dong and Edward Li and Shashank Gupta and Ashish Sabharwal and Niranjan Balasubramanian},
booktitle={ACL},
year={2024}
}
Interleaving Retrieval with Chain-of-Thought Reasoning for Knowledge-Intensive Multi-Step Questions
ACL 2023
Conference
Harsh Trivedi, Niranjan Balasubramanian, Tushar Khot, Ashish Sabharwal
Paper
Tweet
Video
Code
Copy Bib
@inproceedings{ircot-acl23,
title = "Interleaving Retrieval with Chain-of-Thought Reasoning for Knowledge-Intensive Multi-Step Questions",
author = "Trivedi, Harsh and
Balasubramanian, Niranjan and
Khot, Tushar and
Sabharwal, Ashish",
booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
month = jul,
year = "2023",
address = "Toronto, Canada",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2023.acl-long.557",
doi = "10.18653/v1/2023.acl-long.557",
pages = "10014--10037",
}
Decomposed Prompting: A Modular Approach for Solving Complex Tasks
ICLR 2023
Conference
Tushar Khot, Harsh Trivedi, Matthew Finlayson, Yao Fu, Kyle Richardson, Peter Clark, Ashish Sabharwal
Paper
Code
Copy Bib
@inproceedings{
decomp-iclr23,
title={Decomposed Prompting: A Modular Approach for Solving Complex Tasks},
author={Tushar Khot and Harsh Trivedi and Matthew Finlayson and Yao Fu and Kyle Richardson and Peter Clark and Ashish Sabharwal},
booktitle={The Eleventh International Conference on Learning Representations },
year={2023},
url={https://openreview.net/forum?id=_nGgzQjzaRy}
}
Two-Turn Debate Does Not Help Humans Answer Hard Reading-Comprehension Questions
ML Safety @ NeurIPS 2022
Workshop
🏆 Best Paper Award
Alicia Parrish*, Harsh Trivedi*, Nikita Nangia, Vishakh Padmakumar, Jason Phang, Amanpreet Singh Saimbhi, Samuel R. Bowman
Paper
Copy Bib
@article{two-turn-debate-neurips22,
title={Two-Turn Debate Doesn't Help Humans Answer Hard Reading Comprehension Questions},
author={Parrish, Alicia and Trivedi, Harsh and Nangia, Nikita and Padmakumar, Vishakh and Phang, Jason and Saimbhi, Amanpreet Singh and Bowman, Samuel R},
journal={arXiv preprint arXiv:2210.10860},
year={2022}
}
Teaching Broad Reasoning Skills for Multi-Step QA by Generating Hard Contexts
EMNLP 2022
Conference
Harsh Trivedi, Niranjan Balasubramanian, Tushar Khot, Ashish Sabharwal
Paper
Video
Code
Copy Bib
@inproceedings{teabreac-emnlp22,
title = "Teaching Broad Reasoning Skills for Multi-Step {QA} by Generating Hard Contexts",
author = "Trivedi, Harsh and
Balasubramanian, Niranjan and
Khot, Tushar and
Sabharwal, Ashish",
booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",
month = dec,
year = "2022",
address = "Abu Dhabi, United Arab Emirates",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2022.emnlp-main.439",
doi = "10.18653/v1/2022.emnlp-main.439",
pages = "6541--6566",
}
MuSiQue: Multihop Questions via Single-hop Question Composition
TACL, presented at NAACL 2022
Journal
Harsh Trivedi, Niranjan Balasubramanian, Tushar Khot, Ashish Sabharwal
Paper
Video
Code
Leaderboard
Copy Bib
@article{musique-tacl22,
title={{M}u{S}i{Q}ue: Multihop Questions via Single-hop Question Composition},
author={Trivedi, Harsh and Balasubramanian, Niranjan and Khot, Tushar and Sabharwal, Ashish},
journal={Transactions of the Association for Computational Linguistics},
year={2022}
publisher={MIT Press}
}
Single-Turn Debate Does Not Help Humans Answer Hard Reading-Comprehension Questions
Learning with Natural Language Supervision @ ACL 2022
Workshop
Alicia Parrish*, Harsh Trivedi*, Ethan Perez, Angelica Chen, Nikita Nangia, Jason Phang, Samuel R. Bowman
Paper
Video
Data
Copy Bib
@article{one-turn-debate-acl22,
title={Single-turn debate does not help humans answer hard reading-comprehension questions},
author={Parrish, Alicia and Trivedi, Harsh and Perez, Ethan and Chen, Angelica and Nangia, Nikita and Phang, Jason and Bowman, Samuel R},
journal={arXiv preprint arXiv:2204.05212},
year={2022}
}
Summarize-then-Answer: Generating Concise Explanations for Multi-hop Reading Comprehension
EMNLP 2021
Conference
Naoya Inoue, Harsh Trivedi, Steven Sinha, Niranjan Balasubramanian, Kentaro Inui
Paper
Video
Code
Copy Bib
@inproceedings{suqa-emnlp21,
title = "Summarize-then-Answer: Generating Concise Explanations for Multi-hop Reading Comprehension",
author = "Inoue, Naoya and
Trivedi, Harsh and
Sinha, Steven and
Balasubramanian, Niranjan and
Inui, Kentaro",
editor = "Moens, Marie-Francine and
Huang, Xuanjing and
Specia, Lucia and
Yih, Scott Wen-tau",
booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
month = nov,
year = "2021",
address = "Online and Punta Cana, Dominican Republic",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2021.emnlp-main.490",
doi = "10.18653/v1/2021.emnlp-main.490",
pages = "6064--6080",
}
IrEne-viz: Visualizing Energy Consumption of Transformer Models
EMNLP 2021
Demo
Yash Kumar Lal, Reetu Singh, Harsh Trivedi, Qingqing Cao, Aruna Balasubramanian, Niranjan Balasubramanian
Paper
Code
Copy Bib
@inproceedings{ireneviz-emnlp21,
title = "{I}r{E}ne-viz: Visualizing Energy Consumption of Transformer Models",
author = "Lal, Yash Kumar and
Singh, Reetu and
Trivedi, Harsh and
Cao, Qingqing and
Balasubramanian, Aruna and
Balasubramanian, Niranjan",
editor = "Adel, Heike and
Shi, Shuming",
booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing: System Demonstrations",
month = nov,
year = "2021",
address = "Online and Punta Cana, Dominican Republic",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2021.emnlp-demo.29",
doi = "10.18653/v1/2021.emnlp-demo.29",
pages = "251--258",
}
What Ingredients Make for an Effective Crowdsourcing Protocol for Difficult NLU Data Collection Tasks?
ACL 2021
Conference
Nikita Nangia*, Saku Sugawara*, Harsh Trivedi, Alex Warstadt, Clara Vania, Samuel R. Bowman
Paper
Video
Code
Copy Bib
@inproceedings{crowdsourcing-ingredients-acl21,
title = "What Ingredients Make for an Effective Crowdsourcing Protocol for Difficult {NLU} Data Collection Tasks?",
author = "Nangia, Nikita and
Sugawara, Saku and
Trivedi, Harsh and
Warstadt, Alex and
Vania, Clara and
Bowman, Samuel R.",
editor = "Zong, Chengqing and
Xia, Fei and
Li, Wenjie and
Navigli, Roberto",
booktitle = "Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)",
month = aug,
year = "2021",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2021.acl-long.98",
doi = "10.18653/v1/2021.acl-long.98",
pages = "1221--1235",
}
IrEne: Interpretable Energy Prediction for Transformers
ACL 2021
Conference
Qingqing Cao, Yash Kumar Lal, Harsh Trivedi, Aruna Balasubramanian, Niranjan Balasubramanian
Paper
Video
Code
Copy Bib
@inproceedings{deformer-acl21,
title = "{I}r{E}ne: Interpretable Energy Prediction for Transformers",
author = "Cao, Qingqing and
Lal, Yash Kumar and
Trivedi, Harsh and
Balasubramanian, Aruna and
Balasubramanian, Niranjan",
editor = "Zong, Chengqing and
Xia, Fei and
Li, Wenjie and
Navigli, Roberto",
booktitle = "Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)",
month = aug,
year = "2021",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2021.acl-long.167",
doi = "10.18653/v1/2021.acl-long.167",
pages = "2145--2157",
}
Is Multihop QA in DiRe Condition? Measuring and Reducing Disconnected Reasoning
EMNLP 2020
Conference
Harsh Trivedi, Niranjan Balasubramanian, Tushar Khot, Ashish Sabharwal
Paper
Code
Video
Slides
Copy Bib
@inproceedings{dire-emnlp20,
title = "Is Multihop {QA} in {DiRe} Condition? Measuring and Reducing Disconnected Reasoning",
author = "Trivedi, Harsh and
Balasubramanian, Niranjan and
Khot, Tushar and
Sabharwal, Ashish",
editor = "Webber, Bonnie and
Cohn, Trevor and
He, Yulan and
Liu, Yang",
booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
month = nov,
year = "2020",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2020.emnlp-main.712",
doi = "10.18653/v1/2020.emnlp-main.712",
pages = "8846--8863",
}
DeFormer: Decomposing Pre-trained Transformers for Faster Question Answering
ACL 2020
Conference
Qingqing Cao, Harsh Trivedi, Aruna Balasubramanian, Niranjan Balasubramanian
Paper
Video
Slides
Code
Copy Bib
@inproceedings{deformer-acl20,
title = "{D}e{F}ormer: Decomposing Pre-trained Transformers for Faster Question Answering",
author = "Cao, Qingqing and
Trivedi, Harsh and
Balasubramanian, Aruna and
Balasubramanian, Niranjan",
editor = "Jurafsky, Dan and
Chai, Joyce and
Schluter, Natalie and
Tetreault, Joel",
booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
month = jul,
year = "2020",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2020.acl-main.411",
doi = "10.18653/v1/2020.acl-main.411",
pages = "4487--4497",
}
Repurposing Entailment for Multi-Hop Question Answering Tasks
NAACL 2019
Conference
Harsh Trivedi, Heeyoung Kwon, Tushar Khot, Ashish Sabharwal, Niranjan Balasubramanian
Paper
Code
Slides
Copy Bib
@inproceedings{multee-naacl19,
title = "Repurposing Entailment for Multi-Hop Question Answering Tasks",
author = "Trivedi, Harsh and
Kwon, Heeyoung and
Khot, Tushar and
Sabharwal, Ashish and
Balasubramanian, Niranjan",
editor = "Burstein, Jill and
Doran, Christy and
Solorio, Thamar",
booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",
month = jun,
year = "2019",
address = "Minneapolis, Minnesota",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/N19-1302",
doi = "10.18653/v1/N19-1302",
pages = "2948--2958",
}
Controlling Information Aggregation for Complex Question Answering
ECIR 2018
Conference
Heeyoung Kwon, Harsh Trivedi, Peter Jansen, Mihai Surdeanu, Niranjan Balasubramanian
Paper
Poster
Copy Bib
@inproceedings{aggregation-ecir19,
author="Kwon, Heeyoung
and Trivedi, Harsh
and Jansen, Peter
and Surdeanu, Mihai
and Balasubramanian, Niranjan",
editor="Pasi, Gabriella
and Piwowarski, Benjamin
and Azzopardi, Leif
and Hanbury, Allan",
title="Controlling Information Aggregation for Complex Question Answering",
booktitle="Advances in Information Retrieval",
year="2018",
publisher="Springer International Publishing",
address="Cham",
pages="750--757",
isbn="978-3-319-76941-7"
}
* denotes equal contribution