To address the limitations in describing the diversity of question randomization, we introduce a randomization framework that outlines six distinct randomization levels. We also include practical guides, including a decision tree and a usage table, to assist educators in effectively using the framework.
Published abstract https://doi.org/10.1145/3702652.3744222
@inproceedings{10.1145/3702652.3744222,
author = {Mahinpei, Romina and Xu, Iris and Wolfman, Steven and Moosvi, Firas},
title = {A Generalized Framework for Describing Question Randomization},
year = {2025},
isbn = {9798400713408},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3702652.3744222},
doi = {10.1145/3702652.3744222},
abstract = {The rise of digital assessments has motivated the development of randomized question banks, with randomization referring to the generation of question variants. Currently, questions are described as randomized or not. However, not all randomization techniques are equally effective in generating question isomorphs. They present different sets of benefits and challenges. To address the limitations in describing the diversity of randomization designs and implementations, we introduce a comprehensive framework that outlines six distinct randomization levels. Unlike existing taxonomies, we used a bottom-up approach grounded in empirical data to propose a framework that incorporates the randomization strategies instructors are already using. Additionally, we designed practical guides to assist educators in effectively using the framework, aligning with their pedagogical objectives. Through our application of this framework to classify 186 questions from two courses, we further highlight the generalizability of the framework and reveal insights into the considerations and challenges associated with incorporating question randomization into computer science curricula.},
booktitle = {Proceedings of the 2025 ACM Conference on International Computing Education Research V.1},
pages = {344–353},
numpages = {10},
keywords = {Question Randomization, Online Assessments, Question Design},
location = {
},
series = {ICER '25}
}