@inproceedings{e52d17f955db4eaf9190207542e03967,
title = "Online Runtime Prediction Method for Distributed Iterative Jobs",
abstract = "Predicting the runtime of distributed iterative jobs can help reduce the deployment cost of clusters and optimize their resource allocation and scheduling strategies, but the runtime depends on various factors which are difficult to be acquired before execution. In this paper, we propose a generalized online prediction method for the runtime of distributed iterative jobs, which is centered on a series of online machine learning models. The method consists of three phases: 1) estimating the number of iterations for the current iterative job. 2) predicting the runtime metrics of each iteration by an online polynomial regression model. 3) Runtime metrics sequence is analyzed using an LSTM trained with online learning to predict the runtime of each iteration. We conducted experiments on typical Flink iterative jobs, and the experimental results show that our method improves the accuracy by 4.79% compared to the state-of-the-art methods, while for the improvement in accuracy for delta iterative jobs is even more than 15%.",
keywords = "Flink, Iterative job, LSTM, Online runtime prediction, Polynomial regression",
author = "Xiaofei Yue and Lan Shi and Yuhai Zhao and Hangxu Ji and Guoren Wang",
note = "Publisher Copyright: {\textcopyright} 2021, Springer Nature Switzerland AG.; 18th International Conference on Web Information Systems and Applications, WISA 2021 ; Conference date: 24-09-2021 Through 26-09-2021",
year = "2021",
doi = "10.1007/978-3-030-87571-8_14",
language = "English",
isbn = "9783030875701",
series = "Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)",
publisher = "Springer Science and Business Media Deutschland GmbH",
pages = "156--168",
editor = "Chunxiao Xing and Xiaoming Fu and Yong Zhang and Guigang Zhang and Chaolemen Borjigin",
booktitle = "Web Information Systems and Applications - 18th International Conference, WISA 2021, Proceedings",
address = "Germany",
}