Description
The efficient implementation of complex DAWs in various scientific disciplines requires deep knowledge of a large stack – consisting of an abstract DAW description, compilation of a logical plan, mapping onto the currently available infrastructure, and appropriate configuration of execution engines. Components and configurations developed for one computational infrastructure are often unsuitable for another, either leading to an undesirable platform lock-in or to a considerable loss of efficiency.
The goal of subproject B1 is therefore to improve portability. To this end, we
- compare DAW requirements with declarative descriptions of the available infrastructure,
- profile both DAWs and infrastructure as needed, and
- then map the DAWs onto the infrastructure using novel scheduling and load balancing (SLB) techniques to automatically optimize efficiency.
Ultimately, we aim to allow scientists to focus on the domain-specific challenges in their DAWs, while our new components provide an efficient selection and use of the available computing infrastructure automatically.
PIs
Publications
2022
Jonathan Bader; Fabian Lehmann; Lauritz Thamsen; Jonathan Will; Ulf Leser; Odej Kao
Lotaru: Locally Estimating Runtimes of Scientific Workflow Tasks in Heterogeneous Clusters Inproceedings
In: 34th International Conference on Scientific and Statistical Database Management (SSDBM 2022), pp. to appear, ACM, 2022.
@inproceedings{baderLotaruLocallyEstimating2022,
title = {Lotaru: Locally Estimating Runtimes of Scientific Workflow Tasks in Heterogeneous Clusters},
author = {Jonathan Bader and Fabian Lehmann and Lauritz Thamsen and Jonathan Will and Ulf Leser and Odej Kao},
url = {https://arxiv.org/abs/2205.11181},
doi = {10.1145/3538712.3538739},
year = {2022},
date = {2022-07-01},
urldate = {2022-07-01},
booktitle = {34th International Conference on Scientific and Statistical Database Management (SSDBM 2022)},
pages = {to appear},
publisher = {ACM},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Lauritz Thamsen; Dominik Scheinert; Jonathan Will; Jonathan Bader; Odej Kao
Collaborative Cluster Configuration for Distributed Data-Parallel Processing: A Research Overview Journal Article
In: Datenbank-Spektrum, 2022.
@article{Thamsen2022-zu,
title = {Collaborative Cluster Configuration for Distributed Data-Parallel Processing: A Research Overview},
author = {Lauritz Thamsen and Dominik Scheinert and Jonathan Will and Jonathan Bader and Odej Kao},
year = {2022},
date = {2022-05-01},
urldate = {2022-05-01},
journal = {Datenbank-Spektrum},
abstract = {Many organizations routinely analyze large datasets using systems
for distributed data-parallel processing and clusters of
commodity resources. Yet, users need to configure adequate
resources for their data processing jobs. This requires
significant insights into expected job runtimes and scaling
behavior, resource characteristics, input data distributions, and
other factors. Unable to estimate performance accurately, users
frequently overprovision resources for their jobs, leading to low
resource utilization and high costs.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Many organizations routinely analyze large datasets using systems
for distributed data-parallel processing and clusters of
commodity resources. Yet, users need to configure adequate
resources for their data processing jobs. This requires
significant insights into expected job runtimes and scaling
behavior, resource characteristics, input data distributions, and
other factors. Unable to estimate performance accurately, users
frequently overprovision resources for their jobs, leading to low
resource utilization and high costs.
Jonathan Will; Lauritz Thamsen; Jonathan Bader; Dominik Scheinert; Odej Kao
Get Your Memory Right: The Crispy Resource Allocation Assistant for Large-Scale Data Processing Miscellaneous
2022.
@misc{will2022memory,
title = {Get Your Memory Right: The Crispy Resource Allocation Assistant for Large-Scale Data Processing},
author = {Jonathan Will and Lauritz Thamsen and Jonathan Bader and Dominik Scheinert and Odej Kao},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
keywords = {},
pubstate = {published},
tppubtype = {misc}
}
Jonathan Bader; Fabian Lehmann; Alexander Groth; Lauritz Thamsen; Dominik Scheinert; Jonathan Will; Ulf Leser; Odej Kao
Reshi: Recommending Resources for Scientific Workflow Tasks on Heterogeneous Infrastructures Inproceedings
In: 41th International Performance Computing and Communications Conference 2022, IEEE, 2022.
@inproceedings{baderReshi2022IPCCC,
title = {Reshi: Recommending Resources for Scientific Workflow Tasks on Heterogeneous Infrastructures},
author = {Jonathan Bader and Fabian Lehmann and Alexander Groth and Lauritz Thamsen and Dominik Scheinert and Jonathan Will and Ulf Leser and Odej Kao},
year = {2022},
date = {2022-01-01},
urldate = {2022-01-01},
booktitle = {41th International Performance Computing and Communications Conference 2022},
publisher = {IEEE},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2021
Jonathan Will; Lauritz Thamsen; Dominik Scheinert; Jonathan Bader; Odej Kao
C3O: Collaborative Cluster Configuration Optimization for Distributed Data Processing in Public Clouds Journal Article
In: 2021 IEEE International Conference on Cloud Engineering (IC2E), pp. 43-52, 2021.
@article{Will2021C3OCC,
title = {C3O: Collaborative Cluster Configuration Optimization for Distributed Data Processing in Public Clouds},
author = {Jonathan Will and Lauritz Thamsen and Dominik Scheinert and Jonathan Bader and Odej Kao},
year = {2021},
date = {2021-01-01},
urldate = {2021-01-01},
journal = {2021 IEEE International Conference on Cloud Engineering (IC2E)},
pages = {43-52},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Dominik Scheinert; Alireza Alamgiralem; Jonathan Bader; Jonathan Will; Thorsten Wittkopp; Lauritz Thamsen
On the Potential of Execution Traces for Batch Processing Workload Optimization in Public Clouds Inproceedings
In: 2021 IEEE International Conference on Big Data (Big Data), pp. 3113-3118, 2021.
@inproceedings{9671275,
title = {On the Potential of Execution Traces for Batch Processing Workload Optimization in Public Clouds},
author = {Dominik Scheinert and Alireza Alamgiralem and Jonathan Bader and Jonathan Will and Thorsten Wittkopp and Lauritz Thamsen},
doi = {10.1109/BigData52589.2021.9671275},
year = {2021},
date = {2021-01-01},
urldate = {2021-01-01},
booktitle = {2021 IEEE International Conference on Big Data (Big Data)},
pages = {3113-3118},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Jonathan Will; Onur Arslan; Jonathan Bader; Dominik Scheinert; Lauritz Thamsen
Training Data Reduction for Performance Models of Data Analytics Jobs in the Cloud Inproceedings
In: 2021 IEEE International Conference on Big Data (Big Data), pp. 3141-3146, 2021.
@inproceedings{9671742,
title = {Training Data Reduction for Performance Models of Data Analytics Jobs in the Cloud},
author = {Jonathan Will and Onur Arslan and Jonathan Bader and Dominik Scheinert and Lauritz Thamsen},
doi = {10.1109/BigData52589.2021.9671742},
year = {2021},
date = {2021-01-01},
urldate = {2021-01-01},
booktitle = {2021 IEEE International Conference on Big Data (Big Data)},
pages = {3141-3146},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Jonathan Bader; Lauritz Thamsen; Svetlana Kulagina; Jonathan Will; Henning Meyerhenke; Odej Kao
Tarema: Adaptive Resource Allocation for Scalable Scientific Workflows in Heterogeneous Clusters Inproceedings
In: 2021 IEEE International Conference on Big Data (Big Data), pp. 65-75, 2021.
@inproceedings{9671519,
title = {Tarema: Adaptive Resource Allocation for Scalable Scientific Workflows in Heterogeneous Clusters},
author = {Jonathan Bader and Lauritz Thamsen and Svetlana Kulagina and Jonathan Will and Henning Meyerhenke and Odej Kao},
doi = {10.1109/BigData52589.2021.9671519},
year = {2021},
date = {2021-01-01},
urldate = {2021-01-01},
booktitle = {2021 IEEE International Conference on Big Data (Big Data)},
pages = {65-75},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
2020
Jonathan Will; Jonathan Bader; Lauritz Thamsen
Towards Collaborative Optimization of Cluster Configurations for Distributed Dataflow Jobs Inproceedings
In: 2020 IEEE International Conference on Big Data (Big Data), pp. 2851-2856, 2020.
@inproceedings{9377994,
title = {Towards Collaborative Optimization of Cluster Configurations for Distributed Dataflow Jobs},
author = {Jonathan Will and Jonathan Bader and Lauritz Thamsen},
doi = {10.1109/BigData50022.2020.9377994},
year = {2020},
date = {2020-01-01},
urldate = {2020-01-01},
booktitle = {2020 IEEE International Conference on Big Data (Big Data)},
pages = {2851-2856},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
0000
Marcus Hilbrich; Sebastian Müller; Svetlana Kulagina; Christopher Lazik; Ninon De Mecquenem.; Lars Grunske
A consolidated View on Specification Languages for Data Analysis Workflows Proceeding Forthcoming
Automated Software Re-Engineering (ISoLA2022 · ASRE) (accepted), Forthcoming.
@proceedings{nokey,
title = {A consolidated View on Specification Languages for Data Analysis Workflows},
author = {Marcus Hilbrich and Sebastian Müller and Svetlana Kulagina and Christopher Lazik and Ninon De Mecquenem. and Lars Grunske},
howpublished = {Automated Software Re-Engineering (ISoLA2022 · ASRE) (accepted)},
keywords = {},
pubstate = {forthcoming},
tppubtype = {proceedings}
}
