

@preamble{{\providecommand{\MaxMinAntSystem}{{$\cal MAX$--$\cal MIN$} {Ant} {System}} } # {\providecommand{\rpackage}[1]{{#1}} } # {\providecommand{\softwarepackage}[1]{{#1}} } # {\providecommand{\proglang}[1]{{#1}} }}

@string{and = { and }}

@string{lncs = {Lecture Notes in Computer Science}}

@string{add-berlin = { Berlin, Germany }}

@string{add-cham = { Cham, Switzerland }}

@string{add-heidelberg = { Heidelberg }}

@string{aaaip-pub = {{AAAI} Press}}

@string{ieeep = {IEEE Press}}

@string{ieeep-ad = {Piscataway, NJ}}

@string{springer = {Springer}}

@string{iridia = {IRIDIA, Universit{\'e} Libre de Bruxelles, Belgium}}

@string{proc_of = {Proceedings of }}

@string{proc_of_the = proc_of # { the }}

@string{aaai = proc_of_the # {{AAAI} Conference on Artificial Intelligence}}

@string{cec = { Congress on Evolutionary Computation (CEC}}

@string{cec06 = proc_of_the # {2006} # cec # { 2006)}}

@string{icml = { International Conference on Machine Learning, {ICML} }}

@string{icml2014 = proc_of_the # {31st} # icml # {2014}}

@string{acm-cs = {{ACM} Computing Surveys}}

@string{jair = {Journal of Artificial Intelligence Research}}

@string{joh = {Journal of Heuristics}}

@string{orp = {Operations Research Perspectives}}

@string{balaprakash = {  Prasanna Balaprakash }}

@string{bartz-beielstein = { Thomas Bartz-Beielstein }}

@string{battiti = { Roberto Battiti }}

@string{biedenkapp = { Biedenkapp, Andr{\'e} }}

@string{birattari = { Mauro Birattari }}

@string{chiarandini = { Marco Chiarandini }}

@string{desouza = { Marcelo {De Souza} }}

@string{dubois-lacoste = { J{\'e}r{\'e}mie Dubois-Lacoste }}

@string{fawcett = { Chris Fawcett }}

@string{fonseca = { Carlos M. Fonseca }}

@string{hamadi = { Youssef Hamadi }}

@string{hoos = { Holger H. Hoos }}

@string{hutter = { Frank Hutter }}

@string{leyton-brown = { Kevin Leyton-Brown }}

@string{lindauer_m = { Marius Thomas Lindauer }}

@string{lopez-ibanez = { Manuel L{\'o}pez-Ib{\'a}{\~n}ez }}

@string{mcgeoch_cc = { Catherine C. McGeoch }}

@string{paquete = { Lu{\'i}s Paquete }}

@string{pardalos = { Panos M. Pardalos }}

@string{perez_l = {  P{\'e}rez C{\'a}ceres, Leslie}}

@string{preuss_m = { Mike Preuss }}

@string{ritt = { Marcus Ritt}}

@string{schneider_m = { Marius Schneider }}

@string{schoenauer = { Marc Schoenauer }}

@string{stuetzle = { Thomas St{\"u}tzle }}

@string{yuan_z = { Zhi Yuan }}

@inproceedings{BieLinEggFraFawHoo2017,
  author = biedenkapp # and # lindauer_m # and # {Eggensperger,
                  Katharina} # and # hutter # and # fawcett # and # hoos,
  title = {Efficient Parameter Importance Analysis via Ablation with
                  Surrogates},
  crossref = {AAAI2017},
  url = {https://aaai.org/ocs/index.php/AAAI/AAAI17/paper/view/14750}
}

@incollection{BirYuaBal2010:emaoa,
  author = birattari # and # yuan_z # and # balaprakash # and # stuetzle,
  title = {{F}-Race and Iterated {F}-Race: An Overview},
  pages = {311--336},
  crossref = {BarChiPaqPre2010emaoa},
  keywords = {F-race, iterated F-race, irace, tuning},
  doi = {10.1007/978-3-642-02538-9_13}
}

@article{DesRitLopPer2021acviz,
  author = desouza # and # ritt # and # lopez-ibanez # and # perez_l,
  title = {{\softwarepackage{ACVIZ}}: A Tool for the Visual Analysis of
                  the Configuration of Algorithms with {\rpackage{irace}}},
  journal = orp,
  year = 2021,
  doi = {10.1016/j.orp.2021.100186},
  supplement = {https://zenodo.org/record/4714582},
  abstract = {This paper introduces acviz, a tool that helps to analyze the
                  automatic configuration of algorithms with irace. It provides
                  a visual representation of the configuration process,
                  allowing users to extract useful information, e.g. how the
                  configurations evolve over time. When test data is available,
                  acviz also shows the performance of each configuration on the
                  test instances. Using this visualization, users can analyze
                  and compare the quality of the resulting configurations and
                  observe the performance differences on training and test
                  instances.},
  volume = 8,
  pages = 100186
}

@article{FawHoos2015ablation,
  title = {Analysing Differences Between Algorithm Configurations
                  through Ablation},
  author = fawcett # and # hoos,
  journal = joh,
  pages = {431--458},
  volume = 22,
  number = 4,
  year = 2016
}

@incollection{FonPaqLop06:hypervolume,
  author = fonseca # and # paquete # and # lopez-ibanez,
  title = {An improved dimension-\hspace{0pt}sweep
                  algorithm for the hypervolume indicator},
  crossref = {CEC2006},
  pages = {1157--1163},
  doi = {10.1109/CEC.2006.1688440},
  pdf = {FonPaqLop06-hypervolume.pdf},
  abstract = {This paper presents a recursive, dimension-sweep
                  algorithm for computing the hypervolume indicator of
                  the quality of a set of $n$ non-dominated points in
                  $d>2$ dimensions. It improves upon the existing HSO
                  (Hypervolume by Slicing Objectives) algorithm by
                  pruning the recursion tree to avoid repeated
                  dominance checks and the recalculation of partial
                  hypervolumes. Additionally, it incorporates a recent
                  result for the three-dimensional special case.  The
                  proposed algorithm achieves $O(n^{d-2} \log n)$ time
                  and linear space complexity in the worst-case, but
                  experimental results show that the pruning
                  techniques used may reduce the time complexity
                  exponent even further.}
}

@incollection{HutHooLey2013lion,
  author = hutter # and # hoos # and # leyton-brown,
  title = {Identifying Key Algorithm Parameters and Instance Features
                  using Forward Selection},
  crossref = {LION2013},
  pages = {364--381},
  doi = {10.1007/978-3-642-44973-4_40},
  keywords = {parameter importance}
}

@inproceedings{HutHooLey2014icml,
  author = hutter # and # hoos # and # leyton-brown,
  title = {An Efficient Approach for Assessing Hyperparameter
                  Importance},
  crossref = {ICML2014},
  pages = {754--762},
  url = {http://jmlr.org/proceedings/papers/v32/hutter14.html},
  keywords = {fANOVA, parameter importance}
}

@article{HutHooLeyStu2009jair,
  author = hutter # and # hoos # and # leyton-brown # and # stuetzle,
  title = {{\softwarepackage{ParamILS}:} An Automatic Algorithm Configuration Framework},
  journal = jair,
  year = 2009,
  volume = 36,
  pages = {267--306},
  month = oct,
  doi = {10.1613/jair.2861}
}

@techreport{IRIDIA-2004-001,
  author = birattari,
  title = {On the Estimation of the Expected Performance of a Metaheuristic on a Class of Instances. How Many Instances, How Many Runs?},
  institution = iridia,
  year = 2004,
  number = {TR/IRIDIA/2004-001}
}

@article{LopDubPerStuBir2016irace,
  author = lopez-ibanez # and # dubois-lacoste # and # perez_l # and # stuetzle # and # birattari,
  title = {The {\rpackage{irace}} Package: Iterated Racing for Automatic
                  Algorithm Configuration},
  journal = orp,
  year = 2016,
  supplement = {http://iridia.ulb.ac.be/supp/IridiaSupp2016-003/},
  doi = {10.1016/j.orp.2016.09.002},
  volume = 3,
  pages = {43--58}
}

@techreport{LopDubStu2011irace,
  author = lopez-ibanez # and # dubois-lacoste # and # stuetzle # and # birattari,
  title = {The {\rpackage{irace}} package, Iterated Race for Automatic
                  Algorithm Configuration},
  institution = iridia,
  year = 2011,
  number = {TR/IRIDIA/2011-004},
  url = {http://iridia.ulb.ac.be/IridiaTrSeries/IridiaTr2011-004.pdf},
  note = {Published in } # orp # {~\cite{LopDubPerStuBir2016irace}}
}

@article{McG1992vrt,
  author = mcgeoch_cc,
  title = {Analyzing Algorithms by Simulation: Variance Reduction
                  Techniques and Simulation Speedups},
  abstract = {Although experimental studies have been widely applied to the
                  investigation of algorithm performance, very little attention
                  has been given to experimental method in this area. This is
                  unfortunate, since much can be done to improve the quality of
                  the data obtained; often, much improvement may be needed for
                  the data to be useful. This paper gives a tutorial discussion
                  of two aspects of good experimental technique: the use of
                  variance reduction techniques and simulation speedups in
                  algorithm studies.  In an illustrative study, application of
                  variance reduction techniques produces a decrease in variance
                  by a factor 1000 in one case, giving a dramatic improvement
                  in the precision of experimental results. Furthermore, the
                  complexity of the simulation program is improved from
                  $\Theta(m n/H_n)$ to $\Theta(m + n \log n)$ (where $m$ is
                  typically much larger than $n$), giving a much faster
                  simulation program and therefore more data per unit of
                  computation time. The general application of variance
                  reduction techniques is also discussed for a variety of
                  algorithm problem domains.},
  volume = 24,
  doi = {10.1145/130844.130853},
  number = 2,
  journal = acm-cs,
  year = 1992,
  keywords = {experimental analysis of algorithms, move-to-front rule,
                  self-organizing sequential search, statistical analysis of
                  algorithms, transpose rule, variance reduction techniques},
  pages = {195--212}
}

@incollection{PerLopHooStu2017:lion,
  author = perez_l # and # lopez-ibanez # and # hoos # and # stuetzle,
  title = {An Experimental Study of Adaptive Capping in {\rpackage{irace}}},
  crossref = {LION2017},
  pages = {235--250},
  pdf = {PerLopHooStu2017lion.pdf},
  doi = {10.1007/978-3-319-69404-7_17},
  supplement = {http://iridia.ulb.ac.be/supp/IridiaSupp2016-007/}
}

@incollection{SchHoo2012quanti,
  title = {Quantifying Homogeneity of Instance Sets for
                  Algorithm Configuration},
  crossref = {LION2012},
  keywords = {Quantifying Homogeneity; Empirical Analysis;
                  Parameter Optimization; Algorithm Configuration},
  author = schneider_m # and # hoos,
  pages = {190--204},
  doi = {10.1007/978-3-642-34413-8_14}
}

@book{AAAI2017,
  booktitle = aaai,
  editor = {Satinder P. Singh and Shaul Markovitch},
  title = {Proceedings of the Thirty-First {AAAI} Conference on
                  Artificial Intelligence, February 4-9, 2017, San Francisco,
                  California, {USA}},
  year = 2017,
  month = feb,
  publisher = aaaip-pub
}

@book{BarChiPaqPre2010emaoa,
  title = {Experimental Methods for the Analysis of
                  Optimization Algorithms},
  booktitle = {Experimental Methods for the Analysis of
                  Optimization Algorithms},
  publisher = springer,
  address = add-berlin,
  year = 2010,
  editor = bartz-beielstein # and # chiarandini # and # paquete # and # preuss_m
}

@proceedings{CEC2006,
  key = {IEEE CEC},
  title = {Proceedings of the 2006 Congress on Evolutionary
                  Computation (CEC 2006)},
  booktitle = cec06,
  year = 2006,
  month = jul,
  publisher = ieeep,
  address = ieeep-ad
}

@proceedings{ICML2014,
  editor = {Xing, Eric P. and Jebara, Tony},
  title = {Proceedings of the 31st International Conference on Machine
                  Learning, {ICML} 2014, Beijing, China, 21-26 June 2014},
  booktitle = icml2014,
  volume = 32,
  year = 2014,
  url = {http://jmlr.org/proceedings/papers/v32/}
}

@book{LION2012,
  title = {6th International Conference, LION 6, Paris, France,
                  January 16-20, 2012. Selected Papers},
  booktitle = {Learning and Intelligent Optimization, 6th
                  International Conference, LION 6},
  year = 2012,
  series = lncs,
  publisher = springer,
  address = add-heidelberg,
  editor = hamadi # and # schoenauer,
  volume = 7219
}

@book{LION2013,
  title = {7th International Conference, LION 7, Catania,
                  Italy, January 7-11, 2013. Selected Papers},
  booktitle = {Learning and Intelligent Optimization, 7th
                  International Conference, LION 7},
  year = 2013,
  series = lncs,
  editor = pardalos # and # {G. Nicosia},
  volume = 7997,
  publisher = springer,
  address = add-heidelberg
}

@book{LION2017,
  title = {11th International Conference, LION 11, Nizhny Novgorod,
                  Russia, June 19-21, 2017, Revised Selected Papers},
  booktitle = {Learning and Intelligent Optimization, 11th International
                  Conference, LION 11},
  year = 2017,
  editor = battiti # and # {Dmitri E. Kvasov} # and # {Yaroslav D. Sergeyev},
  volume = 10556,
  series = lncs,
  publisher = springer,
  address = add-cham
}

