<?xml version="1.0" encoding="UTF-8"?>
 <rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://web.resource.org/cc/" xmlns:syn="http://purl.org/rss/1.0/modules/syndication/" xmlns:admin="http://webns.net/mvcb/">
  <channel rdf:about="http://pinboard.in">
    <title>Pinboard (cshalizi)</title>
    <link>https://pinboard.in/u:cshalizi/public/</link>
    <description>recent bookmarks from cshalizi</description>
    <items>
      <rdf:Seq>	<rdf:li rdf:resource="https://arxiv.org/abs/2408.02060"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2309.03742"/>
	<rdf:li rdf:resource="https://doi.org/10.1080/01621459.2023.2197686"/>
	<rdf:li rdf:resource="https://www.tandfonline.com/doi/full/10.1080/01621459.2021.1987920"/>
	<rdf:li rdf:resource="https://direct.mit.edu/rest/article-abstract/101/5/743/58556/Choosing-among-Regularized-Estimators-in-Empirical?redirectedFrom=fulltext"/>
	<rdf:li rdf:resource="https://onlinelibrary.wiley.com/doi/abs/10.1002/sta4.426"/>
	<rdf:li rdf:resource="https://projecteuclid.org/journals/annals-of-statistics/volume-49/issue-3/On-cross-validated-Lasso-in-high-dimensions/10.1214/20-AOS2000.short"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2105.04134"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2104.10471"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2104.04716"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2007.12671"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2104.00673"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2103.01356"/>
	<rdf:li rdf:resource="https://jmlr.org/papers/v21/19-408.html"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2006.12669"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1912.06686"/>
	<rdf:li rdf:resource="https://www.tandfonline.com/doi/full/10.1080/01621459.2019.1668794"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1905.13657"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1910.08904"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1909.06539"/>
	<rdf:li rdf:resource="https://www.tandfonline.com/doi/full/10.1080/10618600.2019.1647846"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1909.04890"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1909.05495"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1909.05299"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1908.08741"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1908.05873"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1605.02214"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1908.00882"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1907.13413"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1908.00325"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1907.12116"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1907.01670"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1905.12580"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1811.00645"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1905.11744"/>
	<rdf:li rdf:resource="https://projecteuclid.org/euclid.aos/1558512018"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1904.02438"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1902.06281"/>
	<rdf:li rdf:resource="https://web.stanford.edu/~wpmarble/docs/MarbleTyler_SurveyIdealPointsAPSA.pdf"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1705.07349"/>
	<rdf:li rdf:resource="http://www.mitpressjournals.org/doi/abs/10.1162/NECO_a_00923"/>
	<rdf:li rdf:resource="http://papers.nips.cc/paper/3294-modeling-homophily-and-stochastic-equivalence-in-symmetric-relational-data"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1502.04585"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1504.01132"/>
	<rdf:li rdf:resource="http://arstechnica.co.uk/security/2016/02/the-nsas-skynet-program-may-be-killing-thousands-of-innocent-people/"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1602.01522"/>
	<rdf:li rdf:resource="http://www.tandfonline.com/doi/abs/10.1080/01621459.1979.10481632"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1507.00066"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1503.03515"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1411.1469"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1405.6974"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1411.1715"/>
	<rdf:li rdf:resource="http://link.springer.com/article/10.1007/s10115-014-0789-0"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1403.4544"/>
	<rdf:li rdf:resource="http://projecteuclid.org/euclid.ejs/1392215030"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1309.2068"/>
	<rdf:li rdf:resource="http://www.jstor.org/stable/2984809"/>
	<rdf:li rdf:resource="http://biostats.bepress.com/ucbbiostat/paper313/"/>
	<rdf:li rdf:resource="http://jpr.sagepub.com/content/47/4/363"/>
	<rdf:li rdf:resource="http://biostats.bepress.com/uncbiostat/art35/"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1303.3128"/>
	<rdf:li rdf:resource="http://www.russpoldrack.org/2012/12/the-perils-of-leave-one-out.html"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1212.1780"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1210.6187"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1210.5830"/>
	<rdf:li rdf:resource="http://projecteuclid.org/DPubS?service=UI&amp;version=1.0&amp;verb=Display&amp;handle=euclid.ejs/1349355603"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1206.6128"/>
	<rdf:li rdf:resource="http://gsp.tamu.edu/Publications/journal-publications/performance-analysis-and-error-estimation"/>
	<rdf:li rdf:resource="http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=6172585"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1206.2248"/>
      </rdf:Seq>
    </items>
  </channel><item rdf:about="https://arxiv.org/abs/2408.02060">
    <title>[2408.02060] Winners with Confidence: Discrete Argmin Inference with an Application to Model Selection</title>
    <dc:date>2025-04-28T01:21:57+00:00</dc:date>
    <link>https://arxiv.org/abs/2408.02060</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We study the problem of finding the index of the minimum value of a vector from noisy observations. This problem is relevant in population/policy comparison, discrete maximum likelihood, and model selection. We develop an asymptotically normal test statistic, even in high-dimensional settings and with potentially many ties in the population mean vector, by integrating concepts and tools from cross-validation and differential privacy. The key technical ingredient is a central limit theorem for globally dependent data. We also propose practical ways to select the tuning parameter that adapts to the signal landscape. Numerical experiments and data examples demonstrate the ability of the proposed method to achieve a favorable bias-variance trade-off in practical scenarios."]]></description>
<dc:subject>to:NB model_selection cross-validation lei.jing</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:656aa8135923/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lei.jing"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2309.03742">
    <title>[2309.03742] Efficient estimation and correction of selection-induced bias with order statistics</title>
    <dc:date>2023-09-21T16:54:53+00:00</dc:date>
    <link>https://arxiv.org/abs/2309.03742</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Model selection aims to identify a sufficiently well performing model that is possibly simpler than the most complex model among a pool of candidates. However, the decision-making process itself can inadvertently introduce non-negligible bias when the cross-validation estimates of predictive performance are marred by excessive noise. In finite data regimes, cross-validated estimates can encourage the statistician to select one model over another when it is not actually better for future data. While this bias remains negligible in the case of few models, when the pool of candidates grows, and model selection decisions are compounded (as in forward search), the expected magnitude of selection-induced bias is likely to grow too. This paper introduces an efficient approach to estimate and correct selection-induced bias based on order statistics. Numerical experiments demonstrate the reliability of our approach in estimating both selection-induced bias and over-fitting along compounded model selection decisions, with specific application to forward search. This work represents a light-weight alternative to more computationally expensive approaches to correcting selection-induced bias, such as nested cross-validation and the bootstrap. Our approach rests on several theoretic assumptions, and we provide a diagnostic to help understand when these may not be valid and when to fall back on safer, albeit more computationally expensive approaches. The accompanying code facilitates its practical implementation and fosters further exploration in this area."]]></description>
<dc:subject>to:NB cross-validation model_selection</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:c59c4b2c1dbf/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://doi.org/10.1080/01621459.2023.2197686">
    <title>Cross-Validation: What Does It Estimate and How Well Does It Do It?</title>
    <dc:date>2023-06-08T15:37:53+00:00</dc:date>
    <link>https://doi.org/10.1080/01621459.2023.2197686</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Cross-validation is a widely used technique to estimate prediction error, but its behavior is complex and not fully understood. Ideally, one would like to think that cross-validation estimates the prediction error for the model at hand, fit to the training data. We prove that this is not the case for the linear model fit by ordinary least squares; rather it estimates the average prediction error of models fit on other unseen training sets drawn from the same population. We further show that this phenomenon occurs for most popular estimates of prediction error, including data splitting, bootstrapping, and Mallow’s $C_p$. Next, the standard confidence intervals for prediction error derived from cross-validation may have coverage far below the desired level. Because each data point is used for both training and testing, there are correlations among the measured accuracies for each fold, and so the usual estimate of variance is too small. We introduce a nested cross-validation scheme to estimate this variance more accurately, and show empirically that this modification leads to intervals with approximately correct coverage in many examples where traditional cross-validation intervals fail. Lastly, our analysis also shows that when producing confidence intervals for prediction accuracy with simple data splitting, one should not refit the model on the combined data, since this invalidates the confidence intervals."

--- (Last part is obvious, no?)]]></description>
<dc:subject>cross-validation statistics tibshirani.robert hastie.trevor to_read to_teach in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:92e07da3433c/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:tibshirani.robert"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hastie.trevor"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://www.tandfonline.com/doi/full/10.1080/01621459.2021.1987920">
    <title>Optimal Simulator Selection: Journal of the American Statistical Association: Vol 0, No 0</title>
    <dc:date>2022-06-11T04:51:08+00:00</dc:date>
    <link>https://www.tandfonline.com/doi/full/10.1080/01621459.2021.1987920</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Computer simulators are widely used for the study of complex systems. In many applications, there are multiple simulators available with different scientific interpretations of the underlying mechanism, and the goal is to identify an optimal simulator based on the observed physical experiments. To achieve the goal, we propose a selection criterion based on leave-one-out cross-validation. This criterion consists of a goodness-of-fit measure and a generalized degrees of freedom penalizing the simulator sensitivity to perturbations in the physical observations. Asymptotic properties of the selected optimal simulator are discussed. It is shown that the proposed procedure includes a conventional calibration method as a special case. The finite sample performance of the proposed procedure is demonstrated through numerical examples. In the application of cell biology, an optimal simulator is selected, which can shed light on the T cell recognition mechanism in the human immune system. Supplementary materials for this article are available online."]]></description>
<dc:subject>cross-validation model_selection simulation-based_inference re:codename:catherine_wheel in_NB to_read</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:e2e4ed6ce1cc/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:simulation-based_inference"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:codename:catherine_wheel"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://direct.mit.edu/rest/article-abstract/101/5/743/58556/Choosing-among-Regularized-Estimators-in-Empirical?redirectedFrom=fulltext">
    <title>Choosing among Regularized Estimators in Empirical Economics: The Risk of Machine Learning | The Review of Economics and Statistics | MIT Press</title>
    <dc:date>2022-03-14T18:19:21+00:00</dc:date>
    <link>https://direct.mit.edu/rest/article-abstract/101/5/743/58556/Choosing-among-Regularized-Estimators-in-Empirical?redirectedFrom=fulltext</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Many settings in empirical economics involve estimation of a large number of parameters. In such settings, methods that combine regularized estimation and data-driven choices of regularization parameters are useful. We provide guidance to applied researchers on the choice between regularized estimators and data-driven selection of regularization parameters. We characterize the risk and relative performance of regularized estimators as a function of the data-generating process and show that data-driven choices of regularization parameters yield estimators with risk uniformly close to the risk attained under the optimal (unfeasible) choice of regularization parameters. We illustrate using examples from empirical economics."]]></description>
<dc:subject>to:NB econometrics estimation statistics learning_theory to_teach:childs_garden_of_statistical_learning_theory re:HEAS downloaded cross-validation</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:765356ad4562/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:econometrics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:estimation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:learning_theory"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:childs_garden_of_statistical_learning_theory"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:HEAS"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:downloaded"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://onlinelibrary.wiley.com/doi/abs/10.1002/sta4.426">
    <title>Consistent Estimation of Number of Communities in Stochastic Block Models using Cross‐Validation - Qin - - Stat - Wiley Online Library</title>
    <dc:date>2021-10-11T19:50:46+00:00</dc:date>
    <link>https://onlinelibrary.wiley.com/doi/abs/10.1002/sta4.426</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Stochastic block model (SBM) and its variants constitute an important family of probabilistic tools for studying network data. There is a rich literature on methods for estimating the block labels and model parameters of stochastic block models. Most of these studies would require the number of communities $K$ as an input, making the estimation of K an important problem. Cross-validation is a natural option for this problem since it is a widely used generic method for evaluating model fitting. However, cross-validation is known to be inconsistent and prone to over-fitting unless impractical split ratios are used. Cross-validation with confidence (CVC) is proposed with better theoretical guarantees in conventional settings. We study the properties of CVC for stochastic block models. Our theoretical studies show that CVC, unlike the standard cross-validation, can consistently pick the optimal K under suitable conditions. We implement this method and check its performance against other established methods on both synthetic and real datasets."

--- Pretty sure I bookmarked the preprint.]]></description>
<dc:subject>to:NB stochastic_block_models community_discovery cross-validation network_data_analysis kith_and_kin lei.jing heard_the_talk</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:23e1965abcdf/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:stochastic_block_models"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:community_discovery"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:network_data_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:kith_and_kin"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lei.jing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:heard_the_talk"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://projecteuclid.org/journals/annals-of-statistics/volume-49/issue-3/On-cross-validated-Lasso-in-high-dimensions/10.1214/20-AOS2000.short">
    <title>On cross-validated Lasso in high dimensions</title>
    <dc:date>2021-08-09T16:26:22+00:00</dc:date>
    <link>https://projecteuclid.org/journals/annals-of-statistics/volume-49/issue-3/On-cross-validated-Lasso-in-high-dimensions/10.1214/20-AOS2000.short</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["In this paper, we derive nonasymptotic error bounds for the Lasso estimator when the penalty parameter for the estimator is chosen using K-fold cross-validation. Our bounds imply that the cross-validated Lasso estimator has nearly optimal rates of convergence in the prediction, L2 and L1 norms. For example, we show that in the model with the Gaussian noise and under fairly general assumptions on the candidate set of values of the penalty parameter, the estimation error of the cross-validated Lasso estimator converges to zero in the prediction norm with the √slogp/n×√log(pn) rate, where n is the sample size of available data, p is the number of covariates and s is the number of nonzero coefficients in the model. Thus, the cross-validated Lasso estimator achieves the fastest possible rate of convergence in the prediction norm up to a small logarithmic factor √log(pn), and similar conclusions apply for the convergence rate both in L2 and in L1 norms. Importantly, our results cover the case when p is (potentially much) larger than n and also allow for the case of non-Gaussian noise. Our paper therefore serves as a justification for the widely spread practice of using cross-validation as a method to choose the penalty parameter for the Lasso estimator."]]></description>
<dc:subject>to:NB cross-validation lasso sparsity high-dimensional_statistics statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:25ec0744b249/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lasso"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:sparsity"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:high-dimensional_statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2105.04134">
    <title>[2105.04134] Bagging cross-validated bandwidth selection in nonparametric regression estimation with applications to large-sized samples</title>
    <dc:date>2021-05-12T18:15:40+00:00</dc:date>
    <link>https://arxiv.org/abs/2105.04134</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Cross-validation is a well-known and widely used bandwidth selection method in nonparametric regression estimation. However, this technique has two remarkable drawbacks: (i) the large variability of the selected bandwidths, and (ii) the inability to provide results in a reasonable time for very large sample sizes. To overcome these problems, bagging cross-validation bandwidths are analyzed in this paper. This approach consists in computing the cross-validation bandwidths for a finite number of subsamples and then rescaling the averaged smoothing parameters to the original sample size. Under a random-design regression model, asymptotic expressions up to a second-order for the bias and variance of the leave-one-out cross-validation bandwidth for the Nadaraya--Watson estimator are obtained. Subsequently, the asymptotic bias and variance and the limit distribution are derived for the bagged cross-validation selector. Suitable choices of the number of subsamples and the subsample size lead to an n−1/2 rate for the convergence in distribution of the bagging cross-validation selector, outperforming the rate n−3/10 of leave-one-out cross-validation. Several simulations and an illustration on a real dataset related to the COVID-19 pandemic show the behavior of our proposal and its better performance, in terms of statistical efficiency and computing time, when compared to leave-one-out cross-validation."]]></description>
<dc:subject>to:NB cross-validation ensemble_methods bootstrap to_teach:data-mining kernel_smoothing</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:f2a3494ec6a3/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:bootstrap"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:kernel_smoothing"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2104.10471">
    <title>[2104.10471] On the Asymptotic Optimality of Cross-Validation based Hyper-parameter Estimators for Regularized Least Squares Regression Problems</title>
    <dc:date>2021-04-29T03:26:43+00:00</dc:date>
    <link>https://arxiv.org/abs/2104.10471</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["The asymptotic optimality (a.o.) of various hyper-parameter estimators with different optimality criteria has been studied in the literature for regularized least squares regression problems. The estimators include e.g., the maximum (marginal) likelihood method, Cp statistics, and generalized cross validation method, and the optimality criteria are based on e.g., the inefficiency, the expectation inefficiency and the risk. In this paper, we consider the regularized least squares regression problems with fixed number of regression parameters, choose the optimality criterion based on the risk, and study the a.o. of several cross validation (CV) based hyper-parameter estimators including the leave k-out CV method, generalized CV method, r-fold CV method and hold out CV method. We find the former three methods can be a.o. under mild assumptions, but not the last one, and we use Monte Carlo simulations to illustrate the efficacy of our findings."]]></description>
<dc:subject>to:NB model_selection regression cross-validation</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:0ff3608d5a6d/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2104.04716">
    <title>[2104.04716] Analytic and Bootstrap-after-Cross-Validation Methods for Selecting Penalty Parameters of High-Dimensional M-Estimators</title>
    <dc:date>2021-04-13T04:01:37+00:00</dc:date>
    <link>https://arxiv.org/abs/2104.04716</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We develop two new methods for selecting the penalty parameter for the ℓ1-penalized high-dimensional M-estimator, which we refer to as the analytic and bootstrap-after-cross-validation methods. For both methods, we derive nonasymptotic error bounds for the corresponding ℓ1-penalized M-estimator and show that the bounds converge to zero under mild conditions, thus providing a theoretical justification for these methods. We demonstrate via simulations that the finite-sample performance of our methods is much better than that of previously available and theoretically justified methods."]]></description>
<dc:subject>to:NB lasso bootstrap cross-validation model_selection</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:1598c1a5531e/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lasso"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:bootstrap"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2007.12671">
    <title>[2007.12671] Cross-validation Confidence Intervals for Test Error</title>
    <dc:date>2021-04-02T23:56:49+00:00</dc:date>
    <link>https://arxiv.org/abs/2007.12671</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This work develops central limit theorems for cross-validation and consistent estimators of its asymptotic variance under weak stability conditions on the learning algorithm. Together, these results provide practical, asymptotically-exact confidence intervals for k-fold test error and valid, powerful hypothesis tests of whether one learning algorithm has smaller k-fold test error than another. These results are also the first of their kind for the popular choice of leave-one-out cross-validation. In our real-data experiments with diverse learning algorithms, the resulting intervals and tests outperform the most popular alternative methods from the literature."

--- "Pierre Bayle"?!? [https://plato.stanford.edu/entries/bayle/]]]></description>
<dc:subject>to:NB cross-validation statistics mackey.lester</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:761b6895974d/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:mackey.lester"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2104.00673">
    <title>[2104.00673] Cross-validation: what does it estimate and how well does it do it?</title>
    <dc:date>2021-04-02T23:55:01+00:00</dc:date>
    <link>https://arxiv.org/abs/2104.00673</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Cross-validation is a widely-used technique to estimate prediction error, but its behavior is complex and not fully understood. Ideally, one would like to think that cross-validation estimates the prediction error for the model at hand, fit to the training data. We prove that this is not the case for the linear model fit by ordinary least squares; rather it estimates the average prediction error of models fit on other unseen training sets drawn from the same population. We further show that this phenomenon occurs for most popular estimates of prediction error, including data splitting, bootstrapping, and Mallow's Cp. Next, the standard confidence intervals for prediction error derived from cross-validation may have coverage far below the desired level. Because each data point is used for both training and testing, there are correlations among the measured accuracies for each fold, and so the usual estimate of variance is too small. We introduce a nested cross-validation scheme to estimate this variance more accurately, and show empirically that this modification leads to intervals with approximately correct coverage in many examples where traditional cross-validation intervals fail. Lastly, our analysis also shows that when producing confidence intervals for prediction accuracy with simple data splitting, one should not re-fit the model on the combined data, since this invalidates the confidence intervals."]]></description>
<dc:subject>to:NB cross-validation statistics tibshirani.robert hastie.trevor</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:2b9468b20416/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:tibshirani.robert"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hastie.trevor"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2103.01356">
    <title>[2103.01356] Statistical learning and cross-validation for point processes</title>
    <dc:date>2021-03-03T04:34:11+00:00</dc:date>
    <link>https://arxiv.org/abs/2103.01356</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This paper presents the first general (supervised) statistical learning framework for point processes in general spaces. Our approach is based on the combination of two new concepts, which we define in the paper: i) bivariate innovations, which are measures of discrepancy/prediction-accuracy between two point processes, and ii) point process cross-validation (CV), which we here define through point process thinning. The general idea is to carry out the fitting by predicting CV-generated validation sets using the corresponding training sets; the prediction error, which we minimise, is measured by means of bivariate innovations. Having established various theoretical properties of our bivariate innovations, we study in detail the case where the CV procedure is obtained through independent thinning and we apply our statistical learning methodology to three typical spatial statistical settings, namely parametric intensity estimation, non-parametric intensity estimation and Papangelou conditional intensity fitting. Aside from deriving theoretical properties related to these cases, in each of them we numerically show that our statistical learning approach outperforms the state of the art in terms of mean (integrated) squared error."]]></description>
<dc:subject>to:NB cross-validation point_processes</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:479c6df000d8/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:point_processes"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://jmlr.org/papers/v21/19-408.html">
    <title>Stable Regression: On the Power of Optimization over Randomization</title>
    <dc:date>2020-12-21T04:34:21+00:00</dc:date>
    <link>https://jmlr.org/papers/v21/19-408.html</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We investigate and ultimately suggest remediation to the widely held belief that the best way to train regression models is via random assignment of our data to training and validation sets. In particular, we show that taking a robust optimization approach, and optimally selecting such training and validation sets, leads to models that not only perform significantly better than their randomly constructed counterparts in terms of prediction error, but more importantly, are considerably more stable in the sense that the standard deviation of the resulting predictions, as well as of the model coefficients, is greatly reduced. Moreover, we show that this optimization approach to training is far more effective at recovering the true support of a given data set, i.e., correctly identifying important features while simultaneously excluding spurious ones. We further compare the robust optimization approach to cross validation and find that optimization continues to have a performance edge albeit smaller. Finally, we show that this optimization approach to training is equivalent to building models that are robust to all subpopulations in the data, and thus in particular are robust to the hardest subpopulation, which leads to interesting domain specific interpretations through the use of optimal classification trees. The proposed robust optimization algorithm is efficient and scales training to essentially any desired size."

]]></description>
<dc:subject>to:NB color_me_skeptical statistics regression cross-validation</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:3238ba3eba5c/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:color_me_skeptical"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2006.12669">
    <title>[2006.12669] Approximate Cross-Validation for Structured Models</title>
    <dc:date>2020-12-02T15:24:55+00:00</dc:date>
    <link>https://arxiv.org/abs/2006.12669</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Many modern data analyses benefit from explicitly modeling dependence structure in data -- such as measurements across time or space, ordered words in a sentence, or genes in a genome. A gold standard evaluation technique is structured cross-validation (CV), which leaves out some data subset (such as data within a time interval or data in a geographic region) in each fold. But CV here can be prohibitively slow due to the need to re-run already-expensive learning algorithms many times. Previous work has shown approximate cross-validation (ACV) methods provide a fast and provably accurate alternative in the setting of empirical risk minimization. But this existing ACV work is restricted to simpler models by the assumptions that (i) data across CV folds are independent and (ii) an exact initial model fit is available. In structured data analyses, both these assumptions are often untrue. In the present work, we address (i) by extending ACV to CV schemes with dependence structure between the folds. To address (ii), we verify -- both theoretically and empirically -- that ACV quality deteriorates smoothly with noise in the initial fit. We demonstrate the accuracy and computational benefits of our proposed methods on a diverse set of real-world applications."]]></description>
<dc:subject>to:NB cross-validation to_read</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:955fea15756f/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1912.06686">
    <title>[1912.06686] Systematic Overestimation of Machine Learning Performance in Neuroimaging Studies of Depression</title>
    <dc:date>2020-11-27T05:58:25+00:00</dc:date>
    <link>https://arxiv.org/abs/1912.06686</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We currently observe a disconcerting phenomenon in machine learning studies in psychiatry: While we would expect larger samples to yield better results due to the availability of more data, larger machine learning studies consistently show much weaker performance than the numerous small-scale studies. Here, we systematically investigated this effect focusing on one of the most heavily studied questions in the field, namely the classification of patients suffering from Major Depressive Disorder (MDD) and healthy controls. Drawing upon a balanced sample of N=1,868 MDD patients and healthy controls from our recent international Predictive Analytics Competition (PAC), we first trained and tested a classification model on the full dataset which yielded an accuracy of 61%. Next, we mimicked the process by which researchers would draw samples of various sizes (N=4 to N=150) from the population and showed a strong risk of overestimation. Specifically, for small sample sizes (N=20), we observe accuracies of up to 95%. For medium sample sizes (N=100) accuracies up to 75% were found. Importantly, further investigation showed that sufficiently large test sets effectively protect against performance overestimation whereas larger datasets per se do not. While these results question the validity of a substantial part of the current literature, we outline the relatively low-cost remedy of larger test sets."]]></description>
<dc:subject>to:NB depression prediction cross-validation to_teach:childs_garden_of_statistical_learning_theory</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:c005dae639c6/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:depression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:childs_garden_of_statistical_learning_theory"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://www.tandfonline.com/doi/full/10.1080/01621459.2019.1668794">
    <title>Improved Small-Sample Estimation of Nonlinear Cross-Validated Prediction Metrics: Journal of the American Statistical Association: Vol 0, No 0</title>
    <dc:date>2019-10-22T13:30:52+00:00</dc:date>
    <link>https://www.tandfonline.com/doi/full/10.1080/01621459.2019.1668794</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["When predicting an outcome is the scientific goal, one must decide on a metric by which to evaluate the quality of predictions. We consider the problem of measuring the performance of a prediction algorithm with the same data that were used to train the algorithm. Typical approaches involve bootstrapping or cross-validation. However, we demonstrate that bootstrap-based approaches often fail and standard cross-validation estimators may perform poorly. We provide a general study of cross-validation-based estimators that highlights the source of this poor performance, and propose an alternative framework for estimation using techniques from the efficiency theory literature. We provide a theorem establishing the weak convergence of our estimators. The general theorem is applied in detail to two specific examples and we discuss possible extensions to other parameters of interest. For the two explicit examples that we consider, our estimators demonstrate remarkable finite-sample improvements over standard approaches."]]></description>
<dc:subject>to:NB cross-validation statistics prediction van_der_laan.mark</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:1cae5c409ffc/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:van_der_laan.mark"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1905.13657">
    <title>[1905.13657] Approximate Cross-Validation in High Dimensions with Guarantees</title>
    <dc:date>2019-10-22T13:19:50+00:00</dc:date>
    <link>https://arxiv.org/abs/1905.13657</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Leave-one-out cross validation (LOOCV) can be particularly accurate among CV variants for estimating out-of-sample error. Unfortunately, LOOCV requires re-fitting a model N times for a dataset of size N. To avoid this prohibitive computational expense, a number of authors have proposed approximations to LOOCV. These approximations work well when the unknown parameter is of small, fixed dimension but suffer in high dimensions; they incur a running time roughly cubic in the dimension, and, in fact, we show their accuracy significantly deteriorates in high dimensions. We demonstrate that these difficulties can be surmounted in ℓ1-regularized generalized linear models when we assume that the unknown parameter, while high dimensional, has a small support. In particular, we show that, under interpretable conditions, the support of the recovered parameter does not change as each datapoint is left out. This result implies that the previously proposed heuristic of only approximating CV along the support of the recovered parameter has running time and error that scale with the (small) support size even when the full dimension is large. Experiments on synthetic and real data support the accuracy of our approximations."]]></description>
<dc:subject>to:NB cross-validation sparsity high-dimensional_statistics computational_statistics statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:6d0b824228ec/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:sparsity"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:high-dimensional_statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:computational_statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1910.08904">
    <title>[1910.08904] $hv$-Block Cross Validation is not a BIBD: a Note on the Paper by Jeff Racine (2000)</title>
    <dc:date>2019-10-22T13:13:32+00:00</dc:date>
    <link>https://arxiv.org/abs/1910.08904</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This note corrects a mistake in the paper "consistent cross-validatory model-selection for dependent data: hv-block cross-validation" by Racine (2000). In his paper, he implied that the therein proposed hv-block cross-validation is consistent in the sense of Shao (1993). To get this intuition, he relied on the speculation that hv-block is a balanced incomplete block design (BIBD). This note demonstrates that this is not the case, and thus the theoretical consistency of hv-block remains an open question. In addition, I also provide a Python program counting the number of occurrences of each sample and each pair of samples."]]></description>
<dc:subject>to:NB cross-validation time_series statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:9f7ec608f103/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:time_series"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1909.06539">
    <title>[1909.06539] Not again! Data Leakage in Digital Pathology</title>
    <dc:date>2019-10-01T17:12:30+00:00</dc:date>
    <link>https://arxiv.org/abs/1909.06539</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Bioinformatics of high throughput omics data (e.g. microarrays and proteomics) has been plagued by uncountable issues with reproducibility at the start of the century. Concerns have motivated international initiatives such as the FDA's led MAQC Consortium, addressing reproducibility of predictive biomarkers by means of appropriate Data Analysis Plans (DAPs). For instance, repreated cross-validation is a standard procedure meant at mitigating the risk that information from held-out validation data may be used during model selection. We prove here that, many years later, Data Leakage can still be a non-negligible overfitting source in deep learning models for digital pathology. In particular, we evaluate the impact of (i) the presence of multiple images for each subject in histology collections; (ii) the systematic adoption of training over collection of subregions (i.e. "tiles" or "patches") extracted for the same subject. We verify that accuracy scores may be inflated up to 41%, even if a well-designed 10x5 iterated cross-validation DAP is applied, unless all images from the same subject are kept together either in the internal training or validation splits. Results are replicated for 4 classification tasks in digital pathology on 3 datasets, for a total of 373 subjects, and 543 total slides (around 27, 000 tiles). Impact of applying transfer learning strategies with models pre-trained on general-purpose or digital pathology datasets is also discussed."]]></description>
<dc:subject>to:NB cross-validation statistics bad_data_analysis to_teach:undergrad-ADA to_teach:data-mining</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:55f36f7dc31d/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:bad_data_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:undergrad-ADA"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://www.tandfonline.com/doi/full/10.1080/10618600.2019.1647846">
    <title>Estimating the Number of Clusters Using Cross-Validation: Journal of Computational and Graphical Statistics: Vol 0, No 0</title>
    <dc:date>2019-10-01T16:20:36+00:00</dc:date>
    <link>https://www.tandfonline.com/doi/full/10.1080/10618600.2019.1647846</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Many clustering methods, including k-means, require the user to specify the number of clusters as an input parameter. A variety of methods have been devised to choose the number of clusters automatically, but they often rely on strong modeling assumptions. This article proposes a data-driven approach to estimate the number of clusters based on a novel form of cross-validation. The proposed method differs from ordinary cross-validation, because clustering is fundamentally an unsupervised learning problem. Simulation and real data analysis results show that the proposed method outperforms existing methods, especially in high-dimensional settings with heterogeneous or heavy-tailed noise. In a yeast cell cycle dataset, the proposed method finds a parsimonious clustering with interpretable gene groupings. Supplementary materials for this article are available online.]]></description>
<dc:subject>to:NB cross-validation clustering k-means statistics perry.patrick_o. to_read to_teach:data-mining</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:da89de164589/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:clustering"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:k-means"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:perry.patrick_o."/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1909.04890">
    <title>[1909.04890] Aggregated Hold-Out</title>
    <dc:date>2019-09-15T14:44:49+00:00</dc:date>
    <link>https://arxiv.org/abs/1909.04890</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Aggregated hold-out (Agghoo) is a method which averages learning rules selected by hold-out (that is, cross-validation with a single split). We provide the first theoretical guarantees on Agghoo, ensuring that it can be used safely: Agghoo performs at worst like the hold-out when the risk is convex. The same holds true in classification with the 0-1 risk, with an additional constant factor. For the hold-out, oracle inequalities are known for bounded losses, as in binary classification. We show that similar results can be proved, under appropriate assumptions, for other risk-minimization problems. In particular, we obtain an oracle inequality for regularized kernel regression with a Lip-schitz loss, without requiring that the Y variable or the regressors be bounded. Numerical experiments show that aggregation brings a significant improvement over the hold-out and that Agghoo is competitive with cross-validation."]]></description>
<dc:subject>to:NB cross-validation model_selection statistics arlot.sylvain to_read</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:09021f6ec87d/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:arlot.sylvain"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1909.05495">
    <title>[1909.05495] Optimal choice of $k$ for $k$-nearest neighbor regression</title>
    <dc:date>2019-09-13T13:06:17+00:00</dc:date>
    <link>https://arxiv.org/abs/1909.05495</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["The k-nearest neighbor algorithm (k-NN) is a widely used non-parametric method for classification and regression. We study the mean squared error of the k-NN estimator when k is chosen by leave-one-out cross-validation (LOOCV). Although it was known that this choice of k is asymptotically consistent, it was not known previously that it is an optimal k. We show, with high probability, the mean squared error of this estimator is close to the minimum mean squared error using the k-NN estimate, where the minimum is over all choices of k."

--- Looks legit on first pass (and we know that LOOCV is generally _predictively_ good).]]></description>
<dc:subject>regression nearest_neighbors statistics cross-validation to_teach:data-mining have_skimmed in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:615c7919c631/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:nearest_neighbors"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_skimmed"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1909.05299">
    <title>[1909.05299] Counterfactual Cross-Validation: Effective Causal Model Selection from Observational Data</title>
    <dc:date>2019-09-13T12:45:35+00:00</dc:date>
    <link>https://arxiv.org/abs/1909.05299</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["What is the most effective way to select the best causal model among potential candidates? In this paper, we propose a method to effectively select the best individual-level treatment effect (ITE) predictors from a set of candidates using only an observational validation set. In model selection or hyperparameter tuning, we are interested in choosing the best model or the value of hyperparameter from potential candidates. Thus, we focus on accurately preserving the rank order of the ITE prediction performance of candidate causal models. The proposed evaluation metric is theoretically proved to preserve the true ranking of the model performance in expectation and to minimize the upper bound of the finite sample uncertainty in model selection. Consistent with the theoretical result, empirical experiments demonstrate that our proposed method is more likely to select the best model and set of hyperparameter in both model selection and hyperparameter tuning."

--- Their goal is a good one, but the fundamental issue is that we don't _have_ observations of individual-level causal effects to cross-validate against.  So they proxy that by a very standard doubly-robust estimator of said effects; at which point, why not just use that estimator?  In any case, I want to see comparisons to the Naive Statistician's approach of just cross-validating for outcomes (rather than differences in potential outcomes).]]></description>
<dc:subject>cross-validation causal_inference statistics model_selection have_skimmed in_NB my_initial_skeptical_coloration_became_on_examination_a_permanent_stain</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:6709abaac1e9/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:causal_inference"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_skimmed"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:my_initial_skeptical_coloration_became_on_examination_a_permanent_stain"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1908.08741">
    <title>[1908.08741] A relation between log-likelihood and cross-validation log-scores</title>
    <dc:date>2019-08-27T00:05:10+00:00</dc:date>
    <link>https://arxiv.org/abs/1908.08741</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["It is shown that the log-likelihood of a hypothesis or model given some data is equivalent to an average of all leave-one-out cross-validation log-scores that can be calculated from all subsets of the data. This relation can be generalized to any k-fold cross-validation log-scores."

--- This sounds funny, because leave-one-out is (asymptotically) equivalent to the robustified AIC (= Takeuchi information criterion).

--- ETA after reading: The algebra looks legit, but kinda pointless.]]></description>
<dc:subject>statistics likelihood cross-validation have_read not_worth_putting_in_notebooks my_initial_skeptical_coloration_became_on_examination_a_permanent_stain</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:61ed55a054fe/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:likelihood"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:not_worth_putting_in_notebooks"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:my_initial_skeptical_coloration_became_on_examination_a_permanent_stain"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1908.05873">
    <title>[1908.05873] Selection of Exponential-Family Random Graph Models via Held-Out Predictive Evaluation (HOPE)</title>
    <dc:date>2019-08-19T13:18:33+00:00</dc:date>
    <link>https://arxiv.org/abs/1908.05873</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Statistical models for networks with complex dependencies pose particular challenges for model selection and evaluation. In particular, many well-established statistical tools for selecting between models assume conditional independence of observations and/or conventional asymptotics, and their theoretical foundations are not always applicable in a network modeling context. While simulation-based approaches to model adequacy assessment are now widely used, there remains a need for procedures that quantify a model's performance in a manner suitable for selecting among competing models. Here, we propose to address this issue by developing a predictive evaluation strategy for exponential family random graph models that is analogous to cross-validation. Our approach builds on the held-out predictive evaluation (HOPE) scheme introduced by Wang et al. (2016) to assess imputation performance. We systematically hold out parts of the observed network to: evaluate how well the model is able to predict the held-out data; identify where the model performs poorly based on which data are held-out, indicating e.g. potential weaknesses; and calculate general summaries of predictive performance that can be used for model selection. As such, HOPE can assist researchers in improving models by indicating where a model performs poorly, and by quantitatively comparing predictive performance across competing models. The proposed method is applied to model selection problem of two well-known data sets, and the results are compared to those obtained via nominal AIC and BIC scores."

--- But unless the ERGM is projective, with hold-out you'd have to fit it by explicitly marginalizing out the configuration of the held-out portion of the graph, which is going to be computationally very expensive...]]></description>
<dc:subject>to:NB cross-validation exponential_family_random_graphs network_data_analysis statistics re:your_favorite_ergm_sucks</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:98a560c44e30/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:exponential_family_random_graphs"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:network_data_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:your_favorite_ergm_sucks"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1605.02214">
    <title>[1605.02214] On cross-validated Lasso</title>
    <dc:date>2019-08-14T19:18:18+00:00</dc:date>
    <link>https://arxiv.org/abs/1605.02214</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["In this paper, we derive non-asymptotic error bounds for the Lasso estimator when the penalty parameter for the estimator is chosen using K-fold cross-validation. Our bounds imply that the cross-validated Lasso estimator has nearly optimal rates of convergence in the prediction, L2, and L1 norms. For example, we show that in the model with the Gaussian noise and under fairly general assumptions on the candidate set of values of the penalty parameter, the estimation error of the cross-validated Lasso estimator converges to zero in the prediction norm with the slogp/n‾‾‾‾‾‾‾‾√×log(pn)‾‾‾‾‾‾‾√ rate, where n is the sample size of available data, p is the number of covariates, and s is the number of non-zero coefficients in the model. Thus, the cross-validated Lasso estimator achieves the fastest possible rate of convergence in the prediction norm up to a small logarithmic factor log(pn)‾‾‾‾‾‾‾√, and similar conclusions apply for the convergence rate both in L2 and in L1 norms. Importantly, our results cover the case when p is (potentially much) larger than n and also allow for the case of non-Gaussian noise. Our paper therefore serves as a justification for the widely spread practice of using cross-validation as a method to choose the penalty parameter for the Lasso estimator."]]></description>
<dc:subject>to:NB cross-validation lasso regression statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:ad3ed7b1828c/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lasso"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1908.00882">
    <title>[1908.00882] Population Predictive Checks</title>
    <dc:date>2019-08-05T12:47:41+00:00</dc:date>
    <link>https://arxiv.org/abs/1908.00882</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Bayesian modeling has become a staple for researchers analyzing data. Thanks to recent developments in approximate posterior inference, modern researchers can easily build, use, and revise complicated Bayesian models for large and rich data. These new abilities, however, bring into focus the problem of model assessment. Researchers need tools to diagnose the fitness of their models, to understand where a model falls short, and to guide its revision. In this paper we develop a new method for Bayesian model checking, the population predictive check (Pop-PC). Pop-PCs are built on posterior predictive checks (PPC), a seminal method that checks a model by assessing the posterior predictive distribution on the observed data. Though powerful, PPCs use the data twice---both to calculate the posterior predictive and to evaluate it---which can lead to overconfident assessments. Pop-PCs, in contrast, compare the posterior predictive distribution to the population distribution of the data. This strategy blends Bayesian modeling with frequentist assessment, leading to a robust check that validates the model on its generalization. Of course the population distribution is not usually available; thus we use tools like the bootstrap and cross validation to estimate the Pop-PC. Further, we extend Pop-PCs to hierarchical models. We study Pop-PCs on classical regression and a hierarchical model of text. We show that Pop-PCs are robust to overfitting and can be easily deployed on a broad family of models."
]]></description>
<dc:subject>to:NB model_checking bayesianism statistics blei.david re:phil-of-bayes_paper to_read cross-validation</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:221ff74c92f3/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_checking"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:bayesianism"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:blei.david"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:phil-of-bayes_paper"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1907.13413">
    <title>[1907.13413] A Leisurely Look at Versions and Variants of the Cross Validation Estimator</title>
    <dc:date>2019-08-02T15:22:15+00:00</dc:date>
    <link>https://arxiv.org/abs/1907.13413</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Many versions of cross-validation (CV) exist in the literature; and each version though has different variants. All are used interchangeably by many practitioners; yet, without explanation to the connection or difference among them. This article has three contributions. First, it starts by mathematical formalization of these different versions and variants that estimate the error rate and the Area Under the ROC Curve (AUC) of a classification rule, to show the connection and difference among them. Second, we prove some of their properties and prove that many variants are either redundant or "not smooth". Hence, we suggest to abandon all redundant versions and variants and only keep the leave-one-out, the K-fold, and the repeated K-fold. We show that the latter is the only among the three versions that is "smooth" and hence looks mathematically like estimating the mean performance of the classification rules. However, empirically, for the known phenomenon of "weak correlation", which we explain mathematically and experimentally, it estimates both conditional and mean performance almost with the same accuracy. Third, we conclude the article with suggesting two research points that may answer the remaining question of whether we can come up with a finalist among the three estimators: (1) a comparative study, that is much more comprehensive than those available in literature and conclude no overall winner, is needed to consider a wide range of distributions, datasets, and classifiers including complex ones obtained via the recent deep learning approach. (2) we sketch the path of deriving a rigorous method for estimating the variance of the only "smooth" version, repeated K-fold CV, rather than those ad-hoc methods available in the literature that ignore the covariance structure among the folds of CV."]]></description>
<dc:subject>to:NB cross-validation statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:c68d178c4c01/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1908.00325">
    <title>[1908.00325] Estimating the Standard Error of Cross-Validation-Based Estimators of Classification Rules Performance</title>
    <dc:date>2019-08-02T13:24:38+00:00</dc:date>
    <link>https://arxiv.org/abs/1908.00325</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["First, we analyze the variance of the Cross Validation (CV)-based estimators used for estimating the performance of classification rules. Second, we propose a novel estimator to estimate this variance using the Influence Function (IF) approach that had been used previously very successfully to estimate the variance of the bootstrap-based estimators. The motivation for this research is that, as the best of our knowledge, the literature lacks a rigorous method for estimating the variance of the CV-based estimators. What is available is a set of ad-hoc procedures that have no mathematical foundation since they ignore the covariance structure among dependent random variables. The conducted experiments show that the IF proposed method has small RMS error with some bias. However, surprisingly, the ad-hoc methods still work better than the IF-based method. Unfortunately, this is due to the lack of enough smoothness if compared to the bootstrap estimator. This opens the research for three points: (1) more comprehensive simulation study to clarify when the IF method win or loose; (2) more mathematical analysis to figure out why the ad-hoc methods work well; and (3) more mathematical treatment to figure out the connection between the appropriate amount of "smoothness" and decreasing the bias of the IF method."]]></description>
<dc:subject>to:NB cross-validation statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:1fb8912d459f/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1907.12116">
    <title>[1907.12116] A Higher-Order Swiss Army Infinitesimal Jackknife</title>
    <dc:date>2019-07-30T17:29:21+00:00</dc:date>
    <link>https://arxiv.org/abs/1907.12116</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Cross validation (CV) and the bootstrap are ubiquitous model-agnostic tools for assessing the error or variability of machine learning and statistical estimators. However, these methods require repeatedly re-fitting the model with different weighted versions of the original dataset, which can be prohibitively time-consuming. For sufficiently regular optimization problems the optimum depends smoothly on the data weights, and so the process of repeatedly re-fitting can be approximated with a Taylor series that can be often evaluated relatively quickly. The first-order approximation is known as the "infinitesimal jackknife" in the statistics literature and has been the subject of recent interest in machine learning for approximate CV. In this work, we consider high-order approximations, which we call the "higher-order infinitesimal jackknife" (HOIJ). Under mild regularity conditions, we provide a simple recursive procedure to compute approximations of all orders with finite-sample accuracy bounds. Additionally, we show that the HOIJ can be efficiently computed even in high dimensions using forward-mode automatic differentiation. We show that a linear approximation with bootstrap weights approximation is equivalent to those provided by asymptotic normal approximations. Consequently, the HOIJ opens up the possibility of enjoying higher-order accuracy properties of the bootstrap using local approximations. Consistency of the HOIJ for leave-one-out CV under different asymptotic regimes follows as corollaries from our finite-sample bounds under additional regularity assumptions. The generality of the computation and bounds motivate the name "higher-order Swiss Army infinitesimal jackknife.""]]></description>
<dc:subject>to:NB cross-validation bootstrap jackknife statistics jordan.michael_i.</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:173b851e114d/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:bootstrap"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:jackknife"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:jordan.michael_i."/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1907.01670">
    <title>[1907.01670] Double Cross Validation for the Number of Factors in Approximate Factor Models</title>
    <dc:date>2019-07-17T20:59:09+00:00</dc:date>
    <link>https://arxiv.org/abs/1907.01670</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Determining the number of factors is essential to factor analysis. In this paper, we propose {an efficient cross validation (CV)} method to determine the number of factors in approximate factor models. The method applies CV twice, first along the directions of observations and then variables, and hence is referred to hereafter as double cross-validation (DCV). Unlike most CV methods, which are prone to overfitting, the DCV is statistically consistent in determining the number of factors when both dimension of variables and sample size are sufficiently large. Simulation studies show that DCV has outstanding performance in comparison to existing methods in selecting the number of factors, especially when the idiosyncratic error has heteroscedasticity, or heavy tail, or relatively large variance."]]></description>
<dc:subject>cross-validation factor_analysis model_selection statistics in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:91e91308038b/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:factor_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1905.12580">
    <title>[1905.12580] Model Similarity Mitigates Test Set Overuse</title>
    <dc:date>2019-05-30T16:18:41+00:00</dc:date>
    <link>https://arxiv.org/abs/1905.12580</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Excessive reuse of test data has become commonplace in today's machine learning workflows. Popular benchmarks, competitions, industrial scale tuning, among other applications, all involve test data reuse beyond guidance by statistical confidence bounds. Nonetheless, recent replication studies give evidence that popular benchmarks continue to support progress despite years of extensive reuse. We proffer a new explanation for the apparent longevity of test data: Many proposed models are similar in their predictions and we prove that this similarity mitigates overfitting. Specifically, we show empirically that models proposed for the ImageNet ILSVRC benchmark agree in their predictions well beyond what we can conclude from their accuracy levels alone. Likewise, models created by large scale hyperparameter search enjoy high levels of similarity. Motivated by these empirical observations, we give a non-asymptotic generalization bound that takes similarity into account, leading to meaningful confidence bounds in practical settings."

--- So, the only reason what we're doing works is that we're not really changing very much?]]></description>
<dc:subject>to:NB learning_theory cross-validation to_read recht.benjamin to_teach:childs_garden_of_statistical_learning_theory</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:02279344de6f/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:learning_theory"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:recht.benjamin"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:childs_garden_of_statistical_learning_theory"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1811.00645">
    <title>[1811.00645] The Holdout Randomization Test: Principled and Easy Black Box Feature Selection</title>
    <dc:date>2019-05-30T16:04:56+00:00</dc:date>
    <link>https://arxiv.org/abs/1811.00645</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We consider the problem of feature selection using black box predictive models. For example, high-throughput devices in science are routinely used to gather thousands of features for each sample in an experiment. The scientist must then sift through the many candidate features to find explanatory signals in the data, such as which genes are associated with sensitivity to a prospective therapy. Often, predictive models are used for this task: the model is fit, error on held out data is measured, and strong performing models are assumed to have discovered some fundamental properties of the system. A model-specific heuristic is then used to inspect the model parameters and rank important features, with top features reported as "discoveries." However, such heuristics provide no statistical guarantees and can produce unreliable results. We propose the holdout randomization test (HRT) as a principled approach to feature selection using black box predictive models. The HRT is model agnostic and produces a valid p-value for each feature, enabling control over the false discovery rate (or Type I error) for any predictive model. Further, the HRT is computationally efficient and, in simulations, has greater power than a competing knockoffs-based approach."]]></description>
<dc:subject>cross-validation variable_selection statistics blei.david have_read in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:e4ed13c6dd3c/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:variable_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:blei.david"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1905.11744">
    <title>[1905.11744] Evaluating time series forecasting models: An empirical study on performance estimation methods</title>
    <dc:date>2019-05-29T20:01:03+00:00</dc:date>
    <link>https://arxiv.org/abs/1905.11744</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Performance estimation aims at estimating the loss that a predictive model will incur on unseen data. These procedures are part of the pipeline in every machine learning project and are used for assessing the overall generalisation ability of predictive models. In this paper we address the application of these methods to time series forecasting tasks. For independent and identically distributed data the most common approach is cross-validation. However, the dependency among observations in time series raises some caveats about the most appropriate way to estimate performance in this type of data and currently there is no settled way to do so. We compare different variants of cross-validation and of out-of-sample approaches using two case studies: One with 62 real-world time series and another with three synthetic time series. Results show noticeable differences in the performance estimation methods in the two scenarios. In particular, empirical experiments suggest that cross-validation approaches can be applied to stationary time series. However, in real-world scenarios, when different sources of non-stationary variation are at play, the most accurate estimates are produced by out-of-sample methods that preserve the temporal order of observations."]]></description>
<dc:subject>to:NB time_series prediction cross-validation model_selection re:XV_for_mixing statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:ed717cceb1c1/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:time_series"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:XV_for_mixing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://projecteuclid.org/euclid.aos/1558512018">
    <title>Richter , Dahlhaus : Cross validation for locally stationary processes</title>
    <dc:date>2019-05-26T22:29:21+00:00</dc:date>
    <link>https://projecteuclid.org/euclid.aos/1558512018</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We propose an adaptive bandwidth selector via cross validation for local M-estimators in locally stationary processes. We prove asymptotic optimality of the procedure under mild conditions on the underlying parameter curves. The results are applicable to a wide range of locally stationary processes such linear and nonlinear processes. A simulation study shows that the method works fairly well also in misspecified situations."]]></description>
<dc:subject>to:NB cross-validation nonparametrics statistics non-stationarity re:XV_for_mixing</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:40c83ead576d/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:nonparametrics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:non-stationarity"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:XV_for_mixing"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1904.02438">
    <title>[1904.02438] Cross-Validation for Correlated Data</title>
    <dc:date>2019-04-08T21:50:30+00:00</dc:date>
    <link>https://arxiv.org/abs/1904.02438</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["K-fold cross-validation (CV) with squared error loss is widely used for evaluating predictive models, especially when strong distributional data assumptions cannot be taken. However, CV with squared error loss is not free from distributional assumptions, in particular in cases involving non-i.i.d data. This paper analyzes CV for correlated data. We present a criterion for suitability of CV, and introduce a bias corrected cross-validation prediction error estimator, CVc, which is suitable in many settings involving correlated data, where CV is invalid. Our theoretical results are also demonstrated numerically."

--- ETA after reading: I don't see why this is better than the approach taken in the earlier literature (like buffers)]]></description>
<dc:subject>to:NB statistics cross-validation time_series rosset.saharon to_teach:undergrad-ADA to_teach:data_over_space_and_time have_read</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:bb3ea15ebfce/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:time_series"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:rosset.saharon"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:undergrad-ADA"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data_over_space_and_time"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1902.06281">
    <title>[1902.06281] Approximate leave-future-out cross-validation for time series models</title>
    <dc:date>2019-02-21T17:07:40+00:00</dc:date>
    <link>https://arxiv.org/abs/1902.06281</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["One of the common goals of time series analysis is to use the observed series to inform predictions for future observations. In the absence of any actual new data to predict, cross-validation can be used to estimate a model's future predictive accuracy, for instance, for the purpose of model comparison or selection. As exact cross-validation for Bayesian models is often computationally expensive, approximate cross-validation methods have been developed; most notably methods for leave-one-out cross-validation (LOO-CV). If the actual prediction task is to predict the future given the past, LOO-CV provides an overly optimistic estimate as the information from future observations is available to influence predictions of the past. To tackle the prediction task properly and account for the time series structure, we can use leave-future-out cross-validation (LFO-CV). Like exact LOO-CV, exact LFO-CV requires refitting the model many times to different subsets of the data. Using Pareto smoothed importance sampling, we propose a method for approximating exact LFO-CV that drastically reduces the computational costs while also providing informative diagnostics about the quality of the approximation."]]></description>
<dc:subject>to:NB time_series cross-validation statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:02ecca66b9f7/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:time_series"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://web.stanford.edu/~wpmarble/docs/MarbleTyler_SurveyIdealPointsAPSA.pdf">
    <title>How Much Should We Trust Ideal Point Estmates from Surveys?</title>
    <dc:date>2017-10-13T16:49:49+00:00</dc:date>
    <link>https://web.stanford.edu/~wpmarble/docs/MarbleTyler_SurveyIdealPointsAPSA.pdf</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA[Ans.: Not at all.
Commentary, in no particular order:
(1) Justin Gross, in his 2010 Ph.D. thesis, looked at the stability of NOMINATE scores in Senate roll-call voting, across various sub-sets of votes.  His main interest there was in getting at uncertainty in statements like "the 32nd most conservative senator", but some of his findings would also apply to CV for that context.  (I don't know if Justin ever published that separately.)
(2) I suspect there is something funny about the way they are doing CV, because when they simulate from 1D ideal-point models, it doesn't favor 1 dimension!  I think it might be better to make the validation set be distributed across both questions and items (as in Dabbs & Junker on network CV, https://arxiv.org/abs/1605.03000), but I am not sure.
(3) Regressing changes in average likelihood  to see which sub-groups benefit most from an ideal point model is just weird.  At the very least, I'd use log-likelihood, and there should be some way of getting at this by estimating a joint model, with both covariates and latent ideal points.]]></description>
<dc:subject>to:NB public_opinion inference_to_latent_objects statistics have_read via:henry_farrell cross-validation</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:73bdf76bb169/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:public_opinion"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:inference_to_latent_objects"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:via:henry_farrell"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1705.07349">
    <title>[1705.07349] $left( beta, varpi right)$-stability for cross-validation and the choice of the number of folds</title>
    <dc:date>2017-08-19T23:50:53+00:00</dc:date>
    <link>https://arxiv.org/abs/1705.07349</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["In this paper, we introduce a new concept of stability for cross-validation, called the (β,ϖ)-stability, and use it as a new perspective to build the general theory for cross-validation. The (β,ϖ)-stability mathematically connects the generalization ability and the stability of the cross-validated model via the Rademacher complexity. Our result reveals mathematically the effect of cross-validation from two sides: on one hand, cross-validation picks the model with the best empirical generalization ability by validating all the alternatives on test sets; on the other hand, cross-validation may compromise the stability of the model selection by causing subsampling error. Moreover, the difference between training and test errors in q\textsuperscript{th} round, sometimes referred to as the generalization error, might be autocorrelated on q. Guided by the ideas above, the (β,ϖ)-stability help us derivd a new class of Rademacher bounds, referred to as the one-round/convoluted Rademacher bounds, for the stability of cross-validation in both the i.i.d.\ and non-i.i.d.\ cases. For both light-tail and heavy-tail losses, the new bounds quantify the stability of the one-round/average test error of the cross-validated model in terms of its one-round/average training error, the sample sizes n, number of folds K, the tail property of the loss (encoded as Orlicz-Ψν norms) and the Rademacher complexity of the model class Λ. The new class of bounds not only quantitatively reveals the stability of the generalization ability of the cross-validated model, it also shows empirically the optimal choice for number of folds K, at which the upper bound of the one-round/average test error is lowest, or, to put it in another way, where the test error is most stable."]]></description>
<dc:subject>to:NB to_read cross-validation stability_of_learning model_selection statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:bef52a741048/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:stability_of_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://www.mitpressjournals.org/doi/abs/10.1162/NECO_a_00923">
    <title>Block-Regularized m × 2 Cross-Validated Estimator of the Generalization Error | Neural Computation | MIT Press Journals</title>
    <dc:date>2017-01-25T15:03:06+00:00</dc:date>
    <link>http://www.mitpressjournals.org/doi/abs/10.1162/NECO_a_00923</link>
    <dc:creator>cshalizi</dc:creator><dc:subject>to:NB cross-validation statistics re:ADAfaEPoV</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:f08cdd59d111/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:ADAfaEPoV"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://papers.nips.cc/paper/3294-modeling-homophily-and-stochastic-equivalence-in-symmetric-relational-data">
    <title>Modeling homophily and stochastic equivalence in symmetric relational data</title>
    <dc:date>2016-05-10T17:44:22+00:00</dc:date>
    <link>http://papers.nips.cc/paper/3294-modeling-homophily-and-stochastic-equivalence-in-symmetric-relational-data</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This article discusses a latent variable model for inference and prediction of symmetric relational data. The model, based on the idea of the eigenvalue decomposition, represents the relationship between two nodes as the weighted inner-product of node-specific vectors of latent characteristics. This eigenmodel'' generalizes other popular latent variable models, such as latent class and distance models: It is shown mathematically that any latent class or distance model has a representation as an eigenmodel, but not vice-versa. The practical implications of this are examined in the context of three real datasets, for which the eigenmodel has as good or better out-of-sample predictive performance than the other two models."

--- Why the EXPLETIVE hadn't I read this before?]]></description>
<dc:subject>network_data_analysis re:network_differences statistics hoff.peter community_discovery inference_to_latent_objects cross-validation re:XV_for_networks have_read in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:2af1af617696/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:network_data_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:network_differences"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hoff.peter"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:community_discovery"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:inference_to_latent_objects"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:XV_for_networks"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1502.04585">
    <title>[1502.04585] The Ladder: A Reliable Leaderboard for Machine Learning Competitions</title>
    <dc:date>2016-04-25T16:58:57+00:00</dc:date>
    <link>http://arxiv.org/abs/1502.04585</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["The organizer of a machine learning competition faces the problem of maintaining an accurate leaderboard that faithfully represents the quality of the best submission of each competing team. What makes this estimation problem particularly challenging is its sequential and adaptive nature. As participants are allowed to repeatedly evaluate their submissions on the leaderboard, they may begin to overfit to the holdout data that supports the leaderboard. Few theoretical results give actionable advice on how to design a reliable leaderboard. Existing approaches therefore often resort to poorly understood heuristics such as limiting the bit precision of answers and the rate of re-submission. 
"In this work, we introduce a notion of "leaderboard accuracy" tailored to the format of a competition. We introduce a natural algorithm called "the Ladder" and demonstrate that it simultaneously supports strong theoretical guarantees in a fully adaptive model of estimation, withstands practical adversarial attacks, and achieves high utility on real submission files from an actual competition hosted by Kaggle. 
"Notably, we are able to sidestep a powerful recent hardness result for adaptive risk estimation that rules out algorithms such as ours under a seemingly very similar notion of accuracy. On a practical note, we provide a completely parameter-free variant of our algorithm that can be deployed in a real competition with no tuning required whatsoever."

--- Basically, return the new score if, but only if, the new submission beats the previous best by some threshold.  I think this blocks my "flood with models" attack...]]></description>
<dc:subject>learning_theory cross-validation have_read via:arthegall</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:b2bafec2bfe3/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:learning_theory"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:via:arthegall"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1504.01132">
    <title>[1504.01132] Recursive Partitioning for Heterogeneous Causal Effects</title>
    <dc:date>2016-03-28T01:02:49+00:00</dc:date>
    <link>http://arxiv.org/abs/1504.01132</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["In this paper we study the problems of estimating heterogeneity in causal effects in experimental or observational studies and conducting inference about the magnitude of the differences in treatment effects across subsets of the population. In applications, our method provides a data-driven approach to determine which subpopulations have large or small treatment effects and to test hypotheses about the differences in these effects. For experiments, our method allows researchers to identify heterogeneity in treatment effects that was not specified in a pre-analysis plan, without concern about invalidating inference due to multiple testing. In most of the literature on supervised machine learning (e.g. regression trees, random forests, LASSO, etc.), the goal is to build a model of the relationship between a unit's attributes and an observed outcome. A prominent role in these methods is played by cross-validation which compares predictions to actual outcomes in test samples, in order to select the level of complexity of the model that provides the best predictive power. Our method is closely related, but it differs in that it is tailored for predicting causal effects of a treatment rather than a unit's outcome. The challenge is that the "ground truth" for a causal effect is not observed for any individual unit: we observe the unit with the treatment, or without the treatment, but not both at the same time. Thus, it is not obvious how to use cross-validation to determine whether a causal effect has been accurately predicted. We propose several novel cross-validation criteria for this problem and demonstrate through simulations the conditions under which they perform better than standard methods for the problem of causal effects. We then apply the method to a large-scale field experiment re-ranking results on a search engine."]]></description>
<dc:subject>to:NB decision_trees regression causal_inference cross-validation statistics athey.susan imbens.guido_w.</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:fe6061dfaa75/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:causal_inference"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:athey.susan"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:imbens.guido_w."/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arstechnica.co.uk/security/2016/02/the-nsas-skynet-program-may-be-killing-thousands-of-innocent-people/">
    <title>The NSA’s SKYNET program may be killing thousands of innocent people | Ars Technica UK</title>
    <dc:date>2016-02-16T17:56:29+00:00</dc:date>
    <link>http://arstechnica.co.uk/security/2016/02/the-nsas-skynet-program-may-be-killing-thousands-of-innocent-people/</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA[We have much to answer for.]]></description>
<dc:subject>the_continuing_crises national_surveillance_state machine_learning classifiers cross-validation bad_data_analysis terrorism_fears drones decision_trees ensemble_methods to_teach:data-mining to:blog random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:1248c0c8cf03/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:the_continuing_crises"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:national_surveillance_state"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:bad_data_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:terrorism_fears"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:drones"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:blog"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1602.01522">
    <title>[1602.01522] Risk estimation for high-dimensional lasso regression</title>
    <dc:date>2016-02-09T02:46:56+00:00</dc:date>
    <link>http://arxiv.org/abs/1602.01522</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["In high-dimensional estimation, analysts are faced with more parameters p than available observations n, and asymptotic analysis of performance allows the ratio p/n→∞. This situation makes regularization both necessary and desirable in order for estimators to possess theoretical guarantees. However, the amount of regularization, often determined by one or more tuning parameters, is integral to achieving good performance. In practice, choosing the tuning parameter is done through resampling methods (e.g. cross-validation), generalized information criteria, or reformulating the optimization problem (e.g. square-root lasso or scaled sparse regression). Each of these techniques comes with varying levels of theoretical guarantee for the low- or high-dimensional regimes. However, there are some notable deficiencies in the literature. The theory, and sometimes practice, of many methods relies on either the knowledge or estimation of the variance parameter, which is difficult to estimate in high dimensions. In this paper, we provide theoretical intuition suggesting that some previously proposed approaches based on information criteria work poorly in high dimensions. We introduce a suite of new risk estimators leveraging the burgeoning literature on high-dimensional variance estimation. Finally, we compare our proposal to many existing methods for choosing the tuning parameters for lasso regression by providing an extensive simulation to examine their finite sample performance. We find that our new estimators perform quite well, often better than the existing approaches across a wide range of simulation conditions and evaluation criteria."]]></description>
<dc:subject>to:NB cross-validation lasso statistics high-dimensional_statistics regression information_criteria kith_and_kin mcdonald.daniel homrighausen.darren</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:bbaae37c2aa3/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lasso"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:high-dimensional_statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:information_criteria"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:kith_and_kin"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:mcdonald.daniel"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:homrighausen.darren"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://www.tandfonline.com/doi/abs/10.1080/01621459.1979.10481632">
    <title>A Predictive Approach to Model Selection - Journal of the American Statistical Association - Volume 74, Issue 365</title>
    <dc:date>2015-11-08T19:15:43+00:00</dc:date>
    <link>http://www.tandfonline.com/doi/abs/10.1080/01621459.1979.10481632</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This article offers a synthesis of Bayesian and sample-reuse approaches to the problem of high structure model selection geared to prediction. Similar methods are used for low structure models. Nested and nonnested paradigms are discussed and examples given."

--- Submitted 1977!]]></description>
<dc:subject>in_NB have_read prediction model_selection statistics cross-validation geisser.seymour eddy.william_f. kith_and_kin</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:31738bd3baa2/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:geisser.seymour"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:eddy.william_f."/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:kith_and_kin"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1507.00066">
    <title>[1507.00066] Fast Cross-Validation for Incremental Learning</title>
    <dc:date>2015-08-05T16:47:21+00:00</dc:date>
    <link>http://arxiv.org/abs/1507.00066</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Cross-validation (CV) is one of the main tools for performance estimation and parameter tuning in machine learning. The general recipe for computing CV estimate is to run a learning algorithm separately for each CV fold, a computationally expensive process. In this paper, we propose a new approach to reduce the computational burden of CV-based performance estimation. As opposed to all previous attempts, which are specific to a particular learning model or problem domain, we propose a general method applicable to a large class of incremental learning algorithms, which are uniquely fitted to big data problems. In particular, our method applies to a wide range of supervised and unsupervised learning tasks with different performance criteria, as long as the base learning algorithm is incremental. We show that the running time of the algorithm scales logarithmically, rather than linearly, in the number of CV folds. Furthermore, the algorithm has favorable properties for parallel and distributed implementation. Experiments with state-of-the-art incremental learning algorithms confirm the practicality of the proposed method."]]></description>
<dc:subject>to:NB cross-validation computational_statistics statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:c4906556b0b4/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:computational_statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1503.03515">
    <title>[1503.03515] Bi-cross-validation for factor analysis</title>
    <dc:date>2015-05-21T14:17:47+00:00</dc:date>
    <link>http://arxiv.org/abs/1503.03515</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Factor analysis is over a century old, but it is still problematic to choose the number of factors for a given data set. The scree test is popular but subjective. The best performing objective methods are recommended on the basis of simulations. We introduce a method based on bi-cross-validation, using randomly held-out submatrices of the data to choose the number of factors. We find it performs better than the leading methods of parallel analysis (PA) and Kaiser's rule. Our performance criterion is based on recovery of the underlying factor-loading (signal) matrix rather than identifying the true number of factors. Like previous comparisons, our work is simulation based. Recent advances in random matrix theory provide principled choices for the number of factors when the noise is homoscedastic, but not for the heteroscedastic case. The simulations we choose are designed using guidance from random matrix theory. In particular, we include factors too small to detect, factors large enough to detect but not large enough to improve the estimate, and two classes of factors large enough to be useful. Much of the advantage of bi-cross-validation comes from cases with factors large enough to detect but too small to be well estimated. We also find that a form of early stopping regularization improves the recovery of the signal matrix."

--- Published version: https://doi.org/10.1214/15-STS539]]></description>
<dc:subject>model_selection factor_analysis cross-validation owen.art statistics re:ADAfaEPoV in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:15784b4d07b6/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:factor_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:owen.art"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:ADAfaEPoV"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1411.1469">
    <title>[1411.1469] A Generic Sample Splitting Approach for Refined Community Recovery in Stochastic Block Models</title>
    <dc:date>2015-01-22T00:26:49+00:00</dc:date>
    <link>http://arxiv.org/abs/1411.1469</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We propose and analyze a generic method for community recovery in stochastic block models and degree corrected block models. This approach can exactly recover the hidden communities with high probability when the expected node degrees are of order logn or higher. Starting from a roughly correct community partition given by some conventional community recovery algorithm, this method refines the partition in a cross clustering step. Our results simplify and extend some of the previous work on exact community recovery, discovering the key role played by sample splitting. The proposed method is simple and can be implemented with many practical community recovery algorithms."]]></description>
<dc:subject>community_discovery cross-validation network_data_analysis statistics lei.jing in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:3de2c784a3cd/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:community_discovery"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:network_data_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lei.jing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1405.6974">
    <title>[1405.6974] Futility Analysis in the Cross-Validation of Machine Learning Models</title>
    <dc:date>2015-01-20T00:04:23+00:00</dc:date>
    <link>http://arxiv.org/abs/1405.6974</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Many machine learning models have important structural tuning parameters that cannot be directly estimated from the data. The common tactic for setting these parameters is to use resampling methods, such as cross--validation or the bootstrap, to evaluate a candidate set of values and choose the best based on some pre--defined criterion. Unfortunately, this process can be time consuming. However, the model tuning process can be streamlined by adaptively resampling candidate values so that settings that are clearly sub-optimal can be discarded. The notion of futility analysis is introduced in this context. An example is shown that illustrates how adaptive resampling can be used to reduce training time. Simulation studies are used to understand how the potential speed--up is affected by parallel processing techniques."]]></description>
<dc:subject>to:NB model_selection cross-validation statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:9a30c602916f/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1411.1715">
    <title>[1411.1715] Network Cross-Validation for Determining the Number of Communities in Network Data</title>
    <dc:date>2014-12-01T22:19:45+00:00</dc:date>
    <link>http://arxiv.org/abs/1411.1715</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["The stochastic block model and its variants have been a popular tool in analyzing large network data with community structures. Model selection for these network models, such as determining the number of communities, has been a challenging statistical inference task. In this paper we develop an efficient cross-validation approach to determine the number of communities, as well as to choose between the regular stochastic block model and the degree corrected block model. Our method, called network cross-validation, is based on a block-wise edge splitting technique, combined with an integrated step of community recovery using sub-blocks of the adjacency matrix. The solid performance of our method is supported by theoretical analysis of the sub-block parameter estimation, and is demonstrated in extensive simulations and a data example. Extensions to more general network models are also discussed."]]></description>
<dc:subject>cross-validation community_discovery network_data_analysis statistics kith_and_kin lei.jing re:XV_for_networks in_NB to:blog have_read to_teach:baby-nets</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:80e907153178/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:community_discovery"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:network_data_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:kith_and_kin"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lei.jing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:XV_for_networks"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:blog"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:baby-nets"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://link.springer.com/article/10.1007/s10115-014-0789-0">
    <title>Evaluating link prediction methods - Online First - Springer</title>
    <dc:date>2014-10-09T13:24:21+00:00</dc:date>
    <link>http://link.springer.com/article/10.1007/s10115-014-0789-0</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Link prediction is a popular research area with important applications in a variety of disciplines, including biology, social science, security, and medicine. The fundamental requirement of link prediction is the accurate and effective prediction of new links in networks. While there are many different methods proposed for link prediction, we argue that the practical performance potential of these methods is often unknown because of challenges in the evaluation of link prediction, which impact the reliability and reproducibility of results. We describe these challenges, provide theoretical proofs and empirical examples demonstrating how current methods lead to questionable conclusions, show how the fallacy of these conclusions is illuminated by methods we propose, and develop recommendations for consistent, standard, and applicable evaluation metrics. We also recommend the use of precision-recall threshold curves and associated areas in lieu of receiver operating characteristic curves due to complications that arise from extreme imbalance in the link prediction classification problem."]]></description>
<dc:subject>to:NB network_data_analysis statistics cross-validation link_prediction re:XV_for_networks to_teach:baby-nets</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:f9bb0e5505a2/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:network_data_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:link_prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:XV_for_networks"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:baby-nets"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1403.4544">
    <title>[1403.4544] On the Sensitivity of the Lasso to the Number of Predictor Variables</title>
    <dc:date>2014-03-21T15:52:20+00:00</dc:date>
    <link>http://arxiv.org/abs/1403.4544</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["The Lasso is a computationally efficient procedure that can produce sparse estimators when the number of predictors (p) is large. Oracle inequalities provide probability loss bounds for the Lasso estimator at a deterministic choice of the regularization parameter. These bounds tend to zero if p is appropriately controlled, and are thus commonly cited as theoretical justification for the Lasso and its ability to handle high-dimensional settings. Unfortunately, in practice the regularization parameter is not selected to be a deterministic quantity, but is instead chosen using a random, data-dependent procedure. To address this shortcoming of previous theoretical work, we study the loss of the Lasso estimator when tuned optimally for prediction. Assuming orthonormal predictors and a sparse true model, we prove that the probability that the best possible predictive performance of the Lasso deteriorates as p increases can be arbitrarily close to one given a sufficiently high signal to noise ratio and sufficiently large p. We further demonstrate empirically that the deterioration in performance can be far worse than is commonly suggested in the literature and provide a real data example where deterioration is observed."]]></description>
<dc:subject>lasso regression variable_selection high-dimensional_statistics cross-validation statistics in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:c0b013c52c01/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lasso"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:variable_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:high-dimensional_statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://projecteuclid.org/euclid.ejs/1392215030">
    <title>Kim , Huo : Asymptotic optimality of a multivariate version of the generalized cross validation in adaptive smoothing splines</title>
    <dc:date>2014-02-20T01:05:33+00:00</dc:date>
    <link>http://projecteuclid.org/euclid.ejs/1392215030</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We consider an adaptive smoothing spline with a piecewise-constant penalty function λ(x), in which a univariate smoothing parameter λ in the classic smoothing spline is converted into an adaptive multivariate parameter λ. Choosing the optimal value of λ is critical for obtaining desirable estimates. We propose to choose λ by minimizing a multivariate version of the generalized cross validation function; the resulting estimator is shown to be consistent and asymptotically optimal under some general conditions—i.e., the counterparts of the nice asymptotic properties of the generalized cross validation in the ordinary smoothing spline are still provable. This provides theoretical justification of adopting the multivariate version of the generalized cross validation principle in adaptive smoothing splines."]]></description>
<dc:subject>splines smoothing nonparametrics cross-validation in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:a1c2eca5acc1/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:splines"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:smoothing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:nonparametrics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1309.2068">
    <title>[1309.2068] Modified Cross-Validation for Penalized High-Dimensional Linear Regression Models</title>
    <dc:date>2013-09-10T18:24:51+00:00</dc:date>
    <link>http://arxiv.org/abs/1309.2068</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["In this paper, for Lasso penalized linear regression models in high-dimensional settings, we propose a modified cross-validation method for selecting the penalty parameter. The methodology is extended to other penalties, such as Elastic Net. We conduct extensive simulation studies and real data analysis to compare the performance of the modified cross-validation method with other methods. It is shown that the popular $K$-fold cross-validation method includes many noise variables in the selected model, while the modified cross-validation works well in a wide range of coefficient and correlation settings. Supplemental materials containing the computer code are available online."]]></description>
<dc:subject>cross-validation lasso regression statistics high-dimensional_statistics variable_selection in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:809b6dd19675/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lasso"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:high-dimensional_statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:variable_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://www.jstor.org/stable/2984809">
    <title>Cross-Validatory Choice and Assessment of Statistical Predictions</title>
    <dc:date>2013-08-13T20:12:22+00:00</dc:date>
    <link>http://www.jstor.org/stable/2984809</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["A generalized form of the cross-validation criterion is applied to the choice and assessment of prediction using the data-analytic concept of a prescription. The examples used to illustrate the application are drawn from the problem areas of univariate estimation, linear regression and analysis of variance."]]></description>
<dc:subject>have_read cross-validation statistics prediction re:XV_for_mixing re:XV_for_networks re:stacs to_teach:undergrad-ADA in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:31be6815120e/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:XV_for_mixing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:XV_for_networks"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:stacs"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:undergrad-ADA"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://biostats.bepress.com/ucbbiostat/paper313/">
    <title>&quot;Subsemble: An Ensemble Method for Combining Subset-Specific Algorithm &quot; by Stephanie Sapp, Mark J. van der Laan et al.</title>
    <dc:date>2013-05-26T14:38:18+00:00</dc:date>
    <link>http://biostats.bepress.com/ucbbiostat/paper313/</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Ensemble methods using the same underlying algorithm trained on different subsets of observations have recently received increased attention as practical prediction tools for massive datasets. We propose Subsemble: a general subset ensemble prediction method, which can be used for small, moderate, or large datasets. Subsemble partitions the full dataset into subsets of observations, fits a specified underlying algorithm on each subset, and uses a clever form of V-fold cross-validation to output a prediction function that combines the subset-specific fits. We give an oracle result that provides a theoretical performance guarantee for Subsemble. Through simulations, we demonstrate that Subsemble can be a beneficial tool for small to moderate sized datasets, and often has better prediction performance than the underlying algorithm fit just once on the full dataset. We also describe how to include Subsemble as a candidate in a SuperLearner library, providing a practical way to evaluate the performance of Subsemble relative to the underlying algorithm fit just once on the full dataset."]]></description>
<dc:subject>ensemble_methods prediction cross-validation van_der_laan.mark machine_learning statistics in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:49b81e700246/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:van_der_laan.mark"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://jpr.sagepub.com/content/47/4/363">
    <title>The perils of policy by p-value: Predicting civil conflicts</title>
    <dc:date>2013-05-25T14:31:48+00:00</dc:date>
    <link>http://jpr.sagepub.com/content/47/4/363</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Large-n studies of conflict have produced a large number of statistically significant results but little accurate guidance in terms of anticipating the onset of conflict. The authors argue that too much attention has been paid to finding statistically significant relationships, while too little attention has been paid to finding variables that improve our ability to predict civil wars. The result can be a distorted view of what matters most to the onset of conflict. Although these models may not be intended to be predictive models, prescriptions based on these models are generally based on statistical significance, and the predictive attributes of the underlying models are generally ignored. These predictions should not be ignored, but rather need to be heuristically evaluated because they may shed light on the veracity of the models. In this study, the authors conduct a side-by-side comparison of the statistical significance and predictive power of the different variables used in two of the most influential models of civil war. The results provide a clear demonstration of how potentially misleading the traditional focus on statistical significance can be. Until out-of-sample heuristics — especially including predictions — are part of the normal evaluative tools in conflict research, we are unlikely to make sufficient theoretical progress beyond broad statements that point to GDP per capita and population as the major causal factors accounting for civil war onset."

- I like the phrase "gazing at the significance stars".  Also, it seems the replication files use R; this should definitely go into ADAfaEPoV, since it uses nothing the Kids won't know.]]></description>
<dc:subject>social_science_methodology p-values hypothesis_testing statistics prediction political_science violence to_teach:undergrad-ADA via:abumuqawama have_read cross-validation to_teach:data-mining logistic_regression in_NB to:blog have_taught</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:28d2152d54d5/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:social_science_methodology"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:p-values"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hypothesis_testing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:political_science"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:violence"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:undergrad-ADA"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:via:abumuqawama"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:logistic_regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:blog"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_taught"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://biostats.bepress.com/uncbiostat/art35/">
    <title>&quot;Cross-Validation for Nonlinear Mixed Effects Models&quot; by Emily Colby and Eric Bair</title>
    <dc:date>2013-03-15T21:57:23+00:00</dc:date>
    <link>http://biostats.bepress.com/uncbiostat/art35/</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Cross-validation is frequently used for model selection in a variety of applications. However, it is difficult to apply cross-validation to mixed effects models (including the nonlinear mixed effects models) due to the fact that cross-validation requires “out-of-sample” predictions of the outcome variable, which cannot be easily calculated when random effects are present.We describe two novel variants of cross-validation that can be applied to nonlinear mixed effects models. One variant, where out-of-sample predictions are based on post hoc estimates of the random effects, can be used to select the overall structural model. Another variant, where cross-validation seeks to minimize the estimated random effects rather than the estimated residuals, can be used to select covariates to include in the model.We show that these methods produce accurate results in a variety of simulated data sets and apply them to two publicly available population pharmacokinetic data sets."]]></description>
<dc:subject>cross-validation regression statistics model_selection hierarchical_statistical_models in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:897ca4263e69/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hierarchical_statistical_models"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1303.3128">
    <title>[1303.3128] Estimation Stability with Cross Validation (ESCV)</title>
    <dc:date>2013-03-14T16:42:40+00:00</dc:date>
    <link>http://arxiv.org/abs/1303.3128</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Cross-validation (CV) is often used to select the regularization parameter in high dimensional problems. However, when applied to the sparse modeling method Lasso, CV leads to models that are unstable in high-dimensions, and consequently not suited for reliable interpretation. In this paper, we propose a model-free criterion ESCV based on a new estimation stability (ES) metric and CV. Our proposed ESCV finds a locally ES-optimal model smaller than the CV choice so that the it fits the data and also enjoys estimation stability property. We demonstrate that ESCV is an effective alternative to CV at a similar easily parallelizable computational cost. In particular, we compare the two approaches with respect to several performance measures when applied to the Lasso on both simulated and real data sets. For dependent predictors common in practice, our main finding is that, ESCV cuts down false positive rates often by a large margin, while sacrificing little of true positive rates. ESCV usually outperforms CV in terms of parameter estimation while giving similar performance as CV in terms of prediction. For the two real data sets from neuroscience and cell biology, the models found by ESCV are less than half of the model sizes by CV. Judged based on subject knowledge, they are more plausible than those by CV as well. We also discuss some regularization parameter alignment issues that come up in both approaches."]]></description>
<dc:subject>to_read cross-validation stability_of_learning statistics yu.bin model_selection in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:183ea9caceaf/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:stability_of_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:yu.bin"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://www.russpoldrack.org/2012/12/the-perils-of-leave-one-out.html">
    <title>russpoldrack.org: The perils of leave-one-out crossvalidation for individual difference analyses</title>
    <dc:date>2012-12-17T13:13:19+00:00</dc:date>
    <link>http://www.russpoldrack.org/2012/12/the-perils-of-leave-one-out.html</link>
    <dc:creator>cshalizi</dc:creator><dc:subject>cross-validation regression fmri to_teach:undergrad-ADA to_teach:data-mining poldrack.russell</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:fcb238b8ca66/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:fmri"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:undergrad-ADA"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:poldrack.russell"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1212.1780">
    <title>[1212.1780] An Empirical Comparison of V-fold Penalisation and Cross Validation for Model Selection in Distribution-Free Regression</title>
    <dc:date>2012-12-14T02:06:44+00:00</dc:date>
    <link>http://arxiv.org/abs/1212.1780</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Model selection is a crucial issue in machine-learning and a wide variety of penalisation methods (with possibly data dependent complexity penalties) have recently been introduced for this purpose. However their empirical performance is generally not well documented in the literature. It is the goal of this paper to investigate to which extent such recent techniques can be successfully used for the tuning of both the regularisation and kernel parameters in support vector regression (SVR) and the complexity measure in regression trees (CART). This task is traditionally solved via V-fold cross-validation (VFCV), which gives efficient results for a reasonable computational cost. A disadvantage however of VFCV is that the procedure is known to provide an asymptotically suboptimal risk estimate as the number of examples tends to infinity. Recently, a penalisation procedure called V-fold penalisation has been proposed to improve on VFCV, supported by theoretical arguments. Here we report on an extensive set of experiments comparing V-fold penalisation and VFCV for SVR/CART calibration on several benchmark datasets. We highlight cases in which VFCV and V-fold penalisation provide poor estimates of the risk respectively and introduce a modified penalisation technique to reduce the estimation error."]]></description>
<dc:subject>cross-validation model_selection regression nonparametrics to_teach:undergrad-ADA in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:c94f8413fdea/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:nonparametrics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:undergrad-ADA"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1210.6187">
    <title>[1210.6187] Kriging-based sequential design strategies using fast cross-validation techniques with extensions to multi-fidelity computer codes</title>
    <dc:date>2012-10-26T13:04:25+00:00</dc:date>
    <link>http://arxiv.org/abs/1210.6187</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Kriging-based surrogate models have become very popular during the last decades to approximate a computer code output from few simulations. In practical applications, it is very common to sequentially add new simulations to obtain more accurate approximations. We propose in this paper a method of kriging-based sequential design which combines both the error evaluation providing by the kriging model and the observed errors of a Leave-One-Out cross-validation procedure. This method is proposed in two versions, the first one selects points one at-a-time. The second one allows us to parallelize the simulations and to add several design points at-a-time. Then, we extend these strategies to multi-fidelity co-kriging models which allow us to surrogate a complex code using fast approximations of it. The main advantage of these extensions is that it not only provides the new locations where to perform simulations but also which versions of code have to be simulated (between the complex one or one of its fast approximations). A real multi-fidelity application is used to illustrate the efficiency of the proposed approaches. In this example, the accurate code is a two-dimensional finite element model and the less accurate one is a one-dimensional approximation of the system."]]></description>
<dc:subject>smoothing statistics simulation cross-validation spatial_statistics in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:a7d7d1eb2de1/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:smoothing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:simulation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:spatial_statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1210.5830">
    <title>[1210.5830] $V$-fold cross-validation and $V$-fold penalization in least-squares density estimation</title>
    <dc:date>2012-10-25T16:20:48+00:00</dc:date>
    <link>http://arxiv.org/abs/1210.5830</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This paper studies $V$-fold cross-validation for model selection in least-squares density estimation. The goal is to provide theoretical grounds for choosing $V$ in order to minimize the least-squares risk of the selected estimator. % We first prove a non asymptotic oracle inequality for $V$-fold cross-validation and its bias-corrected version ($V$-fold penalization), with an upper bound decreasing as a function of $V$. In particular, this result implies $V$-fold penalization is asymptotically optimal. % Then, we compute the variance of $V$-fold cross-validation and related criteria, as well as the variance of key quantities for model selection performances. We show these variances depend on $V$ like $1+1/(V-1)$ (at least in some particular cases), suggesting the performances increase much from V=2 to V=5 or 10, and then is almost constant. % Overall, this explains the common advice to take $V=10 $---at least in our setting and when the computational power is limited---, as confirmed by some simulation experiments."]]></description>
<dc:subject>cross-validation statistics density_estimation to_read arlot.sylvain in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:8af133553736/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:density_estimation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:arlot.sylvain"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://projecteuclid.org/DPubS?service=UI&amp;version=1.0&amp;verb=Display&amp;handle=euclid.ejs/1349355603">
    <title>Lecué , Mitchell : Oracle inequalities for cross-validation type procedures</title>
    <dc:date>2012-10-04T14:47:25+00:00</dc:date>
    <link>http://projecteuclid.org/DPubS?service=UI&amp;version=1.0&amp;verb=Display&amp;handle=euclid.ejs/1349355603</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We prove oracle inequalities for three different types of adaptation procedures inspired by cross-validation and aggregation. These procedures are then applied to the construction of Lasso estimators and aggregation with exponential weights with data-driven regularization and temperature parameters, respectively. We also prove oracle inequalities for the cross-validation procedure itself under some convexity assumptions."

--- It seems to me that their example 2.8, of a case where un-modified CV will do badly, is rather cheating, because the two estimators change behavior as the sample size changes without rhyme or reason.  (They're almost "grue" and "bleen", actually.)  I'm not sure how exactly to phrase this mathematically.]]></description>
<dc:subject>cross-validation statistics re:XV_for_mixing re:XV_for_networks have_read empirical_processes in_NB lasso ensemble_methods</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:a711e9e5dfa3/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:XV_for_mixing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:XV_for_networks"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:empirical_processes"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lasso"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1206.6128">
    <title>[1206.6128] Cross-validation is risk consistent for lasso under orthogonal design</title>
    <dc:date>2012-06-28T13:08:34+00:00</dc:date>
    <link>http://arxiv.org/abs/1206.6128</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["The lasso has become so ubiquitous in statistics that it can be considered a paradigm for inference and prediction. During the last fifteen years, the lasso procedure has been the target of a substantial amount of theoretical and applied research. Correspondingly, many results are known about its behavior for a fixed or optimally chosen smoothing parameter. These results, such as sparsistency and risk consistency, are of some comfort when using the lasso in applications. However, much less is known about its behavior when the smoothing parameter is chosen in a data dependent way, which is almost always the case in practice. To this end, we give the first definitive answer about the risk consistency of lasso when the smoothing parameter is chosen empirically using the same data that are used to fit the lasso estimator. We show that under restrictions on the design matrix, the lasso estimator is still risk consistent when the smoothing parameter is chosen via cross-validation."]]></description>
<dc:subject>have_read kith_and_kin lasso regression statistics cross-validation mcdonald.daniel homrighausen.darren in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:8c73ee48bab1/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:kith_and_kin"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lasso"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:mcdonald.daniel"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:homrighausen.darren"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://gsp.tamu.edu/Publications/journal-publications/performance-analysis-and-error-estimation">
    <title>Performance Analysis and Error Estimation — GSP Lab</title>
    <dc:date>2012-06-23T21:37:49+00:00</dc:date>
    <link>http://gsp.tamu.edu/Publications/journal-publications/performance-analysis-and-error-estimation</link>
    <dc:creator>cshalizi</dc:creator><dc:subject>track_down_references cross-validation statistics machine_learning model_selection to_teach:data-mining to_teach:undergrad-ADA</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:8c125ecfb061/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:track_down_references"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_selection"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:undergrad-ADA"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=6172585">
    <title>On the Stability of Empirical Risk Minimization in the Presence of Multiple Risk Minimizers</title>
    <dc:date>2012-06-12T22:05:38+00:00</dc:date>
    <link>http://ieeexplore.ieee.org/xpl/articleDetails.jsp?arnumber=6172585</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Recently, Kutin and Niyogi investigated several notions of algorithmic stability—a property of a learning map conceptually similar to continuity—showing that training stability is sufficient for consistency of empirical risk minimization (ERM) while distribution-free CV-stability is necessary and sufficient for having finite VC-dimension. This paper concerns a phase transition in the training stability of ERM, conjectured by the same authors. Kutin and Niyogi proved that ERM on finite hypothesis spaces containing a unique risk minimizer has training stability that scales exponentially with sample size, and conjectured that the existence of multiple risk minimizers prevents even super-quadratic convergence. We prove this result for the strictly weaker notion of CV-stability, positively resolving the conjecture."]]></description>
<dc:subject>learning_theory stability_of_learning cross-validation vc-dimension in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:713ee6523cf9/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:learning_theory"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:stability_of_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:vc-dimension"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1206.2248">
    <title>[1206.2248] Fast Cross-Validation via Sequential Testing</title>
    <dc:date>2012-06-12T14:09:59+00:00</dc:date>
    <link>http://arxiv.org/abs/1206.2248</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["With the increasing size of today's data sets, finding the right parameter configuration in model selection via cross-validation can be an extremely time-consuming task. In this paper we propose an improved cross-validation procedure which uses non-parametric testing coupled with sequential analysis to determine the best parameter set on linearly increasing subsets of the data. By eliminating underperforming candidates quickly and keeping promising candidates as long as possible, the method speeds up the computation while preserving the capability of the full cross-validation. Theoretical considerations underline the statistical power of our procedure. The experimental evaluation shows that our method reduces the computation time by a factor of up to 120 compared to a full cross-validation with a negligible impact on the accuracy."

to_teach tags provisional]]></description>
<dc:subject>to_read cross-validation computational_statistics statistics to_teach:data-mining to_teach:undergrad-ADA model_checking in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:9c390e4ea064/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:computational_statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:undergrad-ADA"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:model_checking"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
</rdf:RDF>