<?xml version="1.0" encoding="UTF-8"?>
 <rdf:RDF xmlns="http://purl.org/rss/1.0/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://web.resource.org/cc/" xmlns:syn="http://purl.org/rss/1.0/modules/syndication/" xmlns:admin="http://webns.net/mvcb/">
  <channel rdf:about="http://pinboard.in">
    <title>Pinboard (cshalizi)</title>
    <link>https://pinboard.in/u:cshalizi/public/</link>
    <description>recent bookmarks from cshalizi</description>
    <items>
      <rdf:Seq>	<rdf:li rdf:resource="https://arxiv.org/abs/2601.05444"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2512.01819"/>
	<rdf:li rdf:resource="https://openreview.net/forum?id=skdlnUYRzQ"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2402.01502"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2301.07015"/>
	<rdf:li rdf:resource="https://joss.theoj.org/papers/10.21105/joss.02232"/>
	<rdf:li rdf:resource="https://link.springer.com/article/10.1007/s10618-022-00907-3"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2207.08815"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2206.04661"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2104.13881"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2101.11190"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2012.14563"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/2012.12802"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1902.03999"/>
	<rdf:li rdf:resource="https://mitpress.mit.edu/books/classification-wild"/>
	<rdf:li rdf:resource="https://www.jstatsoft.org/article/view/v054i02"/>
	<rdf:li rdf:resource="https://link.springer.com/article/10.1023%2FA%3A1010933404324"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1911.03054"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1911.00190"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1806.03467"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1909.11799"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1906.10086"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1908.06852"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1807.11408"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1906.07177"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1904.07830"/>
	<rdf:li rdf:resource="http://journal.sjdm.org/17/17217/jdm17217.html"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1311.4555"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1311.6392"/>
	<rdf:li rdf:resource="https://arxiv.org/abs/1202.1561"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1510.04342"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1504.01132"/>
	<rdf:li rdf:resource="http://arstechnica.co.uk/security/2016/02/the-nsas-skynet-program-may-be-killing-thousands-of-innocent-people/"/>
	<rdf:li rdf:resource="https://86f47fe78a5d63da83a76fa1cc0162ae153f55d8.googledrive.com/secure/AB5fxWAw7W8ON_fg5_i2BCNy2LshJNVmE4EMbqSNR5OWGqBwte0kTnus963Bif38YUWr4YlEDexVD9nKw03gVJkD9tkUQwHDZ2Sig-JvNGYxMP7DQp-IAFAzGrOlmbfmQHaS-0OQgvfEcku8_ZFRNLNowdtKDAAJLP-QSTN6s1MOggTcGfTOBgvoSSO3VT4DXrauXYk0w39lek68H_gn9Ugl93SXTKG8fEZ5nyty3TGbcouXclbDxbZS4aOTo0V9tVFKCtLEPo8F54aPqghEGNwqeowM0pakzi7grg3LD3W6Rf6I9zHwTxh3G8cO0eBFZefO2eUhuY9qenswApUYKs3fIr3kKTj27ryP5s0iBQcy1AheJM-uXQGbxjz6jBFHAOuGk99SkY8iAAB2VBSc_ufumkoaaRSRJSR-GOTt-j04L4rpZV7ozIUh6CW4UUVQUUjMe0anAQdhYt8ZsXXDE60nUI38QqDyQtH9ltY1JGGnlkmptReKewJm14wZWV-LPrHZYWnivrqrL9-hf5DAZ-DgF0zPwWWikIgB5sZnKvRsq0qvpfezAWGnpZ-MvWKUI93eBKIK_3zd/host/0Byvk0A8Ic21Yck9ScVUzaEVsNG8/#/"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1405.0352"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1405.1533"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1409.2090"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1405.2881"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1406.1845"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1402.4293"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1310.5677"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1309.7733"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1310.1415"/>
	<rdf:li rdf:resource="http://www.mitpressjournals.org/doi/abs/10.1162/NECO_a_00464"/>
	<rdf:li rdf:resource="http://projecteuclid.org/DPubS?service=UI&amp;version=1.0&amp;verb=Display&amp;handle=euclid.aoas/1365527208"/>
	<rdf:li rdf:resource="http://www.tandfonline.com/doi/abs/10.1080/10618600.2012.657132"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1302.4853"/>
	<rdf:li rdf:resource="http://link.springer.com/article/10.1007/s10618-012-0261-2"/>
	<rdf:li rdf:resource="http://biostats.bepress.com/uncbiostat/art33/"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1206.4620"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1205.2609"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1202.1523"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/1202.1561"/>
	<rdf:li rdf:resource="http://www.springerlink.com/content/ng44781g47736260/"/>
	<rdf:li rdf:resource="http://www.cs.cornell.edu/~daria/papers/Groves.pdf"/>
	<rdf:li rdf:resource="http://www.cs.cornell.edu/~daria/papers/Interactions.pdf"/>
	<rdf:li rdf:resource="http://flowingdata.com/2008/04/23/showing-the-obama-clinton-divide-in-decision-tree-infographic/"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/0804.0650"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/0711.2434"/>
	<rdf:li rdf:resource="http://arxiv.org/abs/0708.1820"/>
      </rdf:Seq>
    </items>
  </channel><item rdf:about="https://arxiv.org/abs/2601.05444">
    <title>[2601.05444] What Functions Does XGBoost Learn?</title>
    <dc:date>2026-06-04T18:10:45+00:00</dc:date>
    <link>https://arxiv.org/abs/2601.05444</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This paper establishes a rigorous theoretical foundation for the function class implicitly learned by XGBoost, bridging the gap between its empirical success and our theoretical understanding. We introduce an infinite-dimensional function class d,s∞−ST that extends finite ensembles of bounded-depth regression trees, together with a complexity measure Vd,s∞−XGB(⋅) that generalizes the L1 regularization penalty used in XGBoost. We show that every optimizer of the XGBoost objective is also an optimizer of an equivalent penalized regression problem over d,s∞−ST with penalty Vd,s∞−XGB(⋅), providing an interpretation of XGBoost as implicitly targeting a broader function class. We also develop a smoothness-based interpretation of d,s∞−ST and Vd,s∞−XGB(⋅) in terms of Hardy--Krause variation. We prove that the least squares estimator over {f∈d,s∞−ST:Vd,s∞−XGB(f)≤V} achieves a nearly minimax-optimal rate of convergence n−2/3(logn)4(min(s,d)−1)/3, thereby avoiding the curse of dimensionality. Our results provide the first rigorous characterization of the function space underlying XGBoost, clarify its connection to classical notions of variation, and identify an important open problem: whether the XGBoost algorithm itself achieves minimax optimality over this class."]]></description>
<dc:subject>to:NB functional_analysis boosting ensemble_methods decision_trees via:msw</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:85d5009a07b7/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:functional_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:boosting"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:via:msw"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2512.01819">
    <title>[2512.01819] Decision Tree Embedding by Leaf-Means</title>
    <dc:date>2025-12-06T14:30:34+00:00</dc:date>
    <link>https://arxiv.org/abs/2512.01819</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Decision trees and random forest remain highly competitive for classification on medium-sized, standard datasets due to their robustness, minimal preprocessing requirements, and interpretability. However, a single tree suffers from high estimation variance, while large ensembles reduce this variance at the cost of substantial computational overhead and diminished interpretability. In this paper, we propose Decision Tree Embedding (DTE), a fast and effective method that leverages the leaf partitions of a trained classification tree to construct an interpretable feature representation. By using the sample means within each leaf region as anchor points, DTE maps inputs into an embedding space defined by the tree's partition structure, effectively circumventing the high variance inherent in decision-tree splitting rules. We further introduce an ensemble extension based on additional bootstrap trees, and pair the resulting embedding with linear discriminant analysis for classification. We establish several population-level theoretical properties of DTE, including its preservation of conditional density under mild conditions and a characterization of the resulting classification error. Empirical studies on synthetic and real datasets demonstrate that DTE strikes a strong balance between accuracy and computational efficiency, outperforming or matching random forest and shallow neural networks while requiring only a fraction of their training time in most cases. Overall, the proposed DTE method can be viewed either as a scalable decision tree classifier that improves upon standard split rules, or as a neural network model whose weights are learned from tree-derived anchor points, achieving an intriguing integration of both paradigms."]]></description>
<dc:subject>to:NB decision_trees neural_networks classifiers priebe.carey_e.</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:a1ccf503ec38/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:neural_networks"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:priebe.carey_e."/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://openreview.net/forum?id=skdlnUYRzQ">
    <title>Learning Accurate and Interpretable Decision Trees | OpenReview</title>
    <dc:date>2024-11-02T19:51:04+00:00</dc:date>
    <link>https://openreview.net/forum?id=skdlnUYRzQ</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Decision trees are a popular tool in machine learning and yield easy-to-understand models. Several techniques have been proposed in the literature for learning a decision tree classifier, with different techniques working well for data from different domains. In this work, we develop approaches to design decision tree learning algorithms given repeated access to data from the same domain. We propose novel parameterized classes of node splitting criteria in top-down algorithms, which interpolate between popularly used entropy and Gini impurity based criteria, and provide theoretical bounds on the number of samples needed to learn the splitting function appropriate for the data at hand. We also study the sample complexity of tuning prior parameters in Bayesian decision tree learning, and extend our results to decision tree regression. We further consider the problem of tuning hyperparameters in pruning the decision tree for classical pruning algorithms including min-cost complexity pruning. We also study the interpretability of the learned decision trees and introduce a data-driven approach for optimizing the explainability versus accuracy trade-off using decision trees. Finally, we demonstrate the significance of our approach on real world datasets by learning data-specific decision trees which are simultaneously more accurate and interpretable."]]></description>
<dc:subject>in_NB to_read decision_trees missing_the_talk</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:e2dda7b0a0e7/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:missing_the_talk"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2402.01502">
    <title>[2402.01502] Why do Random Forests Work? Understanding Tree Ensembles as Self-Regularizing Adaptive Smoothers</title>
    <dc:date>2024-02-27T19:59:40+00:00</dc:date>
    <link>https://arxiv.org/abs/2402.01502</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Despite their remarkable effectiveness and broad application, the drivers of success underlying ensembles of trees are still not fully understood. In this paper, we highlight how interpreting tree ensembles as adaptive and self-regularizing smoothers can provide new intuition and deeper insight to this topic. We use this perspective to show that, when studied as smoothers, randomized tree ensembles not only make predictions that are quantifiably more smooth than the predictions of the individual trees they consist of, but also further regulate their smoothness at test-time based on the dissimilarity between testing and training inputs. First, we use this insight to revisit, refine and reconcile two recent explanations of forest success by providing a new way of quantifying the conjectured behaviors of tree ensembles objectively by measuring the effective degree of smoothing they imply. Then, we move beyond existing explanations for the mechanisms by which tree ensembles improve upon individual trees and challenge the popular wisdom that the superior performance of forests should be understood as a consequence of variance reduction alone. We argue that the current high-level dichotomy into bias- and variance-reduction prevalent in statistics is insufficient to understand tree ensembles -- because the prevailing definition of bias does not capture differences in the expressivity of the hypothesis classes formed by trees and forests. Instead, we show that forests can improve upon trees by three distinct mechanisms that are usually implicitly entangled. In particular, we demonstrate that the smoothing effect of ensembling can reduce variance in predictions due to noise in outcome generation, reduce variability in the quality of the learned function given fixed input data and reduce potential bias in learnable functions by enriching the available hypothesis space."]]></description>
<dc:subject>to_read ensemble_methods random_forests decision_trees learning_theory in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:afc37c93e049/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:learning_theory"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2301.07015">
    <title>[2301.07015] Simplistic Collection and Labeling Practices Limit the Utility of Benchmark Datasets for Twitter Bot Detection</title>
    <dc:date>2023-05-01T20:37:24+00:00</dc:date>
    <link>https://arxiv.org/abs/2301.07015</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Accurate bot detection is necessary for the safety and integrity of online platforms. It is also crucial for research on the influence of bots in elections, the spread of misinformation, and financial market manipulation. Platforms deploy infrastructure to flag or remove automated accounts, but their tools and data are not publicly available. Thus, the public must rely on third-party bot detection. These tools employ machine learning and often achieve near perfect performance for classification on existing datasets, suggesting bot detection is accurate, reliable and fit for use in downstream applications. We provide evidence that this is not the case and show that high performance is attributable to limitations in dataset collection and labeling rather than sophistication of the tools. Specifically, we show that simple decision rules -- shallow decision trees trained on a small number of features -- achieve near-state-of-the-art performance on most available datasets and that bot detection datasets, even when combined together, do not generalize well to out-of-sample datasets. Our findings reveal that predictions are highly dependent on each dataset's collection and labeling procedures rather than fundamental differences between bots and humans. These results have important implications for both transparency in sampling and labeling procedures and potential biases in research using existing bot detection tools for pre-processing."]]></description>
<dc:subject>to:NB classifiers networked_life deceiving_us_has_become_an_industrial_process decision_trees to_teach:data-mining philip_k_dick_and_the_fake_humans_rules_everything_around_me</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:26709234aea1/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:networked_life"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:deceiving_us_has_become_an_industrial_process"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:philip_k_dick_and_the_fake_humans_rules_everything_around_me"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://joss.theoj.org/papers/10.21105/joss.02232">
    <title>Journal of Open Source Software: policytree: Policy learning via doubly robust empirical welfare maximization over trees</title>
    <dc:date>2023-04-27T14:40:08+00:00</dc:date>
    <link>https://joss.theoj.org/papers/10.21105/joss.02232</link>
    <dc:creator>cshalizi</dc:creator><dc:subject>to:NB decision_trees athey.susan wager.stefan statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:364aa068e29b/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:athey.susan"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:wager.stefan"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://link.springer.com/article/10.1007/s10618-022-00907-3">
    <title>Approximation trees: statistical reproducibility in model distillation | SpringerLink</title>
    <dc:date>2023-01-17T06:19:08+00:00</dc:date>
    <link>https://link.springer.com/article/10.1007/s10618-022-00907-3</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This paper examines the reproducibility of learned explanations for black-box predictions via model distillation using classification trees. We find that common tree distillation methods fail to reproduce a single stable explanation when applied to the same teacher model due the randomness of the distillation process. We study this issue of reliable interpretation and propose a standardized framework for tree distillation to achieve reproducibility. The proposed framework consists of (1) a statistical test to stabilize tree splits, and (2) a stopping rule for tree building when using a teacher that provides an estimate of the uncertainty of its predictions, e.g. random forests. We demonstrate the empirical performance of the proposed distillation method on a variety of synthetic and real-world datasets."]]></description>
<dc:subject>to:NB decision_trees data_mining hooker.giles</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:2edc09ab8e94/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:data_mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hooker.giles"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2207.08815">
    <title>[2207.08815] Why do tree-based models still outperform deep learning on tabular data?</title>
    <dc:date>2022-08-25T16:03:35+00:00</dc:date>
    <link>https://arxiv.org/abs/2207.08815</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["While deep learning has enabled tremendous progress on text and image datasets, its superiority on tabular data is not clear. We contribute extensive benchmarks of standard and novel deep learning methods as well as tree-based models such as XGBoost and Random Forests, across a large number of datasets and hyperparameter combinations. We define a standard set of 45 datasets from varied domains with clear characteristics of tabular data and a benchmarking methodology accounting for both fitting models and finding good hyperparameters. Results show that tree-based models remain state-of-the-art on medium-sized data (∼10K samples) even without accounting for their superior speed. To understand this gap, we conduct an empirical investigation into the differing inductive biases of tree-based models and Neural Networks (NNs). This leads to a series of challenges which should guide researchers aiming to build tabular-specific NNs: 1. be robust to uninformative features, 2. preserve the orientation of the data, and 3. be able to easily learn irregular functions. To stimulate research on tabular architectures, we contribute a standard benchmark and raw data for baselines: every point of a 20 000 compute hours hyperparameter search for each learner."]]></description>
<dc:subject>to:NB to_read your_favorite_deep_neural_network_sucks ensemble_methods decision_trees to_teach:data-mining</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:2b5e2a0c03ab/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:your_favorite_deep_neural_network_sucks"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2206.04661">
    <title>[2206.04661] Distillation Decision Tree</title>
    <dc:date>2022-06-10T14:08:46+00:00</dc:date>
    <link>https://arxiv.org/abs/2206.04661</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Black-box machine learning models are criticized as lacking interpretability, although they tend to have good prediction accuracy. Knowledge Distillation (KD) is an emerging tool to interpret the black-box model by distilling its knowledge into a transparent model. With well-known advantages in interpretation, decision tree is a competitive candidate of the transparent model. However, theoretical or empirical understanding for the decision tree generated from KD process is limited. In this paper, we name this kind of decision tree the distillation decision tree (DDT) and lay the theoretical foundations for tree structure stability which determines the validity of DDT's interpretation. We prove that the structure of DDT can achieve stable (convergence) under some mild assumptions. Meanwhile, we develop algorithms for stabilizing the induction of DDT, propose parallel strategies for improving algorithm's computational efficiency, and introduce a marginal principal component analysis method for overcoming the curse of dimensionality in sampling. Simulated and real data studies justify our theoretical results, validate the efficacy of algorithms, and demonstrate that DDT can strike a good balance between model's prediction accuracy and interpretability."]]></description>
<dc:subject>to:NB decision_trees neural_networks</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:2a7e816e9336/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:neural_networks"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2104.13881">
    <title>[2104.13881] Universal Consistency of Decision Trees for High Dimensional Additive Models</title>
    <dc:date>2021-04-29T03:30:43+00:00</dc:date>
    <link>https://arxiv.org/abs/2104.13881</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This paper shows that decision trees constructed with Classification and Regression Trees (CART) methodology are universally consistent for additive models, even when the dimensionality scales exponentially with the sample size, under certain ℓ1 sparsity constraints. The consistency is universal in the sense that there are no a priori assumptions on the distribution of the input variables. Surprisingly, this adaptivity to (approximate or exact) sparsity is achieved with a single tree, as opposed to what might be expected for an ensemble. Finally, we show that these qualitative properties of individual trees are inherited by Breiman's random forests. A key step in the analysis is the establishment of an oracle inequality, which precisely characterizes the goodness-of-fit and complexity tradeoff."]]></description>
<dc:subject>decision_trees additive_models in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:9dcb98265561/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:additive_models"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2101.11190">
    <title>[2101.11190] Boost-S: Gradient Boosted Trees for Spatial Data and Its Application to FDG-PET Imaging Data</title>
    <dc:date>2021-02-05T20:12:15+00:00</dc:date>
    <link>https://arxiv.org/abs/2101.11190</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Boosting Trees are one of the most successful statistical learning approaches that involve sequentially growing an ensemble of simple regression trees (i.e., "weak learners"). However, gradient boosted trees are not yet available for spatially correlated data. This paper proposes a new gradient Boosted Trees algorithm for Spatial Data (Boost-S) with covariate information. Boost-S integrates the spatial correlation structure into the classical framework of gradient boosted trees. Each tree is grown by solving a regularized optimization problem, where the objective function involves two penalty terms on tree complexity and takes into account the underlying spatial correlation. A computationally-efficient algorithm is proposed to obtain the ensemble trees. The proposed Boost-S is applied to the spatially-correlated FDG-PET (fluorodeoxyglucose-positron emission tomography) imaging data collected during cancer chemoradiotherapy. Our numerical investigations successfully demonstrate the advantages of the proposed Boost-S over existing approaches for this particular application."]]></description>
<dc:subject>to:NB ensemble_methods boosting decision_trees spatial_statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:9fdb3ea321fd/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:boosting"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:spatial_statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2012.14563">
    <title>[2012.14563] Random Planted Forest: a directly interpretable tree ensemble</title>
    <dc:date>2021-01-03T20:06:28+00:00</dc:date>
    <link>https://arxiv.org/abs/2012.14563</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We introduce a novel interpretable and tree-based algorithm for prediction in a regression setting in which each tree in a classical random forest is replaced by a family of planted trees that grow simultaneously. The motivation for our algorithm is to estimate the unknown regression function from a functional ANOVA decomposition perspective, where each tree corresponds to a function within that decomposition. Therefore, planted trees are limited in the number of interaction terms. The maximal order of approximation in the ANOVA decomposition can be specified or left unlimited. If a first order approximation is chosen, the result is an additive model. In the other extreme case, if the order of approximation is not limited, the resulting model puts no restrictions on the form of the regression function. In a simulation study we find encouraging prediction and visualisation properties of our random planted forest method. We also develop theory for an idealised version of random planted forests in the case of an underlying additive model. We show that in the additive case, the idealised version achieves up to a logarithmic factor asymptotically optimal one-dimensional convergence rates of order n−2/5."]]></description>
<dc:subject>to:NB regression nonparametrics ensemble_methods decision_trees to_teach:data-mining</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:f6e4ba73e30f/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:nonparametrics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/2012.12802">
    <title>[2012.12802] Machine Learning Advances for Time Series Forecasting</title>
    <dc:date>2020-12-24T15:33:23+00:00</dc:date>
    <link>https://arxiv.org/abs/2012.12802</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["In this paper we survey the most recent advances in supervised machine learning and high-dimensional models for time series forecasting. We consider both linear and nonlinear alternatives. Among the linear methods we pay special attention to penalized regressions and ensemble of models. The nonlinear methods considered in the paper include shallow and deep neural networks, in their feed-forward and recurrent versions, and tree-based methods, such as random forests and boosted trees. We also consider ensemble and hybrid models by combining ingredients from different alternatives. Tests for superior predictive ability are briefly reviewed. Finally, we discuss application of machine learning in economics and finance and provide an illustration with high-frequency financial data."]]></description>
<dc:subject>to:NB time_series prediction data_mining decision_trees random_forests neural_networks ensemble_methods to_teach:data_over_space_and_time</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:30ad1b56b9ba/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:time_series"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:data_mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:neural_networks"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data_over_space_and_time"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1902.03999">
    <title>[1902.03999] KTBoost: Combined Kernel and Tree Boosting</title>
    <dc:date>2020-11-19T19:46:20+00:00</dc:date>
    <link>https://arxiv.org/abs/1902.03999</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We introduce a novel boosting algorithm called `KTBoost' which combines kernel boosting and tree boosting. In each boosting iteration, the algorithm adds either a regression tree or reproducing kernel Hilbert space (RKHS) regression function to the ensemble of base learners. Intuitively, the idea is that discontinuous trees and continuous RKHS regression functions complement each other, and that this combination allows for better learning of functions that have parts with varying degrees of regularity such as discontinuities and smooth parts. We empirically show that KTBoost significantly outperforms both tree and kernel boosting in terms of predictive accuracy in a comparison on a wide array of data sets."]]></description>
<dc:subject>to:NB ensemble_methods boosting kernel_methods decision_trees to_teach:data-mining</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:8dff463b6f9b/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:boosting"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:kernel_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://mitpress.mit.edu/books/classification-wild">
    <title>Classification in the Wild | The MIT Press</title>
    <dc:date>2020-09-21T04:00:00+00:00</dc:date>
    <link>https://mitpress.mit.edu/books/classification-wild</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This book focuses on classification—allocating objects into categories—“in the wild,” in real-world situations and far from the certainty of the lab. In the wild, unlike in typical psychological experiments, the future is not knowable and uncertainty cannot be meaningfully reduced to probability. Connecting the science of heuristics with machine learning, the book shows how to create formal models using classification rules that are simple, fast, and transparent and that can be as accurate as mathematically sophisticated algorithms developed for machine learning.
"The authors—whose individual expertise ranges from empirical psychology to mathematical modeling to artificial intelligence and data science—offer real-world examples, including voting, HIV screening, and magistrate decision making; present an accessible guide to inducing the models statistically; compare the performance of such models to machine learning algorithms when applied to problems that include predicting diabetes or bank failure; and discuss conceptual and historical connections to cognitive psychology. Finally, they analyze such challenging safety-related applications as decreasing civilian casualties in checkpoints and regulating investment banks."

--- Electronic version: https://doi.org/10.7551/mitpress/11790.001.0001
--- ETA after reading the introduction and chapter 1: I am not happy with their example of judges overwhelmingly agreeing with prosecutors.  This _could_ be, as they say, because the judges are applying a fast and frugal heuristic, to the detriment of the accused.  But, _as far as their data goes_ it could _also_ be the case that (i) judges are vigorous champions of civil liberties and the rights of the accused, and skeptical of the executive branch in the form of the prosecutors, and (ii) prosecutors (unlike defense attorneys) have discretion over which cases to take up, and it's really bad for prosecutors if they loose cases.  Smart prosecutors will then overwhelmingly bring cases they know they win in front of even the most hard-nosed judge, so judges will overwhelmingly agree with prosectuors.  Whether this points to a more general flaw with the approach in cases of strategic interaction, I'm not sure, but it makes me uneasy.]]></description>
<dc:subject>books:noted gigerenzer.gerd decision-making decision_trees heuristics in_NB downloaded</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:be88add55585/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:books:noted"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:gigerenzer.gerd"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision-making"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:heuristics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:downloaded"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://www.jstatsoft.org/article/view/v054i02">
    <title>adabag: An R Package for Classification with Boosting and Bagging | Alfaro | Journal of Statistical Software</title>
    <dc:date>2019-12-01T15:47:34+00:00</dc:date>
    <link>https://www.jstatsoft.org/article/view/v054i02</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Boosting and bagging are two widely used ensemble methods for classification. Their common goal is to improve the accuracy of a classifier combining single classifiers which are slightly better than random guessing. Among the family of boosting algorithms, AdaBoost (adaptive boosting) is the best known, although it is suitable only for dichotomous tasks. AdaBoost.M1 and SAMME (stagewise additive modeling using a multi-class exponential loss function) are two easy and natural extensions to the general case of two or more classes. In this paper, the adabag R package is introduced. This version implements AdaBoost.M1, SAMME and bagging algorithms with classification trees as base classifiers. Once the ensembles have been trained, they can be used to predict the class of new samples. The accuracy of these classifiers can be estimated in a separated data set or through cross validation. Moreover, the evolution of the error as the ensemble grows can be analysed and the ensemble can be pruned. In addition, the margin in the class prediction and the probability of each class for the observations can be calculated. Finally, several classic examples in classification literature are shown to illustrate the use of this package."]]></description>
<dc:subject>to:NB boosting bagging ensemble_methods classifiers decision_trees R to_teach:data-mining</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:af00024c2969/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:boosting"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:bagging"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:R"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://link.springer.com/article/10.1023%2FA%3A1010933404324">
    <title>Random Forests | SpringerLink</title>
    <dc:date>2019-11-25T15:59:39+00:00</dc:date>
    <link>https://link.springer.com/article/10.1023%2FA%3A1010933404324</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Random forests are a combination of tree predictors such that each tree depends on the values of a random vector sampled independently and with the same distribution for all trees in the forest. The generalization error for forests converges a.s. to a limit as the number of trees in the forest becomes large. The generalization error of a forest of tree classifiers depends on the strength of the individual trees in the forest and the correlation between them. Using a random selection of features to split each node yields error rates that compare favorably to Adaboost (Y. Freund & R. Schapire, Machine Learning: Proceedings of the Thirteenth International conference, ***, 148–156), but are more robust with respect to noise. Internal estimates monitor error, strength, and correlation and these are used to show the response to increasing the number of features used in the splitting. Internal estimates are also used to measure variable importance. These ideas are also applicable to regression."]]></description>
<dc:subject>have_read breiman.leo ensemble_methods decision_trees random_forests to_teach:data-mining machine_learning statistics prediction in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:11aee8d7c62c/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:breiman.leo"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1911.03054">
    <title>[1911.03054] An Experimental Comparison of Old and New Decision Tree Algorithms</title>
    <dc:date>2019-11-11T20:46:39+00:00</dc:date>
    <link>https://arxiv.org/abs/1911.03054</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This paper presents a detailed comparison of a recently proposed algorithm for optimizing decision trees, tree alternating optimization (TAO), with other popular, established algorithms, such as CART and C5.0. We compare their performance on a number of datasets of different size, dimensionality and number of classes, across different performance factors: accuracy and tree size (in terms of the number of leaves or the depth of the tree). We find that TAO achieves higher accuracy in every single dataset, often by a large margin."]]></description>
<dc:subject>decision_trees color_me_skeptical in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:93442542625d/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:color_me_skeptical"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1911.00190">
    <title>[1911.00190] Randomization as Regularization: A Degrees of Freedom Explanation for Random Forest Success</title>
    <dc:date>2019-11-10T22:34:10+00:00</dc:date>
    <link>https://arxiv.org/abs/1911.00190</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Random forests remain among the most popular off-the-shelf supervised machine learning tools with a well-established track record of predictive accuracy in both regression and classification settings. Despite their empirical success as well as a bevy of recent work investigating their statistical properties, a full and satisfying explanation for their success has yet to be put forth. Here we aim to take a step forward in this direction by demonstrating that the additional randomness injected into individual trees serves as a form of implicit regularization, making random forests an ideal model in low signal-to-noise ratio (SNR) settings. Specifically, from a model-complexity perspective, we show that the mtry parameter in random forests serves much the same purpose as the shrinkage penalty in explicitly regularized regression procedures like lasso and ridge regression. To highlight this point, we design a randomized linear-model-based forward selection procedure intended as an analogue to tree-based random forests and demonstrate its surprisingly strong empirical performance. Numerous demonstrations on both real and synthetic data are provided."]]></description>
<dc:subject>to:NB random_forests statistics decision_trees ensemble_methods</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:cf2088f4c949/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1806.03467">
    <title>[1806.03467] Orthogonal Random Forest for Causal Inference</title>
    <dc:date>2019-10-01T16:11:28+00:00</dc:date>
    <link>https://arxiv.org/abs/1806.03467</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We propose the orthogonal random forest, an algorithm that combines Neyman-orthogonality to reduce sensitivity with respect to estimation error of nuisance parameters with generalized random forests (Athey et al., 2017)--a flexible non-parametric method for statistical estimation of conditional moment models using random forests. We provide a consistency rate and establish asymptotic normality for our estimator. We show that under mild assumptions on the consistency rate of the nuisance estimator, we can achieve the same error rate as an oracle with a priori knowledge of these nuisance parameters. We show that when the nuisance functions have a locally sparse parametrization, then a local ℓ1-penalized regression achieves the required rate. We apply our method to estimate heterogeneous treatment effects from observational data with discrete treatments or continuous treatments, and we show that, unlike prior work, our method provably allows to control for a high-dimensional set of variables under standard sparsity conditions. We also provide a comprehensive empirical evaluation of our algorithm on both synthetic and real data."]]></description>
<dc:subject>to:NB decision_trees ensemble_methods regression causal_inference statistics nonparametrics random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:64b039275305/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:causal_inference"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:nonparametrics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1909.11799">
    <title>[1909.11799] Manifold Forests: Closing the Gap on Neural Networks</title>
    <dc:date>2019-10-01T13:53:57+00:00</dc:date>
    <link>https://arxiv.org/abs/1909.11799</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Decision forests (DF), in particular random forests and gradient boosting trees, have demonstrated state-of-the-art accuracy compared to other methods in many supervised learning scenarios. In particular, DFs dominate other methods in tabular data, that is, when the feature space is unstructured, so that the signal is invariant to permuting feature indices. However, in structured data lying on a manifold---such as images, text, and speech---neural nets (NN) tend to outperform DFs. We conjecture that at least part of the reason for this is that the input to NN is not simply the feature magnitudes, but also their indices (for example, the convolution operation uses "feature locality"). In contrast, naïve DF implementations fail to explicitly consider feature indices. A recently proposed DF approach demonstrates that DFs, for each node, implicitly sample a random matrix from some specific distribution. Here, we build on that to show that one can choose distributions in a \emph{manifold aware fashion}. For example, for image classification, rather than randomly selecting pixels, one can randomly select contiguous patches. We demonstrate the empirical performance of data living on three different manifolds: images, time-series, and a torus. In all three cases, our Manifold Forest (\Mf) algorithm empirically dominates other state-of-the-art approaches that ignore feature space structure, achieving a lower classification error on all sample sizes. This dominance extends to the MNIST data set as well. Moreover, both training and test time is significantly faster for manifold forests as compared to deep nets. This approach, therefore, has promise to enable DFs and other machine learning methods to close the gap with deep nets on manifold-valued data."]]></description>
<dc:subject>to:NB decision_trees ensemble_methods to_read random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:9e7ddf9742fb/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1906.10086">
    <title>[1906.10086] Analyzing CART</title>
    <dc:date>2019-09-04T19:48:03+00:00</dc:date>
    <link>https://arxiv.org/abs/1906.10086</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Decision trees with binary splits are popularly constructed using Classification and Regression Trees (CART) methodology. For binary classification and regression models, this approach recursively divides the data into two near-homogenous daughter nodes according to a split point that maximizes the reduction in sum of squares error (the impurity) along a particular variable. This paper aims to study the bias and adaptive properties of regression trees constructed with CART. In doing so, we derive an interesting connection between the bias and the mean decrease in impurity (MDI) measure of variable importance---a tool widely used for model interpretability---defined as the sum of impurity reductions over all non-terminal nodes in the tree. In particular, we show that the probability content of a terminal subnode for a variable is small when the MDI for that variable is large and that this relationship is exponential---confirming theoretically that decision trees with CART have small bias and are adaptive to signal strength and direction. Finally, we apply these individual tree bounds to tree ensembles and show consistency of Breiman's random forests. The context is surprisingly general and applies to a wide variety of multivariable data generating distributions and regression functions. The main technical tool is an exact characterization of the conditional probability content of the daughter nodes arising from an optimal split, in terms of the partial dependence function and reduction in impurity."]]></description>
<dc:subject>statistics decision_trees cart in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:63bdf2a831c6/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cart"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1908.06852">
    <title>[1908.06852] SIRUS: making random forests interpretable</title>
    <dc:date>2019-08-20T14:22:39+00:00</dc:date>
    <link>https://arxiv.org/abs/1908.06852</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["State-of-the-art learning algorithms, such as random forests or neural networks, are often qualified as "black-boxes" because of the high number and complexity of operations involved in their prediction mechanism. This lack of interpretability is a strong limitation for applications involving critical decisions, typically the analysis of production processes in the manufacturing industry. In such critical contexts, models have to be interpretable, i.e., simple, stable, and predictive. To address this issue, we design SIRUS (Stable and In-terpretable RUle Set), a new classification algorithm based on random forests, which takes the form of a short list of rules. While simple models are usually unstable with respect to data perturbation, SIRUS achieves a remarkable stability improvement over cutting-edge methods. Furthermore, SIRUS inherits a predictive accuracy close to random forests, combined with the simplicity of decision trees. These properties are assessed both from a theoretical and empirical point of view, through extensive numerical experiments based on our R/C++ software implementation sirus."

--- Not sure that there's really much new here, beyond limiting the forest to very shallow trees.]]></description>
<dc:subject>to:NB classifiers ensemble_methods random_forests decision_trees data_mining statistics to_teach:data-mining have_skimmed</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:e72b24d4a589/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:data_mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_skimmed"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1807.11408">
    <title>[1807.11408] Local Linear Forests</title>
    <dc:date>2019-06-23T17:33:39+00:00</dc:date>
    <link>https://arxiv.org/abs/1807.11408</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Random forests are a powerful method for non-parametric regression, but are limited in their ability to fit smooth signals, and can show poor predictive performance in the presence of strong, smooth effects. Taking the perspective of random forests as an adaptive kernel method, we pair the forest kernel with a local linear regression adjustment to better capture smoothness. The resulting procedure, local linear forests, enables us to improve on asymptotic rates of convergence for random forests with smooth signals, and provides substantial gains in accuracy on both real and simulated data. We prove a central limit theorem valid under regularity conditions on the forest and smoothness constraints, and propose a computationally efficient construction for confidence intervals. Moving to a causal inference application, we discuss the merits of local regression adjustments for heterogeneous treatment effect estimation, and give an example on a dataset exploring the effect word choice has on attitudes to the social safety net. Last, we include simulation results on real and generated data."]]></description>
<dc:subject>to:NB linear_regression ensemble_methods decision_trees athey.susan statistics random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:e4550a9d18a1/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:linear_regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:athey.susan"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1906.07177">
    <title>[1906.07177] (f)RFCDE: Random Forests for Conditional Density Estimation and Functional Data</title>
    <dc:date>2019-06-19T15:33:10+00:00</dc:date>
    <link>https://arxiv.org/abs/1906.07177</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Random forests is a common non-parametric regression technique which performs well for mixed-type unordered data and irrelevant features, while being robust to monotonic variable transformations. Standard random forests, however, do not efficiently handle functional data and runs into a curse-of dimensionality when presented with high-resolution curves and surfaces. Furthermore, in settings with heteroskedasticity or multimodality, a regression point estimate with standard errors do not fully capture the uncertainty in our predictions. A more informative quantity is the conditional density p(y | x) which describes the full extent of the uncertainty in the response y given covariates x. In this paper we show how random forests can be efficiently leveraged for conditional density estimation, functional covariates, and multiple responses without increasing computational complexity. We provide open-source software for all procedures with R and Python versions that call a common C++ library."]]></description>
<dc:subject>to:NB ensemble_methods regression density_estimation statistics kith_and_kin decision_trees lee.ann_b. random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:7a354f01b8c9/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:density_estimation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:kith_and_kin"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:lee.ann_b."/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1904.07830">
    <title>[1904.07830] Scalable and Efficient Hypothesis Testing with Random Forests</title>
    <dc:date>2019-05-10T03:17:06+00:00</dc:date>
    <link>https://arxiv.org/abs/1904.07830</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Throughout the last decade, random forests have established themselves as among the most accurate and popular supervised learning methods. While their black-box nature has made their mathematical analysis difficult, recent work has established important statistical properties like consistency and asymptotic normality by considering subsampling in lieu of bootstrapping. Though such results open the door to traditional inference procedures, all formal methods suggested thus far place severe restrictions on the testing framework and their computational overhead precludes their practical scientific use. Here we propose a permutation-style testing approach to formally assess feature significance. We establish asymptotic validity of the test via exchangeability arguments and show that the test maintains high power with orders of magnitude fewer computations. As importantly, the procedure scales easily to big data settings where large training and testing sets may be employed without the need to construct additional models. Simulations and applications to ecological data where random forests have recently shown promise are provided."]]></description>
<dc:subject>to:NB ensemble_methods decision_trees classifiers statistics hypothesis_testing random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:bfb279e5da3b/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hypothesis_testing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://journal.sjdm.org/17/17217/jdm17217.html">
    <title>FFTrees: A toolbox to create, visualize, and evaluate fast-and-frugal decision trees</title>
    <dc:date>2017-08-01T14:47:49+00:00</dc:date>
    <link>http://journal.sjdm.org/17/17217/jdm17217.html</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Fast-and-frugal trees (FFTs) are simple algorithms that facilitate efficient and accurate decisions based on limited information. But despite their successful use in many applied domains, there is no widely available toolbox that allows anyone to easily create, visualize, and evaluate FFTs. We fill this gap by introducing the R package FFTrees. In this paper, we explain how FFTs work, introduce a new class of algorithms called fan for constructing FFTs, and provide a tutorial for using the FFTrees package. We then conduct a simulation across ten real-world datasets to test how well FFTs created by FFTrees can predict data. Simulation results show that FFTs created by FFTrees can predict data as well as popular classification algorithms such as regression and random forests, while remaining simple enough for anyone to understand and use."

--- I am skeptical about that "simple enough for anyone to understand and use"]]></description>
<dc:subject>decision_trees heuristics cognitive_science R have_read to_teach:undergrad-ADA re:ADAfaEPoV in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:cc94e08f69e8/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:heuristics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cognitive_science"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:R"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:undergrad-ADA"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:ADAfaEPoV"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1311.4555">
    <title>[1311.4555] Confidence Intervals for Random Forests: The Jackknife and the Infinitesimal Jackknife</title>
    <dc:date>2016-12-04T21:31:22+00:00</dc:date>
    <link>https://arxiv.org/abs/1311.4555</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We study the variability of predictions made by bagged learners and random forests, and show how to estimate standard errors for these methods. Our work builds on variance estimates for bagging proposed by Efron (1992, 2012) that are based on the jackknife and the infinitesimal jackknife (IJ). In practice, bagged predictors are computed using a finite number B of bootstrap replicates, and working with a large B can be computationally expensive. Direct applications of jackknife and IJ estimators to bagging require B on the order of n^{1.5} bootstrap replicates to converge, where n is the size of the training set. We propose improved versions that only require B on the order of n replicates. Moreover, we show that the IJ estimator requires 1.7 times less bootstrap replicates than the jackknife to achieve a given accuracy. Finally, we study the sampling distributions of the jackknife and IJ variance estimates themselves. We illustrate our findings with multiple experiments and simulation studies."]]></description>
<dc:subject>to:NB bootstrap confidence_sets ensemble_methods random_forests decision_trees statistics nonparametrics efron.bradley hastie.trevor</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:7e987d8e279f/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:bootstrap"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:confidence_sets"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:nonparametrics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:efron.bradley"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hastie.trevor"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1311.6392">
    <title>[1311.6392] A Comprehensive Approach to Universal Piecewise Nonlinear Regression Based on Trees</title>
    <dc:date>2016-12-01T20:29:54+00:00</dc:date>
    <link>https://arxiv.org/abs/1311.6392</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["In this paper, we investigate adaptive nonlinear regression and introduce tree based piecewise linear regression algorithms that are highly efficient and provide significantly improved performance with guaranteed upper bounds in an individual sequence manner. We use a tree notion in order to partition the space of regressors in a nested structure. The introduced algorithms adapt not only their regression functions but also the complete tree structure while achieving the performance of the "best" linear mixture of a doubly exponential number of partitions, with a computational complexity only polynomial in the number of nodes of the tree. While constructing these algorithms, we also avoid using any artificial "weighting" of models (with highly data dependent parameters) and, instead, directly minimize the final regression error, which is the ultimate performance goal. The introduced methods are generic such that they can readily incorporate different tree construction methods such as random trees in their framework and can use different regressor or partitioning functions as demonstrated in the paper."]]></description>
<dc:subject>regression decision_trees nonparametrics statistics ensemble_methods color_me_skeptical in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:d14d31d5a173/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:nonparametrics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:color_me_skeptical"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://arxiv.org/abs/1202.1561">
    <title>[1202.1561] Tree models for difference and change detection in a complex environment</title>
    <dc:date>2016-11-30T01:56:46+00:00</dc:date>
    <link>https://arxiv.org/abs/1202.1561</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["A new family of tree models is proposed, which we call "differential trees." A differential tree model is constructed from multiple data sets and aims to detect distributional differences between them. The new methodology differs from the existing difference and change detection techniques in its nonparametric nature, model construction from multiple data sets, and applicability to high-dimensional data. Through a detailed study of an arson case in New Zealand, where an individual is known to have been laying vegetation fires within a certain time period, we illustrate how these models can help detect changes in the frequencies of event occurrences and uncover unusual clusters of events in a complex environment."]]></description>
<dc:subject>decision_trees statistics re:network_differences have_read in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:047cb0ac88d5/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:network_differences"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1510.04342">
    <title>[1510.04342] Estimation and Inference of Heterogeneous Treatment Effects using Random Forests</title>
    <dc:date>2016-03-28T01:03:42+00:00</dc:date>
    <link>http://arxiv.org/abs/1510.04342</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Many scientific and engineering challenges---ranging from personalized medicine to customized marketing recommendations---require an understanding of treatment effect heterogeneity. In this paper, we develop a non-parametric causal forest for estimating heterogeneous treatment effects that extends Breiman's widely used random forest algorithm. Given a potential outcomes framework with unconfoundedness, we show that causal forests are pointwise consistent for the true treatment effect, and have an asymptotically Gaussian and centered sampling distribution. We also discuss a practical method for constructing asymptotic confidence intervals for the true treatment effect that are centered at the causal forest estimates. Our theoretical results rely on a generic Gaussian theory for a large family of random forest algorithms, to our knowledge, this is the first set of results that allows any type of random forest, including classification and regression forests, to be used for provably valid statistical inference. In experiments, we find causal forests to be substantially more powerful than classical methods based on nearest-neighbor matching, especially as the number of covariates increases."]]></description>
<dc:subject>to:NB decision_trees causal_inference statistics regression ensemble_methods athey.susan random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:07956dd36801/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:causal_inference"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:athey.susan"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1504.01132">
    <title>[1504.01132] Recursive Partitioning for Heterogeneous Causal Effects</title>
    <dc:date>2016-03-28T01:02:49+00:00</dc:date>
    <link>http://arxiv.org/abs/1504.01132</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["In this paper we study the problems of estimating heterogeneity in causal effects in experimental or observational studies and conducting inference about the magnitude of the differences in treatment effects across subsets of the population. In applications, our method provides a data-driven approach to determine which subpopulations have large or small treatment effects and to test hypotheses about the differences in these effects. For experiments, our method allows researchers to identify heterogeneity in treatment effects that was not specified in a pre-analysis plan, without concern about invalidating inference due to multiple testing. In most of the literature on supervised machine learning (e.g. regression trees, random forests, LASSO, etc.), the goal is to build a model of the relationship between a unit's attributes and an observed outcome. A prominent role in these methods is played by cross-validation which compares predictions to actual outcomes in test samples, in order to select the level of complexity of the model that provides the best predictive power. Our method is closely related, but it differs in that it is tailored for predicting causal effects of a treatment rather than a unit's outcome. The challenge is that the "ground truth" for a causal effect is not observed for any individual unit: we observe the unit with the treatment, or without the treatment, but not both at the same time. Thus, it is not obvious how to use cross-validation to determine whether a causal effect has been accurately predicted. We propose several novel cross-validation criteria for this problem and demonstrate through simulations the conditions under which they perform better than standard methods for the problem of causal effects. We then apply the method to a large-scale field experiment re-ranking results on a search engine."]]></description>
<dc:subject>to:NB decision_trees regression causal_inference cross-validation statistics athey.susan imbens.guido_w.</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:fe6061dfaa75/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:causal_inference"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:athey.susan"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:imbens.guido_w."/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arstechnica.co.uk/security/2016/02/the-nsas-skynet-program-may-be-killing-thousands-of-innocent-people/">
    <title>The NSA’s SKYNET program may be killing thousands of innocent people | Ars Technica UK</title>
    <dc:date>2016-02-16T17:56:29+00:00</dc:date>
    <link>http://arstechnica.co.uk/security/2016/02/the-nsas-skynet-program-may-be-killing-thousands-of-innocent-people/</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA[We have much to answer for.]]></description>
<dc:subject>the_continuing_crises national_surveillance_state machine_learning classifiers cross-validation bad_data_analysis terrorism_fears drones decision_trees ensemble_methods to_teach:data-mining to:blog random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:1248c0c8cf03/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:the_continuing_crises"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:national_surveillance_state"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cross-validation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:bad_data_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:terrorism_fears"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:drones"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:blog"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="https://86f47fe78a5d63da83a76fa1cc0162ae153f55d8.googledrive.com/secure/AB5fxWAw7W8ON_fg5_i2BCNy2LshJNVmE4EMbqSNR5OWGqBwte0kTnus963Bif38YUWr4YlEDexVD9nKw03gVJkD9tkUQwHDZ2Sig-JvNGYxMP7DQp-IAFAzGrOlmbfmQHaS-0OQgvfEcku8_ZFRNLNowdtKDAAJLP-QSTN6s1MOggTcGfTOBgvoSSO3VT4DXrauXYk0w39lek68H_gn9Ugl93SXTKG8fEZ5nyty3TGbcouXclbDxbZS4aOTo0V9tVFKCtLEPo8F54aPqghEGNwqeowM0pakzi7grg3LD3W6Rf6I9zHwTxh3G8cO0eBFZefO2eUhuY9qenswApUYKs3fIr3kKTj27ryP5s0iBQcy1AheJM-uXQGbxjz6jBFHAOuGk99SkY8iAAB2VBSc_ufumkoaaRSRJSR-GOTt-j04L4rpZV7ozIUh6CW4UUVQUUjMe0anAQdhYt8ZsXXDE60nUI38QqDyQtH9ltY1JGGnlkmptReKewJm14wZWV-LPrHZYWnivrqrL9-hf5DAZ-DgF0zPwWWikIgB5sZnKvRsq0qvpfezAWGnpZ-MvWKUI93eBKIK_3zd/host/0Byvk0A8Ic21Yck9ScVUzaEVsNG8/#/">
    <title>Machine Learning &amp; Non-Linear Models</title>
    <dc:date>2015-10-30T18:17:56+00:00</dc:date>
    <link>https://86f47fe78a5d63da83a76fa1cc0162ae153f55d8.googledrive.com/secure/AB5fxWAw7W8ON_fg5_i2BCNy2LshJNVmE4EMbqSNR5OWGqBwte0kTnus963Bif38YUWr4YlEDexVD9nKw03gVJkD9tkUQwHDZ2Sig-JvNGYxMP7DQp-IAFAzGrOlmbfmQHaS-0OQgvfEcku8_ZFRNLNowdtKDAAJLP-QSTN6s1MOggTcGfTOBgvoSSO3VT4DXrauXYk0w39lek68H_gn9Ugl93SXTKG8fEZ5nyty3TGbcouXclbDxbZS4aOTo0V9tVFKCtLEPo8F54aPqghEGNwqeowM0pakzi7grg3LD3W6Rf6I9zHwTxh3G8cO0eBFZefO2eUhuY9qenswApUYKs3fIr3kKTj27ryP5s0iBQcy1AheJM-uXQGbxjz6jBFHAOuGk99SkY8iAAB2VBSc_ufumkoaaRSRJSR-GOTt-j04L4rpZV7ozIUh6CW4UUVQUUjMe0anAQdhYt8ZsXXDE60nUI38QqDyQtH9ltY1JGGnlkmptReKewJm14wZWV-LPrHZYWnivrqrL9-hf5DAZ-DgF0zPwWWikIgB5sZnKvRsq0qvpfezAWGnpZ-MvWKUI93eBKIK_3zd/host/0Byvk0A8Ic21Yck9ScVUzaEVsNG8/#/</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA[Lecture slides for a computational sociology course.  I hope I may be forgiven a little pride in noting that all of this is material from our undergraduate courses.]]></description>
<dc:subject>regression additive_models classifiers decision_trees statistics ensemble_methods sociology via:kjhealy have_read</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:4c1d43eafccd/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:additive_models"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:sociology"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:via:kjhealy"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1405.0352">
    <title>[1405.0352] Asymptotic Theory for Random Forests</title>
    <dc:date>2015-01-23T05:09:10+00:00</dc:date>
    <link>http://arxiv.org/abs/1405.0352</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Random forests have proven themselves to be reliable predictive algorithms in many application areas. Not much is known, however, about the statistical properties of random forests. Several authors have established conditions under which their predictions are consistent, but these results do not provide practical estimates of the scale of random forest errors. In this paper, we analyze a random forest model based subsampling, and show that random forest predictions are asymptotically normal provided that the subsample size s scales as s(n)/n = o(log(n)^{-d}), where n is the number of training examples and d is the number of features. Moreover, we show that the asymptotic variance can consistently be estimated using an infinitesimal jackknife for bagged ensembles recently proposed by Efron (2013). In other words, our results let us both characterize and estimate the error-distribution of random forest predictions. Thus, random forests need not only be treated as black-box predictive algorithms, and can also be used for statistical inference."]]></description>
<dc:subject>to:NB ensemble_methods statistics prediction decision_trees</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:c880d1ae24a6/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1405.1533">
    <title>[1405.1533] A consistent deterministic regression tree for non-parametric prediction of time series</title>
    <dc:date>2015-01-22T00:28:48+00:00</dc:date>
    <link>http://arxiv.org/abs/1405.1533</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We study online prediction of bounded stationary ergodic processes. To do so, we consider the setting of prediction of individual sequences and build a deterministic regression tree that performs asymptotically as well as the best L-Lipschitz constant predictors. Then, we show why the obtained regret bound entails the asymptotical optimality with respect to the class of bounded stationary ergodic processes."]]></description>
<dc:subject>to_read time_series nonparametrics decision_trees regression learning_theory statistics in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:68696c063b54/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:time_series"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:nonparametrics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:learning_theory"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1409.2090">
    <title>[1409.2090] On the asymptotics of random forests</title>
    <dc:date>2015-01-20T14:07:18+00:00</dc:date>
    <link>http://arxiv.org/abs/1409.2090</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["The last decade has witnessed a growing interest in random forest models which are recognized to exhibit good practical performance, especially in high-dimensional settings. On the theoretical side, however, their predictive power remains largely unexplained, thereby creating a gap between theory and practice. The aim of this paper is twofold. Firstly, we provide theoretical guarantees to link finite forests used in practice (with a finite number M of trees) to their asymptotic counterparts. Using empirical process theory, we prove a uniform central limit theorem for a large class of random forest estimates, which holds in particular for Breiman's original forests. Secondly, we show that infinite forest consistency implies finite forest consistency and thus, we state the consistency of several infinite forests. In particular, we prove that q quantile forests---close in spirit to Breiman's forests but easier to study---are able to combine inconsistent trees to obtain a final consistent prediction, thus highlighting the benefits of random forests compared to single trees."]]></description>
<dc:subject>to:NB decision_trees ensemble_methods empirical_processes statistics random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:1fb865de58fb/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:empirical_processes"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1405.2881">
    <title>[1405.2881] Consistency of Random Forests</title>
    <dc:date>2015-01-20T14:06:18+00:00</dc:date>
    <link>http://arxiv.org/abs/1405.2881</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Random forests are a learning algorithm proposed by Breiman (2001) which combines several randomized decision trees and aggregates their predictions by averaging. Despite its wide usage and outstanding practical performance, little is known about the mathematical properties of the procedure. This disparity between theory and practice originates in the difficulty to simultaneously analyze both the randomization process and the highly data-dependent tree structure. In the present paper, we take a step forward in forest exploration by proving a consistency result for Breiman's (2001) original algorithm in the context of additive regression models. Our analysis also sheds an interesting light on how random forests can nicely adapt to sparsity in high-dimensional settings."]]></description>
<dc:subject>to:NB ensemble_methods decision_trees statistics random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:02de34d24f22/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1406.1845">
    <title>[1406.1845] Detecting Feature Interactions in Bagged Trees and Random Forests</title>
    <dc:date>2014-07-12T00:28:16+00:00</dc:date>
    <link>http://arxiv.org/abs/1406.1845</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Additive models remain popular statistical tools due to their ease of interpretation and as a result, hypothesis tests for additivity have been developed to asses the appropriateness of these models. However, as data continues to grow in size and complexity, practicioners are relying more heavily on learning algorithms because of their predictive superiority. Due to the black-box nature of these learning methods, the increase in predictive power is assumed to come at the cost of interpretability and understanding. However, recent work suggests that many popular learning algorithms, such as bagged trees and random forests, have desireable asymptotic properties which allow for formal statistical inference when base learners are built with subsamples. This work extends the hypothesis tests previously developed and demonstrates that by constructing an appropriate test set, we may perform formal hypothesis tests for additivity amongst features. We develop notions of total and partial additivity and demonstrate that both tests can be carried out at no additional computational cost to the original ensemble. Simulations and demonstrations on real data are also provided."]]></description>
<dc:subject>to:NB additive_models ensemble_methods statistics hypothesis_testing decision_trees hooker.giles random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:b9147b01495b/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:additive_models"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hypothesis_testing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hooker.giles"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1402.4293">
    <title>[1402.4293] The Random Forest Kernel and other kernels for big data from random partitions</title>
    <dc:date>2014-03-08T22:35:09+00:00</dc:date>
    <link>http://arxiv.org/abs/1402.4293</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We present Random Partition Kernels, a new class of kernels derived by demonstrating a natural connection between random partitions of objects and kernels between those objects. We show how the construction can be used to create kernels from methods that would not normally be viewed as random partitions, such as Random Forest. To demonstrate the potential of this method, we propose two new kernels, the Random Forest Kernel and the Fast Cluster Kernel, and show that these kernels consistently outperform standard kernels on problems involving real-world datasets. Finally, we show how the form of these kernels lend themselves to a natural approximation that is appropriate for certain big data problems, allowing O(N) inference in methods such as Gaussian Processes, Support Vector Machines and Kernel PCA."]]></description>
<dc:subject>to:NB data_mining kernel_methods statistics ghahramani.zoubin decision_trees</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:7d42e150ce86/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:data_mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:kernel_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ghahramani.zoubin"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1310.5677">
    <title>[1310.5677] Penalized Split Criteria for Interpretable Trees</title>
    <dc:date>2013-10-23T19:48:53+00:00</dc:date>
    <link>http://arxiv.org/abs/1310.5677</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This paper describes techniques for growing classification and regression trees designed to induce visually interpretable trees. This is achieved by penalizing splits that extend the subset of features used in a particular branch of the tree. After a brief motivation, we summarize existing methods and introduce new ones, providing illustrative examples throughout. Using a number of real classification and regression datasets, we find that these procedures can offer more interpretable fits than the CART methodology with very modest increases in out-of-sample loss."]]></description>
<dc:subject>decision_trees classifiers regression data_mining statistics in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:a55b5eed28e8/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:data_mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1309.7733">
    <title>[1309.7733] Regression Trees for Longitudinal Data</title>
    <dc:date>2013-10-11T23:28:42+00:00</dc:date>
    <link>http://arxiv.org/abs/1309.7733</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Often when a longitudinal change is studied in a population of interest we find that changes over time are heterogeneous (in terms of time and/or covariates' effect) and a traditional linear mixed effect model [Laird and Ware, 1982] on the entire population assuming common parametric form for covariates and time may not be applicable to the entire population. This is usually the case in studies when there are many possible predictors influencing the response trajectory. For example, Raudenbush [2001] used depression as an example to argue that it is incorrect to assume that all the people in a given population would be experiencing either increasing or decreasing levels of depression. In such cases, a group-averaged trajectory can mask important subgroup differences. Our aim is to identify and characterize longitudinally homogeneous subgroups based on the combination of baseline covariates. We achieve this goal by constructing regression tree through binary partitioning. We propose two steps procedure for binary partitioning: 1) first, choose the most significant partitioning variable and 2) then choose the best split by repetitive evaluation of a goodness of fit criterion at all the splits of chosen partitioning variable. To remedy for the problem of multiple testing, we propose a single test to identify the instability of parameter(s) in longitudinal models for a given partitioning variable. We obtain asymptotic results and examine finite sample behavior of our method through simulation studies. Finally, we apply our method to study the changes in brain metabolite levels of HIV infected patients."]]></description>
<dc:subject>to:NB time_series decision_trees regression nonparametrics statistics to_teach:undergrad-ADA to_teach:data_over_space_and_time</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:f815c23bfec4/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:time_series"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:nonparametrics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:undergrad-ADA"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data_over_space_and_time"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1310.1415">
    <title>[1310.1415] Narrowing the Gap: Random Forests In Theory and In Practice</title>
    <dc:date>2013-10-10T13:29:08+00:00</dc:date>
    <link>http://arxiv.org/abs/1310.1415</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Despite widespread interest and practical use, the theoretical properties of random forests are still not well understood. In this paper we contribute to this understanding in two ways. We present a new theoretically tractable variant of random regression forests and prove that our algorithm is consistent. We also provide an empirical evaluation, comparing our algorithm and other theoretically tractable random forest models to the random forest algorithm used in practice. Our experiments provide insight into the relative importance of different simplifications that theoreticians have made to obtain tractable models for analysis."]]></description>
<dc:subject>to:NB ensemble_methods decision_trees regression statistics random_forests</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:52b2f5841082/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://www.mitpressjournals.org/doi/abs/10.1162/NECO_a_00464">
    <title>Encoding Through Patterns: Regression Tree–Based Neuronal Population Models</title>
    <dc:date>2013-06-27T16:36:43+00:00</dc:date>
    <link>http://www.mitpressjournals.org/doi/abs/10.1162/NECO_a_00464</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Although the existence of correlated spiking between neurons in a population is well known, the role such correlations play in encoding stimuli is not. We address this question by constructing pattern-based encoding models that describe how time-varying stimulus drive modulates the expression probabilities of population-wide spike patterns. The challenge is that large populations may express an astronomical number of unique patterns, and so fitting a unique encoding model for each individual pattern is not feasible. We avoid this combinatorial problem using a dimensionality-reduction approach based on regression trees. Using the insight that some patterns may, from the perspective of encoding, be statistically indistinguishable, the tree divisively clusters the observed patterns into groups whose member patterns possess similar encoding properties. These groups, corresponding to the leaves of the tree, are much smaller in number than the original patterns, and the tree itself constitutes a tractable encoding model for each pattern. Our formalism can detect an extremely weak stimulus-driven pattern structure and is based on maximizing the data likelihood, not making a priori assumptions as to how patterns should be grouped. Most important, by comparing pattern encodings with independent neuron encodings, one can determine if neurons in the population are driven independently or collectively. We demonstrate this method using multiple unit recordings from area 17 of anesthetized cat in response to a sinusoidal grating and show that pattern-based encodings are superior to those of independent neuron models. The agnostic nature of our clustering approach allows us to investigate encoding by the collective statistics that are actually present rather than those (such as pairwise) that might be presumed."]]></description>
<dc:subject>to:NB neural_data_analysis neural_coding_and_decoding neuroscience statistics kith_and_kin haslinger.rob decision_trees</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:c49f9237a525/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:neural_data_analysis"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:neural_coding_and_decoding"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:neuroscience"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:kith_and_kin"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:haslinger.rob"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://projecteuclid.org/DPubS?service=UI&amp;version=1.0&amp;verb=Display&amp;handle=euclid.aoas/1365527208">
    <title>Loh , Zheng : Regression trees for longitudinal and multiresponse data</title>
    <dc:date>2013-04-10T15:52:21+00:00</dc:date>
    <link>http://projecteuclid.org/DPubS?service=UI&amp;version=1.0&amp;verb=Display&amp;handle=euclid.aoas/1365527208</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Previous algorithms for constructing regression tree models for longitudinal and multiresponse data have mostly followed the CART approach. Consequently, they inherit the same selection biases and computational difficulties as CART. We propose an alternative, based on the GUIDE approach, that treats each longitudinal data series as a curve and uses chi-squared tests of the residual curve patterns to select a variable to split each node of the tree. Besides being unbiased, the method is applicable to data with fixed and random time points and with missing values in the response or predictor variables. Simulation results comparing its mean squared prediction error with that of MVPART are given, as well as examples comparing it with standard linear mixed effects and generalized estimating equation models. Conditions for asymptotic consistency of regression tree function estimates are also given."]]></description>
<dc:subject>decision_trees regression time_series statistics in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:b864d6960d24/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:time_series"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://www.tandfonline.com/doi/abs/10.1080/10618600.2012.657132">
    <title>Taylor &amp; Francis Online :: Morse–Smale Regression - Journal of Computational and Graphical Statistics - Volume 22, Issue 1</title>
    <dc:date>2013-03-28T17:30:10+00:00</dc:date>
    <link>http://www.tandfonline.com/doi/abs/10.1080/10618600.2012.657132</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["This article introduces a novel partition-based regression approach that incorporates topological information. Partition-based regression typically introduces a quality-of-fit-driven decomposition of the domain. The emphasis in this work is on a topologically meaningful segmentation. Thus, the proposed regression approach is based on a segmentation induced by a discrete approximation of the Morse–Smale complex. This yields a segmentation with partitions corresponding to regions of the function with a single minimum and maximum that are often well approximated by a linear model. This approach yields regression models that are amenable to interpretation and have good predictive capacity. Typically, regression estimates are quantified by their geometrical accuracy. For the proposed regression, an important aspect is the quality of the segmentation itself. Thus, this article introduces a new criterion that measures the topological accuracy of the estimate. The topological accuracy provides a complementary measure to the classical geometrical error measures and is very sensitive to overfitting. The Morse–Smale regression is compared to state-of-the-art approaches in terms of geometry and topology and yields comparable or improved fits in many cases. Finally, a detailed study on climate-simulation data demonstrates the application of the Morse–Smale regression. Supplementary Materials are available online and contain an implementation of the proposed approach in the R package msr, an analysis and simulations on the stability of the Morse–Smale complex approximation, and additional tables for the climate-simulation study."]]></description>
<dc:subject>to:NB decision_trees regression topology statistics</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:df788d063c97/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:topology"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1302.4853">
    <title>[1302.4853] Consistency of Online Random Forests</title>
    <dc:date>2013-02-21T23:39:26+00:00</dc:date>
    <link>http://arxiv.org/abs/1302.4853</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["As a testament to their success, the theory of random forests has long been outpaced by their application in practice. In this paper, we take a step towards narrowing this gap by providing a consistency result for online random forests."]]></description>
<dc:subject>to:NB decision_trees classifiers machine_learning learning_theory</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:c0910fc190af/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:learning_theory"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://link.springer.com/article/10.1007/s10618-012-0261-2">
    <title>Enhanced spatiotemporal relational probability trees and forests - Springer</title>
    <dc:date>2013-01-14T21:11:51+00:00</dc:date>
    <link>http://link.springer.com/article/10.1007/s10618-012-0261-2</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Many real world domains are inherently spatiotemporal in nature. In this work, we introduce significant enhancements to two spatiotemporal relational learning methods, the spatiotemporal relational probability tree and the spatiotemporal relational random forest, that increase their ability to learn using spatiotemporal data. We enabled the models to formulate questions on both objects and the scalar and vector fields within and around objects, allowing the models to differentiate based on the gradient, divergence, and curl and to recognize the shape of point clouds defined by fields. This enables the model to ask questions about the change of a shape over time or about its orientation. These additions are validated on several real-world hazardous weather datasets. We demonstrate that these additions enable the models to learn robust classifiers that outperform the versions without these new additions. In addition, analysis of the learned models shows that the findings are consistent with current meteorological theories."]]></description>
<dc:subject>to:NB data_mining spatio-temporal_statistics statistics relational_learning decision_trees</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:fe6f222fbdb9/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:data_mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:spatio-temporal_statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:relational_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://biostats.bepress.com/uncbiostat/art33/">
    <title>&quot;Reinforcement Learning Trees&quot; by Ruoqing Zhu, Donglin Zeng et al.</title>
    <dc:date>2013-01-11T14:43:34+00:00</dc:date>
    <link>http://biostats.bepress.com/uncbiostat/art33/</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["In this paper, we introduce a new type of tree-based regression method, reinforcement learning trees (RLT), which exhibits significantly improved performance over traditional methods such as random forests (Breiman, 2001). The innovations are three-fold. First, the new method implements reinforcement learning at each selection of a splitting variable during the tree construction processes. By splitting on the variable that brings the greatest future improvement in later splits, rather than choosing the one with largest marginal effect from the immediate split, the constructed tree utilizes the available samples in a more efficient way. Moreover, such an approach can be adapted to make high-dimensional cuts available at a relatively small computational cost. Second, we propose a variable screening method that progressively mutes noise variables during the construction of each individual tree. The muting procedure also takes advantage of reinforcement learning and prevents noise variables from being considered in the search for splitting rules, so that towards a terminal node when the sample size is small, the splitting rules are still constructed from only strong variables. Last, we investigate asymptotic properties of the proposed method. We can show that under the proposed splitting variable selection procedure, the constructed trees are consistent. The error bounds for the proposed RLT are shown to depend on a pre-selected number p0, where p0 is an educated guess of the number of strong variables which is usually much smaller than the total number of variables p but at least as large as the true number of strong variables p1. Hence when p0 is properly chosen, the error bounds can be significantly improved."]]></description>
<dc:subject>regression statistics reinforcement_learning decision_trees to_teach:data-mining in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:e00f77a7dea3/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:reinforcement_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1206.4620">
    <title>[1206.4620] Improved Information Gain Estimates for Decision Tree Induction</title>
    <dc:date>2012-06-23T13:49:09+00:00</dc:date>
    <link>http://arxiv.org/abs/1206.4620</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Ensembles of classification and regression trees remain popular machine learning methods because they define flexible non-parametric models that predict well and are computationally efficient both during training and testing. During induction of decision trees one aims to find predicates that are maximally informative about the prediction target. To select good predicates most approaches estimate an information-theoretic scoring function, the information gain, both for classification and regression problems. We point out that the common estimation procedures are biased and show that by replacing them with improved estimators of the discrete and the differential entropy we can obtain better decision trees. In effect our modifications yield improved predictive performance and are simple to implement in any decision tree code."]]></description>
<dc:subject>decision_trees data_mining machine_learning entropy_estimation information_theory classifiers to_teach:data-mining in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:a03c429b2274/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:data_mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:entropy_estimation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:information_theory"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1205.2609">
    <title>[1205.2609] Which Spatial Partition Trees are Adaptive to Intrinsic Dimension?</title>
    <dc:date>2012-05-15T11:33:37+00:00</dc:date>
    <link>http://arxiv.org/abs/1205.2609</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Recent theory work has found that a special type of spatial partition tree - called a random projection tree - is adaptive to the intrinsic dimension of the data from which it is built. Here we examine this same question, with a combination of theory and experiments, for a broader class of trees that includes k-d trees, dyadic trees, and PCA trees. Our motivation is to get a feel for (i) the kind of intrinsic low dimensional structure that can be empirically verified, (ii) the extent to which a spatial partition can exploit such structure, and (iii) the implications for standard statistical tasks such as regression, vector quantization, and nearest neighbor search."]]></description>
<dc:subject>to:NB decision_trees prediction regression statistics dimension_reduction machine_learning</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:b908fc0ee0d7/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:dimension_reduction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1202.1523">
    <title>[1202.1523] Information Forests</title>
    <dc:date>2012-02-10T18:12:11+00:00</dc:date>
    <link>http://arxiv.org/abs/1202.1523</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["We describe Information Forests, an approach to classification that generalizes Random Forests by replacing the splitting criterion of non-leaf nodes from a discriminative one -- based on the entropy of the label distribution -- to a generative one -- based on maximizing the information divergence between the class-conditional distributions in the resulting partitions. The basic idea consists of deferring classification until a measure of "classification confidence" is sufficiently high, and instead breaking down the data so as to maximize this measure. In an alternative interpretation, Information Forests attempt to partition the data into subsets that are "as informative as possible" for the purpose of the task, which is to classify the data. Classification confidence, or informative content of the subsets, is quantified by the Information Divergence. Our approach relates to active learning, semi-supervised learning, mixed generative/discriminative learning."

After reading: meh.]]></description>
<dc:subject>decision_trees information_theory classifiers machine_learning to_teach:data-mining re:AoS_project have_read</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:7f7eb0d797fe/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:information_theory"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:AoS_project"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/1202.1561">
    <title>[1202.1561] Tree Models for Difference and Change Detection in a Complex Environment</title>
    <dc:date>2012-02-10T05:18:16+00:00</dc:date>
    <link>http://arxiv.org/abs/1202.1561</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["A new family of tree models is proposed, which we call "differential trees." A differential tree model is constructed from multiple data sets and aims to detect distributional differences between them. The new methodology differs from the existing difference and change detection techniques in its nonparametric nature, model construction from multiple data sets, and applicability to high-dimensional data. Through a detailed study of an arson case in New Zealand, where an individual is known to have been laying vegetation fires within a certain time period, we illustrate how these models can help detect changes in the frequencies of event occurrences and uncover unusual clusters of events in a complex environment."

--- After reading, I think their exposition is needlessly hard to follow, but let me take a stab at it.  In an ordinary classification tree, we are interested in the distribution of the class labels Y given the predictors X, i.e., Pr(Y|X), and make splits on X so that (in essence) the conditional entropy H[Y|X] becomes small.  This is of course equivalent to making splits so that the divergence of Pr(Y|X) from Pr(Y) is maximized.  What they are interested in is not classification but _describing_ how the different classes are distinct, so the relevant distribution is Pr(X|Y), and they want a big divergence between Pr(X) and Pr(X|Y).

ETA: Published version:
http://projecteuclid.org/euclid.aoas/1346418578 .  Haven't compared it to what  I read.
]]></description>
<dc:subject>re:network_differences statistics hypothesis_testing density_estimation decision_trees have_read data_mining two-sample_tests in_NB</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:1d69327d5561/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:re:network_differences"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hypothesis_testing"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:density_estimation"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:data_mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:two-sample_tests"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://www.springerlink.com/content/ng44781g47736260/">
    <title>RE-EM Trees: A Data Ming Approach for Longitudinal and Clustered Data</title>
    <dc:date>2012-01-24T23:49:05+00:00</dc:date>
    <link>http://www.springerlink.com/content/ng44781g47736260/</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["Longitudinal data refer to the situation where repeated observations are available for each sampled object. Clustered data, where observations are nested in a hierarchical structure within objects (without time necessarily being involved) represent a similar type of situation. Methodologies that take this structure into account allow for the possibilities of systematic differences between objects that are not related to attributes and autocorrelation within objects across time periods. A standard methodology in the statistics literature for this type of data is the mixed effects model, where these differences between objects are represented by so-called “random effects” that are estimated from the data (population-level relationships are termed “fixed effects,” together resulting in a mixed effects model). This paper presents a methodology that combines the structure of mixed effects models for longitudinal and clustered data with the flexibility of tree-based estimation methods. We apply the resulting estimation method, called the RE-EM tree, to pricing in online transactions, showing that the RE-EM tree is less sensitive to parametric assumptions and provides improved predictive power compared to linear models with random effects and regression trees without random effects. We also apply it to a smaller data set examining accident fatalities, and show that the RE-EM tree strongly outperforms a tree without random effects while performing comparably to a linear model with random effects. We also perform extensive simulation experiments to show that the estimator improves predictive performance relative to regression trees without random effects and is comparable or superior to using linear models with random effects in more general situations."]]></description>
<dc:subject>to:NB machine_learning decision_trees data_mining statistics hierarchical_statistical_models</dc:subject>
<dc:source>https://pinboard.in/</dc:source>
<dc:identifier>https://pinboard.in/u:cshalizi/b:5da6a371e870/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to:NB"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:data_mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:hierarchical_statistical_models"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://www.cs.cornell.edu/~daria/papers/Groves.pdf">
    <title>Additive Groves of Regression Trees - Sorokina et al.</title>
    <dc:date>2008-07-17T17:50:33+00:00</dc:date>
    <link>http://www.cs.cornell.edu/~daria/papers/Groves.pdf</link>
    <dc:creator>cshalizi</dc:creator><dc:subject>regression decision_trees ensemble_methods statistics machine_learning sorokina.daria caruana.rich riedewald.mirek have_read</dc:subject>
<dc:identifier>https://pinboard.in/u:cshalizi/b:f69860d3cd51/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:sorokina.daria"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:caruana.rich"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:riedewald.mirek"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://www.cs.cornell.edu/~daria/papers/Interactions.pdf">
    <title>Detecting Statistical Interactions with Additive Groves of Trees - Sorokina et al.</title>
    <dc:date>2008-07-17T17:47:41+00:00</dc:date>
    <link>http://www.cs.cornell.edu/~daria/papers/Interactions.pdf</link>
    <dc:creator>cshalizi</dc:creator><dc:subject>regression decision_trees statistics ensemble_methods sorokina.daria caruana.rich riedewald.mirek machine_learning statistical_interaction fink.daniel have_read</dc:subject>
<dc:identifier>https://pinboard.in/u:cshalizi/b:69e01d82961f/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:regression"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:sorokina.daria"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:caruana.rich"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:riedewald.mirek"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistical_interaction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:fink.daniel"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://flowingdata.com/2008/04/23/showing-the-obama-clinton-divide-in-decision-tree-infographic/">
    <title>Showing the Obama-Clinton Divide in Decision Tree Infographic | FlowingData</title>
    <dc:date>2008-04-24T13:26:56+00:00</dc:date>
    <link>http://flowingdata.com/2008/04/23/showing-the-obama-clinton-divide-in-decision-tree-infographic/</link>
    <dc:creator>cshalizi</dc:creator><dc:subject>to_teach:data-mining decision_trees classifiers cox.amanda via:klk us_politics obama.barack clinton.hillary blogged</dc:subject>
<dc:identifier>https://pinboard.in/u:cshalizi/b:711b9a159942/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:classifiers"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:cox.amanda"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:via:klk"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:us_politics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:obama.barack"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:clinton.hillary"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:blogged"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/0804.0650">
    <title>[0804.0650] Storms prediction : Logistic regression vs random forest for unbalanced data</title>
    <dc:date>2008-04-10T19:03:44+00:00</dc:date>
    <link>http://arxiv.org/abs/0804.0650</link>
    <dc:creator>cshalizi</dc:creator><description><![CDATA["satellite measurements of cloud systems which are to be classified either in convective or non convective systems. Convective cloud systems correspond to lightning and detecting such systems is of main importance for thunderstorm monitoring and warning"
]]></description>
<dc:subject>have_read machine_learning weather_prediction decision_trees to_teach:data-mining ensemble_methods random_forests</dc:subject>
<dc:identifier>https://pinboard.in/u:cshalizi/b:0c8fb572085f/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:have_read"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:machine_learning"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:weather_prediction"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:ensemble_methods"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:random_forests"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/0711.2434">
    <title>[0711.2434] Variable importance in binary regression trees and forests</title>
    <dc:date>2007-11-16T22:17:33+00:00</dc:date>
    <link>http://arxiv.org/abs/0711.2434</link>
    <dc:creator>cshalizi</dc:creator><dc:subject>statistics decision_trees to_teach:data-mining in_NB</dc:subject>
<dc:identifier>https://pinboard.in/u:cshalizi/b:91be0a2ef5c0/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:to_teach:data-mining"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
<item rdf:about="http://arxiv.org/abs/0708.1820">
    <title>[0708.1820] Confidence sets for split points in decision trees</title>
    <dc:date>2007-11-15T15:06:40+00:00</dc:date>
    <link>http://arxiv.org/abs/0708.1820</link>
    <dc:creator>cshalizi</dc:creator><dc:subject>confidence_sets decision_trees statistics in_NB</dc:subject>
<dc:identifier>https://pinboard.in/u:cshalizi/b:26310d6f4e3d/</dc:identifier>
<taxo:topics><rdf:Bag>	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:confidence_sets"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:decision_trees"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:statistics"/>
	<rdf:li rdf:resource="https://pinboard.in/u:cshalizi/t:in_NB"/>
</rdf:Bag></taxo:topics>
</item>
</rdf:RDF>