{ "id": "eppe9aapxa", "slug": "sparse-bayesian-learning", "title": "Sparse Bayesian learning", "updated": "2026-10-21", "topic_path": [ { "id": "technology", "label": "Technology and the built world", "api_url": "https://www.edgepedia.org/api/v1/topics/technology" }, { "id": "technology.computing", "label": "Computing and digital systems", "api_url": "https://www.edgepedia.org/api/v1/topics/technology.computing" }, { "id": "technology.computing.ai", "label": "Artificial intelligence and data", "api_url": "https://www.edgepedia.org/api/v1/topics/technology.computing.ai" }, { "id": "technology.computing.ai.ml_computation", "label": "Machine learning and neural computation", "api_url": "https://www.edgepedia.org/api/v1/topics/technology.computing.ai.ml_computation" }, { "id": "technology.computing.ai.ml_methods", "label": "Machine learning methods", "api_url": "https://www.edgepedia.org/api/v1/topics/technology.computing.ai.ml_methods" }, { "id": "technology.computing.ai.supervised_unsupervised", "label": "Supervised, unsupervised, and semi-supervised learning", "api_url": "https://www.edgepedia.org/api/v1/topics/technology.computing.ai.supervised_unsupervised" }, { "id": "technology.computing.ai.supervised_unsupervised.feature_selection_engineering", "label": "Feature selection and feature engineering", "api_url": "https://www.edgepedia.org/api/v1/topics/technology.computing.ai.supervised_unsupervised.feature_selection_engineering" } ], "geo": [ { "id": "geo.nongeo.t1946.technology.computing.ai.supervised_unsupervised", "label": "Non-geographic · 1946 to 2000: Supervised, unsupervised, and semi-supervised learning", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t1946.technology.computing.ai.supervised_unsupervised", "path": [ { "id": "geo.nongeo", "label": "Non-geographic", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo" }, { "id": "geo.nongeo.t1946", "label": "Non-geographic · 1946 to 2000", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t1946" }, { "id": "geo.nongeo.t1946.technology", "label": "Technology and the built world", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t1946.technology" }, { "id": "geo.nongeo.t1946.technology.computing", "label": "Computing and digital systems", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t1946.technology.computing" }, { "id": "geo.nongeo.t1946.technology.computing.ai", "label": "Artificial intelligence and data", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t1946.technology.computing.ai" }, { "id": "geo.nongeo.t1946.technology.computing.ai.ml_computation", "label": "Machine learning and neural computation", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t1946.technology.computing.ai.ml_computation" }, { "id": "geo.nongeo.t1946.technology.computing.ai.ml_methods", "label": "Machine learning methods", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t1946.technology.computing.ai.ml_methods" }, { "id": "geo.nongeo.t1946.technology.computing.ai.supervised_unsupervised", "label": "Supervised, unsupervised, and semi-supervised learning", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t1946.technology.computing.ai.supervised_unsupervised" } ] }, { "id": "geo.nongeo.t2001.technology.computing.ai.supervised_unsupervised", "label": "Non-geographic · 2001 to 2020: Supervised, unsupervised, and semi-supervised learning", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t2001.technology.computing.ai.supervised_unsupervised", "path": [ { "id": "geo.nongeo", "label": "Non-geographic", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo" }, { "id": "geo.nongeo.t2001", "label": "Non-geographic · 2001 to 2020", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t2001" }, { "id": "geo.nongeo.t2001.technology", "label": "Technology and the built world", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t2001.technology" }, { "id": "geo.nongeo.t2001.technology.computing", "label": "Computing and digital systems", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t2001.technology.computing" }, { "id": "geo.nongeo.t2001.technology.computing.ai", "label": "Artificial intelligence and data", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t2001.technology.computing.ai" }, { "id": "geo.nongeo.t2001.technology.computing.ai.ml_computation", "label": "Machine learning and neural computation", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t2001.technology.computing.ai.ml_computation" }, { "id": "geo.nongeo.t2001.technology.computing.ai.ml_methods", "label": "Machine learning methods", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t2001.technology.computing.ai.ml_methods" }, { "id": "geo.nongeo.t2001.technology.computing.ai.supervised_unsupervised", "label": "Supervised, unsupervised, and semi-supervised learning", "api_url": "https://www.edgepedia.org/api/v1/geo/geo.nongeo.t2001.technology.computing.ai.supervised_unsupervised" } ] } ], "excerpt": "Sparse Bayesian learning is a Bayesian machine learning method that fits regression and classification models by pruning irrelevant basis functions, producing sparse models with probabilistic outputs.", "snippet": "Sparse Bayesian learning is a Bayesian machine learning method that fits regression and classification models by pruning irrelevant basis functions, producing sparse models with probabilistic outputs.", "node": "technology.computing.ai.supervised_unsupervised.feature_selection_engineering", "markdown": "# Sparse Bayesian learning\n\nSparse Bayesian learning (SBL) is a Bayesian machine learning method that fits linear regression and classification models by giving each weight its own sparsity-inducing prior, so that irrelevant basis functions are automatically pruned during training. Its canonical instance is the relevance vector machine (RVM), a sparse linear model of the same functional form as the support vector machine that, unlike the SVM, produces probabilistic outputs and accepts arbitrary basis functions, including non-Mercer kernels.[1](https://jmlr.org/papers/volume1/tipping01a/tipping01a.pdf) Training maximizes the marginal likelihood (the type-II evidence) rather than the weights themselves, and the surviving nonzero weights identify retained basis functions; in common RVM setups these are associated with training inputs, which are then called relevance vectors.[1](https://jmlr.org/papers/volume1/tipping01a/tipping01a.pdf)\n\n| Key fact | Value |\n|---|---|\n| Introducing publications | Tipping, NIPS 2000 conference version and JMLR 2001 (vol. 1, pp. 211–244) |\n| Objective | Marginal likelihood \\( L(\\alpha) = -\\tfrac{1}{2}[N\\log 2\\pi + \\log|C| + t^{\\mathrm T} \\cdot C^{-1} \\cdot t] \\), \\( C = \\sigma^2 \\cdot I + \\Phi \\cdot A^{-1} \\cdot \\Phi^{\\mathrm T} \\) |\n| Sparsity mechanism | One precision hyperparameter \\( \\alpha_i \\) per weight; \\( \\alpha_i \\to \\infty \\) prunes basis function \\( i \\) |\n| Sparsity vs SVM | 4 vs 38 kernels (Ripley synthetic, test error 9.3% vs 10.6%); 3 vs 44 kernels on a 200-example set |\n| Training cost | \\( O(N^2) \\) memory, \\( O(N^3) \\) computation; impractical beyond about 5,000 examples in the original form |\n| Compressed sensing | Global minimum at the maximally sparse solution, often with fewer local minima than MAP methods such as basis pursuit |\n\n## How it works\n\nThe model is a generalized linear basis model \\( t = \\Phi \\cdot w + \\epsilon \\) with Gaussian noise \\( \\epsilon \\sim \\mathcal N(0, \\sigma^2) \\). Each weight \\( w_i \\) receives a zero-mean Gaussian prior with its own precision \\( \\alpha_i \\); this one-hyperparameter-per-weight scheme, a form of automatic relevance determination (ARD), is the feature responsible for sparsity.[2](https://proceedings.neurips.cc/paper_files/paper/1999/file/f3144cefe89a60d6a1afaf7859c5076b-Paper.pdf) Integrating out the \\( \\alpha_i \\) under gamma hyperpriors yields a marginal weight prior that is a product of Student-t distributions, sharply peaked at zero like the Laplace prior used as an L1 regularizer.[1](https://jmlr.org/papers/volume1/tipping01a/tipping01a.pdf)\n\nDuring training, many \\( \\alpha_i \\) drive toward infinity, the posterior over the corresponding \\( w_i \\) becomes infinitely peaked at zero, and those basis functions are pruned.[2](https://proceedings.neurips.cc/paper_files/paper/1999/file/f3144cefe89a60d6a1afaf7859c5076b-Paper.pdf) In regression, the resulting relevance learning is equivalent to a [Gaussian process](https://www.edgepedia.org/gaussian-process) model with covariance \\( C = \\sigma^2 \\cdot I + \\Phi \\cdot A^{-1} \\cdot \\Phi^{\\mathrm T} \\).[1](https://jmlr.org/papers/volume1/tipping01a/tipping01a.pdf)\n\n## How it is done\n\nThe posterior over the weights is Gaussian with covariance \\( \\Sigma = (A + \\sigma^{-2} \\cdot \\Phi^{\\mathrm T} \\cdot \\Phi)^{-1} \\) and mean \\( \\mu = \\sigma^{-2} \\cdot \\Sigma \\cdot \\Phi^{\\mathrm T} \\cdot t \\), where \\( A = \\mathrm{diag}(\\alpha_1, \\ldots, \\alpha_M) \\).[3](https://miketipping.com/papers/met-fastsbl.pdf) Training maximizes the log marginal likelihood \\( L(\\alpha) = -\\tfrac{1}{2}[N\\log 2\\pi + \\log|C| + t^{\\mathrm T} \\cdot C^{-1} \\cdot t] \\) with \\( C = \\sigma^2 \\cdot I + \\Phi \\cdot A^{-1} \\cdot \\Phi^{\\mathrm T} \\).[4](https://proceedings.neurips.cc/paper_files/paper/2001/file/02b1be0d48924c327124732726097157-Paper.pdf)\n\nHyperparameter updates alternate between EM-style re-estimation \\( \\alpha_i^{\\mathrm{new}} = 1/(\\Sigma_{ii} + \\mu_i^2) \\) and the faster direct update \\( \\alpha_i^{\\mathrm{new}} = \\gamma_i/\\mu_i^2 \\) with \\( \\gamma_i = 1 - \\alpha_i\\Sigma_{ii} \\), together with a noise-variance re-estimate.[2](https://proceedings.neurips.cc/paper_files/paper/1999/file/f3144cefe89a60d6a1afaf7859c5076b-Paper.pdf) Faul and Tipping showed that \\( L(\\alpha) \\) has a unique closed-form maximum in each individual \\( \\alpha_i \\), and that this maximum lies at \\( \\alpha_i = \\infty \\), equivalent to removing basis function \\( i \\), whenever the criterion \\( Q_i^2 - S_i \\) is negative; \\( Q_i \\) measures how well \\( \\phi_i \\) helps explain the data and \\( S_i \\) how much including it inflates \\( C \\).[4](https://proceedings.neurips.cc/paper_files/paper/2001/file/02b1be0d48924c327124732726097157-Paper.pdf) Because stationary points of sequential single-hyperparameter optimization are not saddle points, the closed-form solution also lets basis functions outside the model be assessed and added, enabling constructive algorithms.[4](https://proceedings.neurips.cc/paper_files/paper/2001/file/02b1be0d48924c327124732726097157-Paper.pdf) For classification, where the marginal likelihood is not analytically integrable, a Laplace (Gaussian) approximation is used; the log posterior is concave, supporting the approximation.[2](https://proceedings.neurips.cc/paper_files/paper/1999/file/f3144cefe89a60d6a1afaf7859c5076b-Paper.pdf)\n\n## Origin\n\nThe approach builds on earlier ARD-style Bayesian methods, and on the type-II evidence procedure.[1](https://jmlr.org/papers/volume1/tipping01a/tipping01a.pdf) The RVM was motivated by the SVM, a model of identical functional form that lacks probabilistic outputs and imposes trade-off-parameter and Mercer-kernel restrictions that the RVM avoids, and the SVM is credited to Boser, Guyon, and Vapnik (1992).[5](https://dl.acm.org/doi/10.1162/15324430152748236)\n\n## Variants\n\n**Fast marginal likelihood maximization.** An accelerated algorithm was introduced at the Ninth International Workshop on Artificial Intelligence and [Statistics](https://www.edgepedia.org/statistics) (PMLR R4, pages 276–283).[6](https://proceedings.mlr.press/r4/tipping03a.html) It initializes with an empty model and sequentially adds or deletes basis functions using \\( \\theta_i = q_i^2 - s_i \\): if \\( \\theta_i > 0 \\) and \\( \\alpha_i = \\infty \\), add \\( \\phi_i \\) with an updated \\( \\alpha_i \\); if \\( \\theta_i \\le 0 \\) and \\( \\alpha_i < \\infty \\), delete \\( \\phi_i \\) and set \\( \\alpha_i = \\infty \\).[3](https://miketipping.com/papers/met-fastsbl.pdf) Analytic deletion improves sparsity relative to the original numeric pruning.[3](https://miketipping.com/papers/met-fastsbl.pdf)\n\n**Variational SBL.** Variational inference solves the RVM, giving a posterior distribution over both parameters and hyperparameters instead of type-II point estimates; the solution is computationally more expensive but expected to help most on small datasets.[7](https://www.miketipping.com/papers/Bishop-VRVM-UAI-00.pdf) Shutin, Buchgraber, Kulkarni, and Poor later developed fast variational SBL for superimposed signals, whose pruning conditions coincide with those of fast marginal likelihood maximization and correspond to removing components with signal-to-noise ratio below a 0 dB threshold.[8](https://www.princeton.edu/~kulkarni/Papers/Journals/j080_2011_ShuBuchKulPoor_TransSP.pdf)\n\n**SBL for basis selection.** Wipf and Rao adapted SBL to basis selection from overcomplete dictionaries in IEEE Transactions on Signal Processing in 2004.[9](https://doi.org/10.1109/tsp.2004.831016) The Bayesian backfitting RVM reduces training to \\( O(N^2) \\) per update cycle versus \\( O(N^3) \\) for the standard RVM.[10](https://icml.cc/Conferences/2004/proceedings/papers/115.pdf)\n\n## Applications\n\nBeyond regression and classification benchmarks, SBL is used for sparse recovery from overcomplete dictionaries, where simulations show improved recovery over basis pursuit and the FOCUSS class of algorithms.[9](https://doi.org/10.1109/tsp.2004.831016) In Monte Carlo trials with 1000 runs per condition, SBL recovered sparse vectors better than basis pursuit and OMP across dictionary redundancies up to \\( M/N \\) of 5 and up to 30 nonzero weights, including the worst case of unit-amplitude nonzero weights.[11](https://papers.nips.cc/paper_files/paper/2005/file/d8e1344e27a5b08cdfd5d027d9b8d6de-Paper.pdf) Named sparse-representation applications include neuroelectromagnetic source localization, compressed sensing, sparse component analysis, feature selection, image restoration and compression, and neural coding, with strong results on large, ill-posed neuroimaging problems.[12](https://escholarship.org/uc/item/1kh6989n) In communications, SBL estimates massive MIMO channels by treating each channel entry's variance as a sparsity-enforcing weight estimated after marginalizing the channel out of the model.[13](https://doi.org/10.48550/arxiv.2501.07969)\n\n## Limitations and alternatives\n\nThe primary disadvantage is computational cost: memory and computation scale with the square and cube of the number of basis functions, making the original algorithm impractical for several thousand training examples.[1](https://jmlr.org/papers/volume1/tipping01a/tipping01a.pdf) The EM and MacKay updates cannot ensure global convergence because they can get trapped at a fixed point rather than a stationary point of the marginal likelihood, and EM converges slowly numerically.[14](https://raw.githubusercontent.com/mlresearch/v235/main/assets/wang24al/wang24al.pdf) SBL's reliance on priors chosen for analytical tractability can mismatch actual signal statistics and cause over-shrinkage.[15](https://eurasip.org/Proceedings/Eusipco/Eusipco2026/pdfs/0000911.pdf)\n\nCompared with alternatives, nearly all competing sparse-recovery methods, including OMP, basis pursuit (the LASSO), and minimum \\( L_p \\) quasi-norm methods, perform MAP estimation with a fixed sparsity-inducing prior, whereas SBL is an empirical-Bayes evidence-maximization approach.[12](https://escholarship.org/uc/item/1kh6989n) In sparse recovery the global SBL minimum is always achieved at the maximally sparse solution, unlike the basis pursuit cost function, often with fewer local minima than comparable MAP methods.[12](https://escholarship.org/uc/item/1kh6989n) Against the SVM, the RVM trades comparable accuracy for far fewer basis functions and probabilistic outputs, but trains more slowly on large datasets.[2](https://proceedings.neurips.cc/paper_files/paper/1999/file/f3144cefe89a60d6a1afaf7859c5076b-Paper.pdf)\n\nRecent work has clarified convergence. Popular SBL algorithms, including the EM-based method and Tipping's multiplicative update, can be derived as majorization-minimization descent steps on a common majorizer, giving convergence guarantees previously unknown for this class.[16](https://arxiv.org/html/2604.02513) An iterative Min-Min method based on the concave-convex procedure globally converges to a local minimum or saddle point of the marginal likelihood from any starting point, without fixing the noise level.[14](https://raw.githubusercontent.com/mlresearch/v235/main/assets/wang24al/wang24al.pdf)\n\n## References\n\n1. [Sparse Bayesian Learning and the Relevance Vector Machine (Tipping, JMLR 2001)](https://jmlr.org/papers/volume1/tipping01a/tipping01a.pdf)\n2. [The Relevance Vector Machine (Tipping, NeurIPS 1999 conference, NIPS 12, 2000 proceedings)](https://proceedings.neurips.cc/paper_files/paper/1999/file/f3144cefe89a60d6a1afaf7859c5076b-Paper.pdf)\n3. [Fast Marginal Likelihood Maximisation for Sparse Bayesian Models (Tipping & Faul)](https://miketipping.com/papers/met-fastsbl.pdf)\n4. [Analysis of Sparse Bayesian Learning (Faul & Tipping, NIPS 2001)](https://proceedings.neurips.cc/paper_files/paper/2001/file/02b1be0d48924c327124732726097157-Paper.pdf)\n5. [Sparse bayesian learning and the relevance vector machine, JMLR Volume 1, Pages 211-244, DOI 10.1162/15324430152748236, published 01 September 2001](https://dl.acm.org/doi/10.1162/15324430152748236)\n6. [Fast Marginal Likelihood Maximisation for Sparse Bayesian Models (PMLR record)](https://proceedings.mlr.press/r4/tipping03a.html)\n7. [Variational Relevance Vector Machines (Bishop & Tipping, UAI 2000)](https://www.miketipping.com/papers/Bishop-VRVM-UAI-00.pdf)\n8. [Fast Variational Sparse Bayesian Learning With Automatic Relevance Determination for Superimposed Signals (Shutin, Buchgraber, Kulkarni, Poor, IEEE Trans. Signal Processing, 2011)](https://www.princeton.edu/~kulkarni/Papers/Journals/j080_2011_ShuBuchKulPoor_TransSP.pdf)\n9. [D.P. Wipf, B.D. Rao (2004). Sparse Bayesian Learning for Basis Selection. IEEE Transactions on Signal Processing.](https://doi.org/10.1109/tsp.2004.831016)\n10. [The Bayesian Backfitting Relevance Vector Machine (ICML 2004)](https://icml.cc/Conferences/2004/proceedings/papers/115.pdf)\n11. [Comparing the Effects of Different Weight Distributions on Finding Sparse Representations (Wipf & Rao, NIPS 2005)](https://papers.nips.cc/paper_files/paper/2005/file/d8e1344e27a5b08cdfd5d027d9b8d6de-Paper.pdf)\n12. [Bayesian methods for finding sparse representations (Wipf PhD thesis, UC San Diego, 2006)](https://escholarship.org/uc/item/1kh6989n)\n13. [Arjas, Arttu, Atzeni, Italo (2025). Enhanced Sparse Bayesian Learning Methods with Application to Massive MIMO Channel Estimation. arXiv (Cornell University).](https://doi.org/10.48550/arxiv.2501.07969)\n14. [An Iterative Min-Min Optimization Method for Sparse Bayesian Learning (PMLR v235, 2024)](https://raw.githubusercontent.com/mlresearch/v235/main/assets/wang24al/wang24al.pdf)\n15. [Cross-Predictive Sparse Bayesian Learning with application to near-field XL-MIMO channel estimation (EUSIPCO 2026)](https://eurasip.org/Proceedings/Eusipco/Eusipco2026/pdfs/0000911.pdf)\n16. [Sparse Bayesian Learning Algorithms Revisited: From Learning Majorizers to Structured Algorithmic Learning using Neural Networks](https://arxiv.org/html/2604.02513)\n\n---\n*Topic: Encyclopedia › Technology and the built world › Computing and digital systems › Artificial intelligence and data › Machine learning and neural computation › Machine learning methods › Supervised, unsupervised, and semi-supervised learning › Feature selection and feature engineering*\n\n*Initially written Sep 29, 2026 · Reviewed: Sep 30, 2026 · Edited: Sep 30, 2026; Oct 21, 2026 · Last review: Sep 30, 2026*\n\n*Copyright 2026 EdgeChat AI, a subsidiary of Biostate AI.*\n\nLicense: Edgepedia Community License 1.0, https://www.edgepedia.org/edgepedia/license\n", "same_as": [ "https://www.princeton.edu/~kulkarni/Papers/Journals/j080_2011_ShuBuchKulPoor_TransSP.pdf" ], "url": "https://www.edgepedia.org/sparse-bayesian-learning", "markdown_url": "https://www.edgepedia.org/sparse-bayesian-learning.md", "license": { "name": "Edgepedia Community License 1.0", "url": "https://www.edgepedia.org/edgepedia/license", "summary": "Free with credit, commercial use included. AI training is open to everyone. For other uses, organizations over USD 100M in revenue or 100M monthly users license separately.", "spdx": "LicenseRef-Edgepedia-Community-1.0" }, "credit": "\"Sparse Bayesian learning\", Edgepedia (EdgeChat), https://www.edgepedia.org/sparse-bayesian-learning. Edgepedia Community License 1.0.", "credit_md": "\"[Sparse Bayesian learning](https://www.edgepedia.org/sparse-bayesian-learning)\", Edgepedia (EdgeChat), [https://www.edgepedia.org/sparse-bayesian-learning](https://www.edgepedia.org/sparse-bayesian-learning). [Edgepedia Community License 1.0](https://www.edgepedia.org/edgepedia/license).", "credit_html": "\"Sparse Bayesian learning\", Edgepedia (EdgeChat), https://www.edgepedia.org/sparse-bayesian-learning. Edgepedia Community License 1.0.", "speakable": "Sparse Bayesian learning is a Bayesian machine learning method that fits regression and classification models by pruning irrelevant basis functions, producing sparse models with probabilistic outputs." }