{
    "componentChunkName": "component---src-templates-note-note-js",
    "path": "/notes/KL-divergence",
    "result": {"data":{"mdx":{"body":"var _excluded = [\"components\"];\nfunction _extends() { _extends = Object.assign ? Object.assign.bind() : function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; return _extends.apply(this, arguments); }\nfunction _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }\nfunction _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }\n/* @jsxRuntime classic */\n/* @jsx mdx */\n\nvar _frontmatter = {};\nvar layoutProps = {\n  _frontmatter: _frontmatter\n};\nvar MDXLayout = \"wrapper\";\nreturn function MDXContent(_ref) {\n  var components = _ref.components,\n    props = _objectWithoutProperties(_ref, _excluded);\n  return mdx(MDXLayout, _extends({}, layoutProps, props, {\n    components: components,\n    mdxType: \"MDXLayout\"\n  }), mdx(\"p\", null, \"KL divergence, or Kullback-Leibler divergence, is a measure of how one probability distribution diverges from a second, expected probability distribution. It is a fundamental concept in information theory and statistics, often used to quantify the difference between two probability distributions.\"), mdx(\"h3\", null, \"Key Points about KL Divergence:\"), mdx(\"ol\", null, mdx(\"li\", {\n    parentName: \"ol\"\n  }, mdx(\"p\", {\n    parentName: \"li\"\n  }, mdx(\"strong\", {\n    parentName: \"p\"\n  }, \"Definition\"), \":\"), mdx(\"ul\", {\n    parentName: \"li\"\n  }, mdx(\"li\", {\n    parentName: \"ul\"\n  }, \"For discrete probability distributions \", \"(\", \" P \", \")\", \" and \", \"(\", \" Q \", \")\", \", the KL divergence from \", \"(\", \" Q \", \")\", \" to \", \"(\", \" P \", \")\", \" is defined as:\\n$$ D\", mdx(\"em\", {\n    parentName: \"li\"\n  }, \"{KL}(P \\\\parallel Q) = \\\\sum\"), \"{x} P(x) \\\\log \\\\frac{P(x)}{Q(x)} $$\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, \"For continuous distributions, the sum is replaced by an integral.\"))), mdx(\"li\", {\n    parentName: \"ol\"\n  }, mdx(\"p\", {\n    parentName: \"li\"\n  }, mdx(\"strong\", {\n    parentName: \"p\"\n  }, \"Interpretation\"), \":\"), mdx(\"ul\", {\n    parentName: \"li\"\n  }, mdx(\"li\", {\n    parentName: \"ul\"\n  }, \"KL divergence measures the \\\"extra\\\" amount of information required to encode samples from distribution \", \"(\", \" P \", \")\", \" using a code optimized for distribution \", \"(\", \" Q \", \")\", \".\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, \"It is not symmetric, meaning \", \"(\", \" D\", mdx(\"em\", {\n    parentName: \"li\"\n  }, \"{KL}(P \\\\parallel Q) \\\\neq D\"), \"{KL}(Q \\\\parallel P) \", \")\", \".\"))), mdx(\"li\", {\n    parentName: \"ol\"\n  }, mdx(\"p\", {\n    parentName: \"li\"\n  }, mdx(\"strong\", {\n    parentName: \"p\"\n  }, \"Properties\"), \":\"), mdx(\"ul\", {\n    parentName: \"li\"\n  }, mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"strong\", {\n    parentName: \"li\"\n  }, \"Non-negativity\"), \": \", \"(\", \" D_{KL}(P \\\\parallel Q) \\\\geq 0 \", \")\", \", with equality if and only if \", \"(\", \" P = Q \", \")\", \" almost everywhere.\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"strong\", {\n    parentName: \"li\"\n  }, \"Asymmetry\"), \": As mentioned, it is not symmetric, which means it is not a true metric or distance.\"))), mdx(\"li\", {\n    parentName: \"ol\"\n  }, mdx(\"p\", {\n    parentName: \"li\"\n  }, mdx(\"strong\", {\n    parentName: \"p\"\n  }, \"Applications\"), \":\"), mdx(\"ul\", {\n    parentName: \"li\"\n  }, mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"strong\", {\n    parentName: \"li\"\n  }, \"Machine Learning\"), \": Used in algorithms like Variational Autoencoders (\", mdx(\"a\", {\n    parentName: \"li\",\n    \"href\": \"/notes/VAEs\",\n    \"title\": \"VAEs\"\n  }, \"VAEs\"), \") and in regularizing policy updates in reinforcement learning (e.g., \", mdx(\"a\", {\n    parentName: \"li\",\n    \"href\": \"/notes/PPO\",\n    \"title\": \"PPO\"\n  }, \"PPO\"), \").\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"strong\", {\n    parentName: \"li\"\n  }, \"Statistics\"), \": Helps in hypothesis testing and model selection.\"), mdx(\"li\", {\n    parentName: \"ul\"\n  }, mdx(\"strong\", {\n    parentName: \"li\"\n  }, \"Information Theory\"), \": Measures the inefficiency of assuming that the distribution is \", \"(\", \" Q \", \")\", \" when the true distribution is \", \"(\", \" P \", \")\", \".\")))), mdx(\"p\", null, \"KL divergence is a powerful tool for comparing probability distributions and is widely used in various fields of data science and machine learning.\"));\n}\n;\nMDXContent.isMDXComponent = true;","inboundReferences":[]}},"pageContext":{"slug":"KL-divergence","node":{"childMdx":{"slug":"KL-divergence"}}}},
    "staticQueryHashes": []}