{
    "componentChunkName": "component---src-templates-note-note-js",
    "path": "/notes/LLM-breakdown",
    "result": {"data":{"mdx":{"body":"var _excluded = [\"components\"];\nfunction _extends() { _extends = Object.assign ? Object.assign.bind() : function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; return _extends.apply(this, arguments); }\nfunction _objectWithoutProperties(source, excluded) { if (source == null) return {}; var target = _objectWithoutPropertiesLoose(source, excluded); var key, i; if (Object.getOwnPropertySymbols) { var sourceSymbolKeys = Object.getOwnPropertySymbols(source); for (i = 0; i < sourceSymbolKeys.length; i++) { key = sourceSymbolKeys[i]; if (excluded.indexOf(key) >= 0) continue; if (!Object.prototype.propertyIsEnumerable.call(source, key)) continue; target[key] = source[key]; } } return target; }\nfunction _objectWithoutPropertiesLoose(source, excluded) { if (source == null) return {}; var target = {}; var sourceKeys = Object.keys(source); var key, i; for (i = 0; i < sourceKeys.length; i++) { key = sourceKeys[i]; if (excluded.indexOf(key) >= 0) continue; target[key] = source[key]; } return target; }\n/* @jsxRuntime classic */\n/* @jsx mdx */\n\nvar _frontmatter = {};\nvar layoutProps = {\n  _frontmatter: _frontmatter\n};\nvar MDXLayout = \"wrapper\";\nreturn function MDXContent(_ref) {\n  var components = _ref.components,\n    props = _objectWithoutProperties(_ref, _excluded);\n  return mdx(MDXLayout, _extends({}, layoutProps, props, {\n    components: components,\n    mdxType: \"MDXLayout\"\n  }), mdx(\"ol\", null, mdx(\"li\", {\n    parentName: \"ol\"\n  }, \"Part 1 - Tokenization (words to integers) - \", mdx(\"a\", {\n    parentName: \"li\",\n    \"href\": \"https://mikexcohen.substack.com/p/llm-breakdown-16-tokenization-words?triedRedirect=true\"\n  }, \"https://mikexcohen.substack.com/p/llm-breakdown-16-tokenization-words?triedRedirect=true\")), mdx(\"li\", {\n    parentName: \"ol\"\n  }, \"Part 2 -  Logits and next-token prediction - \", mdx(\"a\", {\n    parentName: \"li\",\n    \"href\": \"https://mikexcohen.substack.com/p/llm-breakdown-26-logits-and-next\"\n  }, \"https://mikexcohen.substack.com/p/llm-breakdown-26-logits-and-next\")), mdx(\"li\", {\n    parentName: \"ol\"\n  }, \"Part 3 - Embeddings -  \", mdx(\"a\", {\n    parentName: \"li\",\n    \"href\": \"https://mikexcohen.substack.com/p/llm-breakdown-36-embeddings\"\n  }, \"https://mikexcohen.substack.com/p/llm-breakdown-36-embeddings\")), mdx(\"li\", {\n    parentName: \"ol\"\n  }, \"Part 4 - Transformer outputs (hidden states)-  \", mdx(\"a\", {\n    parentName: \"li\",\n    \"href\": \"https://mikexcohen.substack.com/p/llm-breakdown-46-transformer-outputs\"\n  }, \"https://mikexcohen.substack.com/p/llm-breakdown-46-transformer-outputs\")), mdx(\"li\", {\n    parentName: \"ol\"\n  }, \"Part 5 - Attention-  \", mdx(\"a\", {\n    parentName: \"li\",\n    \"href\": \"https://mikexcohen.substack.com/p/llm-breakdown-56-attention?r=658yg&triedRedirect=true\"\n  }, \"https://mikexcohen.substack.com/p/llm-breakdown-56-attention?r=658yg&triedRedirect=true\")), mdx(\"li\", {\n    parentName: \"ol\"\n  })), mdx(\"p\", null, \"Notes\"), mdx(\"h2\", null, \"Embeddings -\"), mdx(\"p\", null, mdx(\"a\", {\n    parentName: \"p\",\n    \"href\": \"https://mikexcohen.substack.com/p/llm-breakdown-36-embeddings\"\n  }, \"https://mikexcohen.substack.com/p/llm-breakdown-36-embeddings\")), mdx(\"ol\", null, mdx(\"li\", {\n    parentName: \"ol\"\n  }, \"Embeddings vectors are part of the LLM, and as such, they are trained through next-token prediction alongside the rest of the model. In this sense, the embeddings vectors are not privileged or special; they are parameters to optimize just like every other parameter in the LLM.\"), mdx(\"li\", {\n    parentName: \"ol\"\n  }, \"After each prediction, the weights are adjusted to increase the model\\u2019s next-token prediction accuracy.\")), mdx(\"h2\", null, \"Attention\"), mdx(\"ol\", null, mdx(\"li\", {\n    parentName: \"ol\"\n  }, mdx(\"ul\", {\n    parentName: \"li\"\n  }, mdx(\"li\", {\n    parentName: \"ul\"\n  }, \"That first term in the softmax function (\", mdx(\"em\", {\n    parentName: \"li\"\n  }, \"QK^T/sqrt(d_k)\"), \") produces the \\u201Craw attention scores.\\u201D The idea is that the query vectors encode what each token is\\xA0\", mdx(\"em\", {\n    parentName: \"li\"\n  }, \"searching\"), \"\\xA0for, and the keys vectors encode what each token has to offer the query vectors. Imagine a dating app for embeddings vectors: Q is each token\\u2019s dating profile and K is the profile of the other tokens in the text. When there\\u2019s a good match, their dot product is high; and when there isn\\u2019t a good match (meaning the two tokens have no relevant contextual information in common), then the dot product is negative.\"))), mdx(\"li\", {\n    parentName: \"ol\"\n  })));\n}\n;\nMDXContent.isMDXComponent = true;","inboundReferences":[]}},"pageContext":{"slug":"LLM-breakdown","node":{"childMdx":{"slug":"LLM-breakdown"}}}},
    "staticQueryHashes": []}