{ "packId": "llm-101", "packName": "LLM 101", "packVersion": "1.1.4", "icon": "🤖", "shortName": "LLM 101", "description": "A clear primer on large language models: their history from n-grams to the Transformer to ChatGPT, how they work (tokens, attention, context windows), the training pipeline (pretraining, SFT, RLHF, alignment), prompting and RAG, the key companies and people, and the numbers and milestones that define the field.", "author": "Flash Feed", "language": "en", "tagsVocabulary": [ "agents", "anthropic", "attention", "bert", "chain-of-thought", "chatgpt", "companies", "comparison", "compute", "context-window", "deep-learning", "ecosystem", "embeddings", "emergence", "few-shot", "fine-tuning", "generation", "google", "gpt", "gpt-4", "hallucination", "hardware", "history", "in-context-learning", "inference", "mechanics", "meta", "misconception", "mistral", "numeric", "open-weights", "parameters", "people", "pre-transformer", "pretraining", "procedure", "prompting", "rag", "recall", "rlaif", "rlhf", "sampling", "scaling-laws", "sft", "tokenization", "tool-use", "training", "training-data", "transformer", "zero-shot", "accessibility", "brainstorming", "business", "classification", "coding", "customer-support", "data-cleaning", "debugging", "drafting", "editing", "explanation", "extraction", "finance", "fit", "healthcare", "high-stakes", "human-in-the-loop", "knowledge", "language", "legal", "meeting-notes", "patterns", "personal", "research", "rewriting", "semantic-search", "summarization", "testing", "translation", "tutoring", "use-case", "alignment", "automation-bias", "bias", "copyright", "cost", "data-handling", "fairness", "governance", "guardrails", "ip", "jailbreak", "knowledge-cutoff", "latency", "limits", "math", "memorization", "misinformation", "mitigation", "model-card", "nist", "non-determinism", "pii", "privacy", "prompt-injection", "reasoning", "red-teaming", "reliability", "responsible-use", "security", "societal", "transparency", "text-units", "model-internals", "behavior", "before-alignment", "prompt-basics", "prompt-techniques", "us-frontier", "intl-other", "godfathers", "transformer-era", "openai-anthropic", "deepmind-infra", "transform-text", "improve-classify", "code", "writing-comms", "thinking-data", "architecture" ], "items": [ { "id": "f-ngrams", "shape": "fact", "tags": [ "history", "pre-transformer" ], "title": "Pre-Transformer NLP: n-grams", "body": "Early language models predicted the next word by counting how often word sequences appeared in text (n-grams). They were fast but couldn't capture meaning across long distances — only the last few words mattered.", "illustration": { "imageSearchTerm": "chain of connected words", "imagePrompt": "An abstract chain of small glowing rectangular blocks linked in a short row, fading into blur after just a few links, evoking a model that only looks at the last few words, no legible text.", "alt": "Pre-Transformer NLP: n-grams", "depictable": false, "credit": "Unsplash · Glen Carrie · Unsplash License", "creditUrl": "https://unsplash.com/photos/a-pile-of-black-and-white-typewriters-sitting-on-top-of-a-blue-surface-wYSn5hjSkFo", "url": "https://cdn.recurxive.com/packs/llm-101/images/f-ngrams.webp" }, "factVariant": "image-heavy", "imageCaption": "Early language models predicted the next word by counting how often word sequences appeared in text (n-grams).", "studyGuideAnchor": "pre-transformer-nlp-n-grams", "uid": "1vaaxum3bxub6" }, { "id": "f-rnn-lstm", "shape": "fact", "tags": [ "history", "pre-transformer" ], "title": "RNNs and LSTMs", "body": "Recurrent Neural Networks (RNNs) and their improved variant, Long Short-Term Memory (LSTM) networks, processed text sequentially, keeping a hidden 'memory' of earlier words. They dominated NLP from roughly 2014–2017 but struggled with long sequences due to vanishing gradients.", "illustration": { "imageSearchTerm": "looping chain memory network", "imagePrompt": "An abstract looping chain of glowing circular nodes feeding into each other in sequence, the glow fading toward the earliest nodes, evoking a memory that weakens over a long sequence.", "alt": "RNNs and LSTMs", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/f-rnn-lstm.jpg", "credit": "Pexels · Magda Ehlers", "creditUrl": "https://www.pexels.com/photo/marina-scenic-view-with-docked-boats-at-twilight-32565866/" }, "studyGuideAnchor": "rnns-and-lstms", "curatedDistractors": [ "Convolutional Neural Networks", "Feedforward Neural Networks", "Generative Adversarial Networks" ], "uid": "67tz6cwty0kk" }, { "id": "f-word2vec", "shape": "fact", "tags": [ "history", "pre-transformer", "embeddings" ], "title": "word2vec (2013)", "body": "Mikolov and colleagues introduced word2vec, which learned dense vector representations of words from large text. It famously showed that vector arithmetic captured meaning (king − man + woman ≈ queen), making it a foundational embedding technique for all later NLP.", "illustration": { "imageSearchTerm": "floating word vectors arithmetic", "imagePrompt": "Abstract glowing rectangular tokens floating in three-dimensional space, connected by directional arrows, evoking discrete units of meaning combined through vector arithmetic, no legible text or letters.", "alt": "word2vec (2013)", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/f-word2vec.jpg", "credit": "Pexels · Markus Winkler", "creditUrl": "https://www.pexels.com/photo/a-scrabble-board-with-the-letters-a-and-a-18548425/" }, "studyGuideAnchor": "word2vec-2013", "uid": "4md1xs16q6lh4" }, { "id": "f-seq2seq", "shape": "fact", "tags": [ "history", "pre-transformer", "attention" ], "title": "seq2seq and attention (2014–2015)", "body": "Sutskever's sequence-to-sequence model used an encoder–decoder LSTM to translate sentences. Bahdanau then added an attention mechanism that let the decoder 'look back' at all encoder states, sharply improving translation — and previewing the self-attention at the heart of the Transformer.", "illustration": { "imageSearchTerm": "encoder decoder attention diagram", "imagePrompt": "An abstract diagram of two glowing towers of nodes connected by multiple curved beams of light reaching back across all levels, evoking an attention mechanism linking every part of an encoded sequence, no text.", "alt": "seq2seq and attention (2014–2015)", "depictable": false, "credit": "AI-generated (gpt-image-1.5)", "url": "https://cdn.recurxive.com/packs/llm-101/images/f-seq2seq.webp" }, "studyGuideAnchor": "seq2seq-and-attention-2014-2015", "uid": "1ino1841nnyauo" }, { "id": "num-transformer-year", "shape": "numeric", "tags": [ "history", "transformer" ], "prompt": { "modality": "text", "value": "Year the Transformer paper 'Attention Is All You Need' was published" }, "value": 2017, "unit": "year", "tolerance": 0, "uid": "idritysz8mym" }, { "id": "pair-attention-paper", "shape": "pair", "tags": [ "history", "transformer" ], "sideA": { "modality": "text", "value": "“Attention Is All You Need” (2017)" }, "sideB": { "modality": "text", "value": "Vaswani et al. (Google) — introduced the Transformer, replacing recurrent layers entirely with self-attention; the architectural foundation of all modern LLMs", "short": "Introduced Transformer model using self-attention" }, "uid": "1vj2qqw10cxwdg" }, { "id": "f-bert", "shape": "fact", "tags": [ "history", "bert" ], "title": "BERT (2018): bidirectional pretraining", "body": "Devlin and colleagues at Google released BERT in October 2018. Unlike left-to-right models, BERT was pretrained to predict masked words using context from both sides of a sentence, setting new state-of-the-art results on eleven NLP benchmarks.", "illustration": { "imageSearchTerm": "sentence context both directions", "imagePrompt": "An abstract row of glowing rectangular blocks representing a sequence of words, with one blank block in the middle, light streaming into it from both the left and right sides simultaneously, evoking bidirectional context, no legible text or letters.", "alt": "BERT (2018): bidirectional pretraining", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/f-bert.jpg", "credit": "Pexels · Matheus Bertelli", "creditUrl": "https://www.pexels.com/photo/chat-gpt-on-computer-screen-16027820/" }, "studyGuideAnchor": "bert-2018-bidirectional-pretraining", "uid": "dqf9v73xs5et" }, { "id": "f-gpt1", "shape": "fact", "tags": [ "history", "gpt" ], "title": "GPT-1 (2018): the first GPT", "body": "OpenAI released GPT-1 in 2018 with 117 million parameters, trained on BooksCorpus. It showed that a generative pretrained Transformer, fine-tuned on downstream tasks, could match task-specific models — validating the pretraining paradigm that everything since has built on.", "illustration": { "imageSearchTerm": "neural network stack books", "imagePrompt": "An abstract glowing neural network emerging from a tall stack of closed books, evoking a language model pretrained by reading a large collection of texts, no legible text.", "alt": "A glowing microprocessor, evoking AI computation", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/f-gpt1.jpg", "credit": "Pexels · Steve A Johnson", "creditUrl": "https://www.pexels.com/photo/abstract-image-of-a-microchip-with-heatmap-colors-28767589/" }, "studyGuideAnchor": "gpt-1-2018-the-first-gpt", "uid": "tqiu461r26dgy" }, { "id": "f-gpt2", "shape": "fact", "tags": [ "history", "gpt" ], "title": "GPT-2 (2019): scaling surprises", "body": "GPT-2 scaled to 1.5 billion parameters, trained on 40 GB of web text. Its fluent, coherent generation was striking enough that OpenAI initially withheld the full model over misuse concerns — an early signal of the field's growing awareness of capability risks.", "illustration": { "imageSearchTerm": "vault door glowing network", "imagePrompt": "An abstract glowing neural network partially enclosed behind a half-open vault door, evoking a powerful model whose full release was cautiously withheld, no text.", "alt": "GPT-2 (2019): scaling surprises", "depictable": false, "credit": "AI-generated (gpt-image-1.5)", "url": "https://cdn.recurxive.com/packs/llm-101/images/f-gpt2.webp" }, "studyGuideAnchor": "gpt-2-2019-scaling-surprises", "uid": "1lssb1oi5v42w" }, { "id": "f-gpt3", "shape": "fact", "tags": [ "history", "gpt", "few-shot" ], "title": "GPT-3 (2020): the few-shot giant", "body": "OpenAI released GPT-3 in 2020 with 175 billion parameters — over 10× larger than any prior dense model. Its key finding: given only a few examples in the prompt ('few-shot'), it could perform tasks it was never explicitly trained on, demonstrating in-context learning at scale.", "illustration": { "imagePrompt": "A vast, dense glowing neural-network sphere dwarfing a few small scattered points of light beside it, evoking a giant model that learns new tasks from just a handful of examples, no text or numbers.", "imageSearchTerm": "massive glowing neural network", "alt": "Rows of servers in a modern AI data center", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/f-gpt3.jpg", "credit": "Pexels · panumas nikhomkhai", "creditUrl": "https://www.pexels.com/photo/data-center-server-racks-with-active-equipment-37730212/" }, "factVariant": "image-heavy", "imageCaption": "175 billion parameters — 10× bigger than anything before. Show it a few examples and it learned tasks it was never trained on.", "studyGuideAnchor": "gpt-3-2020-the-few-shot-giant", "uid": "18xuviz6r71yd" }, { "id": "pair-kaplan", "shape": "pair", "tags": [ "history", "scaling-laws" ], "sideA": { "modality": "text", "value": "Kaplan scaling laws (2020)" }, "sideB": { "modality": "text", "value": "Showed that LLM loss falls predictably as a power-law with model size, dataset size, and compute — justifying the strategy of building ever-larger models", "short": "LLM loss decreases predictably with size" }, "uid": "mnm6sseytxk0" }, { "id": "pair-chinchilla", "shape": "pair", "tags": [ "history", "scaling-laws" ], "sideA": { "modality": "text", "value": "Chinchilla scaling laws (2022)" }, "sideB": { "modality": "text", "value": "DeepMind showed prior large models were undertrained; compute-optimal training needs about 20 tokens of data per parameter — shifting the field toward smaller models trained on far more data", "short": "Optimal training requires more data per parameter" }, "uid": "rm2vi6pthvnq" }, { "id": "f-instructgpt", "shape": "fact", "tags": [ "history", "rlhf" ], "title": "InstructGPT and RLHF (2022)", "body": "OpenAI showed that fine-tuning GPT-3 with Reinforcement Learning from Human Feedback (RLHF) sharply improved helpfulness and reduced harmful output. Remarkably, a 1.3B InstructGPT was preferred by human raters over the raw 175B GPT-3 — bigger is not automatically better.", "illustration": { "imageSearchTerm": "small bright light large", "imagePrompt": "A small, brightly glowing compact sphere of light standing out clearly next to a much larger but dim, murky sphere, evoking a smaller refined model outperforming a larger unrefined one, no text.", "alt": "InstructGPT and RLHF (2022)", "depictable": false }, "studyGuideAnchor": "instructgpt-and-rlhf-2022", "uid": "55n05518e0zgf" }, { "id": "f-chatgpt-launch", "shape": "fact", "tags": [ "history", "chatgpt" ], "title": "ChatGPT launches (Nov 30, 2022)", "body": "OpenAI released ChatGPT as a free public demo on November 30, 2022, powered by a GPT-3.5-class model fine-tuned with RLHF. It reached 1 million users in five days and about 100 million monthly users within two months — the fastest-growing consumer app in history at the time.", "illustration": { "imagePrompt": "Abstract glowing speech-bubble shapes multiplying and spreading rapidly outward across a dark background like ripples, evoking a chat product going viral, no text or logos.", "imageSearchTerm": "chat bubbles spreading rapidly", "alt": "A smartphone showing an AI chatbot conversation", "depictable": false, "credit": "Unsplash · kuu akura · Unsplash License", "creditUrl": "https://unsplash.com/photos/speech-bubbles-on-translucent-screen-pnK6Q-QTHM4", "url": "https://cdn.recurxive.com/packs/llm-101/images/f-chatgpt-launch.webp" }, "factVariant": "image-heavy", "imageCaption": "1 million users in 5 days. 100 million in two months. The fastest-growing consumer app in history.", "studyGuideAnchor": "chatgpt-launches-nov-30-2022", "uid": "1hmjd0wdykxns" }, { "id": "f-gpt4", "shape": "fact", "tags": [ "history", "gpt-4" ], "title": "GPT-4 (March 2023)", "body": "OpenAI released GPT-4 on March 14, 2023. It was multimodal (accepting image and text input), scored at or above human level on many professional exams, and launched with an 8,000-token context window, with a 32K variant following soon after.", "illustration": { "imageSearchTerm": "text and image merging", "imagePrompt": "An abstract glowing network where a stream of flowing horizontal light lines and a stream of pixel-like glowing squares merge into one unified flow, evoking a multimodal model processing both language and images, no legible text.", "alt": "GPT-4 (March 2023)", "depictable": false, "credit": "NASA Hubble (Openverse) · by 2.0", "creditUrl": "https://www.flickr.com/photos/144614754@N02/52201244657", "url": "https://cdn.recurxive.com/packs/llm-101/images/f-gpt4.webp" }, "studyGuideAnchor": "gpt-4-march-2023", "uid": "1rnnezvjc40st" }, { "id": "f-llama", "shape": "fact", "tags": [ "history", "open-weights", "meta" ], "title": "The open-weight wave: LLaMA (2023)", "body": "Meta released LLaMA in February 2023 — 7B to 65B-parameter models trained only on publicly available data. LLaMA-13B outperformed the 175B GPT-3 on most benchmarks, proving smaller, data-efficient models could rival proprietary giants and igniting a huge open-source ecosystem.", "illustration": { "imagePrompt": "An abstract compact glowing network core branching outward into many smaller independent networks shaped by different hands, evoking an open, freely shared model spawning a wide community of derivative projects, no text or animals.", "imageSearchTerm": "open network branching community", "alt": "Open-source code displayed on a computer screen", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/f-llama.jpg", "credit": "Pexels · Digital Buggu", "creditUrl": "https://www.pexels.com/photo/monitor-displaying-computer-application-374559/" }, "factVariant": "image-heavy", "imageCaption": "Meta's 13B model beat OpenAI's 175B giant on most benchmarks — and shipped its weights to everyone.", "studyGuideAnchor": "the-open-weight-wave-llama-2023", "uid": "1rztfyc1v6wk54" }, { "id": "f-mistral", "shape": "fact", "tags": [ "history", "open-weights", "mistral" ], "title": "Mistral AI (2023)", "body": "Former DeepMind and Meta researchers founded Mistral AI in Paris in 2023. Its Mistral 7B (September 2023), released under Apache 2.0, became the most-downloaded open model within weeks, and Mixtral 8×7B matched GPT-3.5 using a Mixture-of-Experts design.", "illustration": { "imageSearchTerm": "multiple expert networks routing", "imagePrompt": "An abstract diagram of several distinct glowing expert modules with a central router directing incoming light toward only a few of them at a time, evoking a mixture-of-experts model architecture, no text.", "alt": "Mistral AI (2023)", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/f-mistral.jpg", "credit": "Pexels · Google DeepMind", "creditUrl": "https://www.pexels.com/photo/symbolic-graphic-representation-25626431/" }, "studyGuideAnchor": "mistral-ai-2023", "curatedDistractors": [ "Former OpenAI", "Former Anthropic", "Former Microsoft" ], "uid": "jpqlme1mrdvwe" }, { "id": "f-claude-gemini", "shape": "fact", "tags": [ "history", "anthropic", "google" ], "title": "Claude and Gemini", "body": "Anthropic released Claude (2023), built with Constitutional AI and RLAIF alignment, while Google released Gemini (December 2023) as its multimodal flagship, replacing Bard and developed by Google DeepMind. Both are major non-OpenAI frontier model families.", "illustration": { "imageSearchTerm": "Anthropic Google AI models", "imagePrompt": "Two distinct streams of glowing abstract light in different colors flowing side by side, representing two separate AI systems, with no symbols or logos.", "alt": "Claude and Gemini", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/f-claude-gemini.jpg", "credit": "Pexels · ILOVESwitzerland", "creditUrl": "https://www.pexels.com/photo/two-bridges-on-the-rocky-cliff-7938540/" }, "studyGuideAnchor": "claude-and-gemini", "curatedDistractors": [ "supervised fine-tuning", "reward modeling", "instruction tuning" ], "uid": "1uyyu5sneo0b4" }, { "id": "def-token", "shape": "definition", "tags": [ "text-units", "mechanics", "tokenization" ], "term": { "modality": "text", "value": "token" }, "definition": { "modality": "text", "value": "The basic unit of text an LLM processes — on average about three-quarters of a word; 'ChatGPT' is one token, while 'unhappy' may split into 'un' + 'happy'. It matters because everything — context limits, pricing, and speed — is measured in tokens, not words or characters." }, "uid": "9iofe6wjsdt2" }, { "id": "def-tokenization", "shape": "definition", "tags": [ "text-units", "mechanics", "tokenization" ], "term": { "modality": "text", "value": "tokenization" }, "definition": { "modality": "text", "value": "The process of splitting raw text into tokens before feeding it to a model, using algorithms such as Byte-Pair Encoding (BPE). It matters because the vocabulary (usually 32K–100K tokens) determines what the model 'sees' and how it handles rare or foreign words." }, "uid": "2hz9a01oc84cc" }, { "id": "def-embedding-llm", "shape": "definition", "tags": [ "text-units", "mechanics", "embeddings" ], "term": { "modality": "text", "value": "embedding" }, "definition": { "modality": "text", "value": "A list of numbers (a vector) that represents a token's meaning in a high-dimensional space, where tokens with similar meanings sit close together. It matters because embeddings are how a model turns discrete text symbols into continuous math it can compute over." }, "illustration": { "imagePrompt": "Abstract 3D visualization of clustered points of light floating in dark space, representing a high-dimensional vector embedding space, blue and teal glow", "imageSearchTerm": "abstract data points 3d space", "alt": "Clustered glowing points representing a vector space", "credit": "Unsplash · Buddha Elemental 3D · Unsplash License", "creditUrl": "https://unsplash.com/photos/a-black-and-blue-background-with-a-circular-design-B2PvQzNyDrA", "url": "https://cdn.recurxive.com/packs/llm-101/images/def-embedding-llm.webp" }, "uid": "1g4ognx1l245un" }, { "id": "def-Transformer", "shape": "definition", "tags": [ "model-internals", "mechanics", "transformer" ], "term": { "modality": "text", "value": "Transformer" }, "definition": { "modality": "text", "value": "A neural network architecture that processes all tokens in a sequence at once, in parallel, using self-attention layers, rather than one token at a time like an RNN. It matters because that parallelism makes Transformers vastly faster to train on GPUs, enabling a scale RNNs could never reach." }, "illustration": { "imagePrompt": "Abstract photograph of a glowing neural network of interconnected nodes and light trails on a dark background, representing artificial intelligence architecture, blue and purple tones", "imageSearchTerm": "neural network abstract glowing", "alt": "An abstract glowing neural network of connected nodes", "credit": "Pexels · Merlin Lightpainting", "creditUrl": "https://www.pexels.com/photo/digital-art-in-close-up-shot-14314636/", "url": "https://cdn.recurxive.com/packs/llm-101/def-Transformer.jpg" }, "uid": "10ue1hjr4bmbh" }, { "id": "def-self-attention", "shape": "definition", "tags": [ "model-internals", "mechanics", "attention" ], "term": { "modality": "text", "value": "self-attention" }, "definition": { "modality": "text", "value": "A mechanism where each token looks at every other token in the context and learns how much to attend to (weight) each one when forming its own representation. It matters because it lets the model link a pronoun to its noun or a question to its answer, no matter how far apart they sit in the text." }, "illustration": { "imagePrompt": "Abstract photograph of many glowing lines connecting words or nodes across a dark canvas, some links brighter than others, representing attention weights", "imageSearchTerm": "abstract connections network lines", "alt": "Glowing lines of varying brightness connecting nodes", "credit": "Pexels · U.Lucas Dubé-Cantin", "creditUrl": "https://www.pexels.com/photo/abstract-orange-lights-10325707/", "url": "https://cdn.recurxive.com/packs/llm-101/def-self-attention.jpg" }, "uid": "zqn2t1cbl60v" }, { "id": "def-context-window", "shape": "definition", "tags": [ "model-internals", "mechanics", "context-window" ], "term": { "modality": "text", "value": "context window" }, "definition": { "modality": "text", "value": "The maximum number of tokens a model can see at once — its working memory for a single conversation or document. It matters because anything outside the window is invisible to the model; larger windows allow longer documents, conversations, and multi-step tasks." }, "uid": "tfthh8qwyzhk" }, { "id": "def-parameters", "shape": "definition", "tags": [ "model-internals", "mechanics", "parameters" ], "term": { "modality": "text", "value": "parameters (weights)" }, "definition": { "modality": "text", "value": "The millions or billions of numerical values inside a model's layers, learned during training, that encode everything the model 'knows.' It matters because parameter count is the main proxy for a model's capacity — more parameters generally means more capability, up to a point." }, "uid": "yj0davo6zljx" }, { "id": "def-next-token", "shape": "definition", "tags": [ "generation", "mechanics", "pretraining" ], "term": { "modality": "text", "value": "next-token prediction" }, "definition": { "modality": "text", "value": "The core objective: given all previous tokens, predict the single most probable next token, repeated over and over to generate text. It matters because this deceptively simple task, applied at massive scale, is what gives LLMs their broad language ability." }, "uid": "1civab3q842t" }, { "id": "def-autoregressive", "shape": "definition", "tags": [ "generation", "mechanics" ], "term": { "modality": "text", "value": "autoregressive generation" }, "definition": { "modality": "text", "value": "Producing text one token at a time, where each new token is chosen based on all prior tokens — including the ones the model just generated. It matters because the model can't go back and edit; it commits to each token in sequence, which is why small errors can compound." }, "uid": "1kct55a1ihic8y" }, { "id": "def-logits", "shape": "definition", "tags": [ "generation", "mechanics", "sampling" ], "term": { "modality": "text", "value": "logits" }, "definition": { "modality": "text", "value": "The raw, unnormalized scores a model assigns to every possible next token, converted into a probability distribution by the softmax function. It matters because understanding logits clarifies what 'temperature' and other sampling knobs are actually adjusting." }, "uid": "44ufu9ciddq" }, { "id": "def-temperature", "shape": "definition", "tags": [ "behavior", "mechanics", "sampling" ], "term": { "modality": "text", "value": "temperature" }, "definition": { "modality": "text", "value": "A sampling knob (typically 0–2) that scales the logits: low temperature makes output more deterministic and focused; high temperature makes it more random and creative. It matters because it is the primary dial for controlling how predictable or varied a model's output is." }, "uid": "z0hjmsjarui4" }, { "id": "def-hallucination", "shape": "definition", "tags": [ "behavior", "mechanics", "hallucination" ], "term": { "modality": "text", "value": "hallucination" }, "definition": { "modality": "text", "value": "When an LLM generates fluent, confident-sounding text that is factually wrong or entirely fabricated, grounded in no real source. It matters because users often can't tell a hallucination from an accurate answer, making verification essential for high-stakes use." }, "illustration": { "imagePrompt": "Abstract photograph of a glitching, fragmented digital portrait dissolving into colorful pixels and noise, representing AI fabrication, surreal mood", "imageSearchTerm": "digital glitch abstract distortion", "alt": "An abstract glitching, fragmented digital image", "credit": "Pexels · Alexey Demidov · Pexels License", "creditUrl": "https://www.pexels.com/photo/a-distorted-photograph-of-a-person-10219028/", "url": "https://cdn.recurxive.com/packs/llm-101/images/def-hallucination.webp" }, "uid": "1anronr1dazilp" }, { "id": "f-why-hallucinate", "shape": "fact", "tags": [ "mechanics", "hallucination" ], "title": "Why hallucinations happen", "body": "LLMs are trained to produce plausible text, not to retrieve verified facts. With no internal fact-checker, they interpolate from patterns in their training data — so they 'fill in' convincing but wrong details the same way they fill in correct ones. Grounding techniques like RAG are the main mitigation.", "illustration": { "imageSearchTerm": "pattern interpolation training data", "imagePrompt": "A dense web of glowing interconnected nodes where some connections blur and dissolve into smooth, uncertain shapes, suggesting gaps being filled with guesses.", "alt": "Why hallucinations happen", "credit": "Pexels · Google DeepMind · Pexels License", "creditUrl": "https://www.pexels.com/photo/shapes-on-a-white-background-25626507/", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/images/f-why-hallucinate.webp" }, "factVariant": "image-heavy", "imageCaption": "It has no fact-checker inside. A model fills in a wrong detail the exact same way it fills in a right one.", "uid": "67kgys68dkp0" }, { "id": "def-emergence", "shape": "definition", "tags": [ "behavior", "mechanics", "emergence" ], "term": { "modality": "text", "value": "emergent abilities" }, "definition": { "modality": "text", "value": "Capabilities that appear in large models but are absent in smaller ones of the same type — not predictable by smoothly extrapolating from smaller scale. Examples include multi-step arithmetic and chain-of-thought reasoning. It matters because emergence means capability jumps can surprise researchers, though there is debate over whether it is a true phase transition or an artifact of how abilities are measured." }, "uid": "qqjdv612fk6ea" }, { "id": "def-pretraining", "shape": "definition", "tags": [ "before-alignment", "training", "pretraining" ], "term": { "modality": "text", "value": "pretraining" }, "definition": { "modality": "text", "value": "Stage one: the model trains on a massive, diverse text corpus (trillions of tokens from the web, books, and code) using self-supervised next-token prediction, with no human labels. This produces a 'base model.' It matters because pretraining is where the vast majority of compute is spent and where the model acquires its broad world knowledge." }, "illustration": { "imagePrompt": "Photograph of a vast library of books receding into the distance with a subtle digital overlay of flowing text, representing training data, warm light", "imageSearchTerm": "library books knowledge data", "alt": "A vast library of books representing training data", "credit": "Open Grid Scheduler / Grid Engine (Openverse) · cc0 1.0", "creditUrl": "https://www.flickr.com/photos/29155878@N03/16915765068", "url": "https://cdn.recurxive.com/packs/llm-101/images/def-pretraining.webp" }, "uid": "xs7skv99r3st" }, { "id": "def-base-model", "shape": "definition", "tags": [ "before-alignment", "training", "pretraining" ], "term": { "modality": "text", "value": "base model" }, "definition": { "modality": "text", "value": "A pretrained-only LLM that has learned to predict text but hasn't been tuned to follow instructions or be helpful; it simply continues a prompt, often in unexpected ways. It matters because it is the foundation every chat model is built on — and is not safe to deploy directly to general users." }, "uid": "s4asy61vq233u" }, { "id": "def-sft", "shape": "definition", "tags": [ "before-alignment", "training", "sft" ], "term": { "modality": "text", "value": "supervised fine-tuning (SFT)" }, "definition": { "modality": "text", "value": "Stage two: the base model is trained on a curated set of (prompt, ideal-response) pairs written or selected by humans, teaching it to follow instructions rather than just continue text. It matters because SFT is what turns a raw text-predictor into an assistant that answers questions in a useful format." }, "uid": "1ii9top1z0c6o3" }, { "id": "def-rlhf", "shape": "definition", "tags": [ "alignment", "training", "rlhf" ], "term": { "modality": "text", "value": "RLHF" }, "definition": { "modality": "text", "value": "Reinforcement Learning from Human Feedback — stage three: human raters compare pairs of outputs, those preferences train a reward model, and the LLM is then fine-tuned with reinforcement learning to maximize the reward model's score. Introduced in OpenAI's InstructGPT work. It matters because RLHF is what makes ChatGPT-style models markedly more helpful and aligned than raw base models." }, "uid": "1rt2fbf4ckyh" }, { "id": "def-reward-model", "shape": "definition", "tags": [ "alignment", "training", "rlhf" ], "term": { "modality": "text", "value": "reward model" }, "definition": { "modality": "text", "value": "A separate neural network trained on human preference data that scores how good a model response is, used as the optimization target during the RL phase of RLHF. It matters because the reward model's quality caps the quality of alignment — a flawed one teaches the LLM to 'game' it rather than genuinely improve." }, "uid": "19nx5fe1k4caqm" }, { "id": "def-rlaif", "shape": "definition", "tags": [ "alignment", "training", "rlaif" ], "term": { "modality": "text", "value": "RLAIF" }, "definition": { "modality": "text", "value": "Reinforcement Learning from AI Feedback — a variant of RLHF where an AI model, rather than human raters, generates the preference labels used to train the reward model. It matters because it enables alignment at scales where human labeling would become the bottleneck." }, "uid": "c5kt6bmx0ia9" }, { "id": "def-cai", "shape": "definition", "tags": [ "training", "rlaif", "anthropic" ], "term": { "modality": "text", "value": "Constitutional AI (CAI)" }, "definition": { "modality": "text", "value": "Anthropic's alignment method in which the model critiques and revises its own outputs against a written set of principles (a 'constitution'), then trains on those AI-generated preferences via RLAIF. It matters because it reduces reliance on human harm-labeling and makes the model's guiding principles explicit and auditable." }, "uid": "155uxwx19w7huj" }, { "id": "def-instruct-model", "shape": "definition", "tags": [ "training", "sft", "rlhf" ], "term": { "modality": "text", "value": "instruct / chat model" }, "definition": { "modality": "text", "value": "An LLM that has gone through SFT and RLHF (or equivalent) on top of a base model, making it follow instructions, refuse harmful requests, and converse naturally — for example, ChatGPT. It matters because this is the type of model most people interact with; base models and chat models behave very differently." }, "uid": "mont3d1dh6i3v" }, { "id": "f-train-vs-infer", "shape": "fact", "tags": [ "training", "inference" ], "title": "Training vs. inference — the key distinction", "body": "TRAINING updates a model's weights with gradient descent over billions of examples — done once or rarely, on massive GPU clusters over weeks or months. INFERENCE runs the finished, frozen model on a prompt — cheap per call, done billions of times a day. This is why fine-tuning is costly but querying an API is cheap.", "illustration": { "imageSearchTerm": "GPU cluster data center", "imagePrompt": "Rows of glowing server racks filled with GPUs in a large data center, representing the massive computing power used to train a model.", "alt": "Server racks in a data center", "credit": "Pexels · panumas nikhomkhai · Pexels License", "creditUrl": "https://www.pexels.com/photo/box-server-illuminated-on-blue-17489160/", "depictable": true, "url": "https://cdn.recurxive.com/packs/llm-101/images/f-train-vs-infer.webp" }, "uid": "qldk2ky66uxo" }, { "id": "def-inference-llm", "shape": "definition", "tags": [ "training", "inference" ], "term": { "modality": "text", "value": "inference" }, "definition": { "modality": "text", "value": "Running a trained, frozen model on a new input to produce an output, with no weights changed. It matters because inference is what happens every time you send a message; the model's knowledge is locked at training time unless supplemented by tools or RAG." }, "curatedDistractors": [ "pretraining", "fine-tuning", "RLHF" ], "uid": "65q7tn127xk9" }, { "id": "def-flops", "shape": "definition", "tags": [ "training", "compute" ], "term": { "modality": "text", "value": "FLOPs" }, "definition": { "modality": "text", "value": "Floating Point Operations — a measure of computational work. Training a large LLM takes on the order of 10^23 to 10^25 FLOPs, equal to thousands of GPU-hours and millions of dollars of compute. It matters because FLOPs budgets determine what model sizes an organization can afford, and scaling laws relate FLOPs to expected performance." }, "uid": "1gkh9ha1txomau" }, { "id": "def-prompt", "shape": "definition", "tags": [ "prompt-basics", "prompting" ], "term": { "modality": "text", "value": "prompt" }, "definition": { "modality": "text", "value": "The input sent to an LLM, typically a system message (behavior instructions), zero or more prior turns, and the current user message. It matters because the prompt is the only way to steer a frozen model, which makes prompt engineering a high-leverage skill." }, "uid": "sglujh11yabkb" }, { "id": "def-system-prompt", "shape": "definition", "tags": [ "prompt-basics", "prompting" ], "term": { "modality": "text", "value": "system prompt" }, "definition": { "modality": "text", "value": "The special instruction block, usually hidden from the end user, that sets the model's persona, constraints, and task context at the start of a conversation. It matters because it is where developers configure a model's behavior for their specific application." }, "uid": "4t9gg25tgr96" }, { "id": "def-zero-shot", "shape": "definition", "tags": [ "prompt-basics", "prompting", "zero-shot" ], "term": { "modality": "text", "value": "zero-shot prompting" }, "definition": { "modality": "text", "value": "Asking a model to perform a task with no examples — just a plain instruction, such as 'Translate this to French.' It matters because it tests a model's general capability; large models handle zero-shot well, while smaller ones often need examples." }, "uid": "1im4g7rc46vrp" }, { "id": "def-few-shot", "shape": "definition", "tags": [ "prompt-techniques", "prompting", "few-shot" ], "term": { "modality": "text", "value": "few-shot prompting" }, "definition": { "modality": "text", "value": "Providing one to several worked examples in the prompt before asking the model to handle a new case. It matters because it can sharply improve performance on structured tasks with no weight updates — teaching the task format inside the context window." }, "uid": "12k1f2ejcbrtq" }, { "id": "def-icl", "shape": "definition", "tags": [ "prompt-techniques", "prompting", "in-context-learning" ], "term": { "modality": "text", "value": "in-context learning" }, "definition": { "modality": "text", "value": "An LLM's ability to adapt its behavior from examples or instructions in the prompt, without changing any weights. It matters because it blurs the line between 'prompting' and 'training' in people's minds — but no learning persists once the context window ends." }, "uid": "nfk1npge3p93" }, { "id": "def-cot", "shape": "definition", "tags": [ "prompt-techniques", "prompting", "chain-of-thought" ], "term": { "modality": "text", "value": "chain-of-thought (CoT) prompting" }, "definition": { "modality": "text", "value": "Prompting the model to reason step by step (for example, 'Let's think step by step') before giving a final answer, via instruction or worked examples. It matters because chain-of-thought sharply improves performance on arithmetic and multi-step reasoning, especially in larger models." }, "uid": "1ci524q1ldc7ti" }, { "id": "def-rag", "shape": "definition", "tags": [ "rag" ], "term": { "modality": "text", "value": "RAG (Retrieval-Augmented Generation)" }, "definition": { "modality": "text", "value": "A pattern in which relevant documents are retrieved from an external store and inserted into the prompt as context before the model answers. Introduced by Lewis et al. (2020). It matters because it gives a model access to fresh, proprietary, or specific knowledge without retraining it." }, "illustration": { "imagePrompt": "Photograph of glowing database server racks connected by light streams to a central node, representing information retrieval, dark technological background", "imageSearchTerm": "database servers technology abstract", "alt": "Database servers connected by streams of light", "credit": "AI-generated (gpt-image-1.5)", "url": "https://cdn.recurxive.com/packs/llm-101/images/def-rag.webp" }, "uid": "18q0gpbjt13u9" }, { "id": "def-vector-db", "shape": "definition", "tags": [ "rag", "embeddings" ], "term": { "modality": "text", "value": "vector database" }, "definition": { "modality": "text", "value": "A specialized database that stores document embeddings and supports fast similarity search, returning the most semantically relevant chunks for a query. Examples include Pinecone, Weaviate, and Chroma. It matters because it is the retrieval backend that makes RAG systems work at scale." }, "uid": "csgthvgf03w1" }, { "id": "def-chunking", "shape": "definition", "tags": [ "rag" ], "term": { "modality": "text", "value": "chunking" }, "definition": { "modality": "text", "value": "Splitting source documents into smaller segments before embedding them for RAG, so retrieved passages fit the context window and stay semantically focused. It matters because chunk size and overlap are key knobs: too large wastes context, too small loses coherence." }, "uid": "v4a9um1rje44y" }, { "id": "proc-rag", "shape": "procedure", "tags": [ "rag", "procedure" ], "goal": "Build a basic RAG system", "steps": [ "Chunk your source documents into focused passages.", "Embed each chunk with an embedding model.", "Store the embeddings in a vector database.", "At query time, embed the user's question.", "Retrieve the top-k most similar chunks.", "Insert the retrieved chunks plus the question into the LLM prompt.", "Have the model generate an answer grounded in those chunks." ], "uid": "ds4d5s6egl5k" }, { "id": "f-prompt-rag-ft", "shape": "fact", "tags": [ "prompting", "rag", "fine-tuning", "comparison" ], "title": "Prompting vs. RAG vs. fine-tuning", "body": "PROMPTING: best for general tasks, formatting, persona, and zero/few-shot — no data needed and zero cost, but limited by context size. RAG: best when you need fresh or proprietary knowledge without touching weights, ideal for document Q&A. FINE-TUNING: best for persistent style, behavior, or domain adaptation baked into the model — expensive and data-hungry, but it persists across every call.", "illustration": { "imageSearchTerm": "prompting RAG fine-tuning tradeoffs", "imagePrompt": "Three separate glowing paths branching outward from a single origin point, each curving toward a different destination, representing three different strategies.", "alt": "Prompting vs. RAG vs. fine-tuning", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/f-prompt-rag-ft.jpg", "credit": "Pexels · Diana ✨", "creditUrl": "https://www.pexels.com/photo/close-up-of-violin-scroll-and-pegs-37210788/" }, "factVariant": "image-heavy", "imageCaption": "Prompt for free, RAG for fresh knowledge, fine-tune for behavior baked into the weights.", "studyGuideAnchor": "prompting-vs-rag-vs-fine-tuning", "uid": "e12zf3c8l2e1" }, { "id": "def-tool-use", "shape": "definition", "tags": [ "agents", "tool-use" ], "term": { "modality": "text", "value": "function calling / tool use" }, "definition": { "modality": "text", "value": "A structured way to let an LLM invoke external tools — APIs, calculators, databases — by emitting a well-formed call the host application runs, then feeding the result back to the model. It matters because it grounds the model in real-time data and gives it abilities, like exact math or search, that pure text generation lacks." }, "uid": "ctmhjr1k3i87x" }, { "id": "def-agent", "shape": "definition", "tags": [ "agents" ], "term": { "modality": "text", "value": "LLM agent" }, "definition": { "modality": "text", "value": "A system in which an LLM acts as a controller that plans, selects tools, takes actions, and observes results in a loop until a goal is met, rather than answering a single prompt. It matters because agents extend LLMs from one-shot answerers into multi-step problem solvers that can browse, write and run code, and manage files." }, "illustration": { "imagePrompt": "Photograph of a sleek humanoid robot hand interacting with a glowing digital interface, representing autonomous AI agents, cool studio lighting", "imageSearchTerm": "robot automation AI", "alt": "A robot hand interacting with a glowing interface", "credit": "AI-generated (gpt-image-1.5)", "url": "https://cdn.recurxive.com/packs/llm-101/images/def-agent.webp" }, "uid": "gknvnihvuolq" }, { "id": "pair-openai", "shape": "pair", "tags": [ "us-frontier", "companies" ], "sideA": { "modality": "text", "value": "OpenAI" }, "sideB": { "modality": "text", "value": "Flagship family: GPT (GPT-3.5, GPT-4, GPT-4o and successors); creator of ChatGPT; founded 2015; long the frontier-setter in language-model capability", "short": "Creator of GPT models and ChatGPT" }, "uid": "1mapm4t81p3iv" }, { "id": "pair-anthropic", "shape": "pair", "tags": [ "us-frontier", "companies" ], "sideA": { "modality": "text", "value": "Anthropic" }, "sideB": { "modality": "text", "value": "Flagship family: Claude; founded 2021 by ex-OpenAI researchers Dario and Daniela Amodei; known for Constitutional AI and safety-focused research", "short": "Focuses on AI safety and Constitutional AI" }, "uid": "1plyqy81f715q4" }, { "id": "pair-deepmind", "shape": "pair", "tags": [ "intl-other", "companies" ], "sideA": { "modality": "text", "value": "Google DeepMind" }, "sideB": { "modality": "text", "value": "Flagship family: Gemini (successor to Bard and PaLM); led by Demis Hassabis; also behind AlphaGo and AlphaFold; central to Google's AI strategy", "short": "Develops Gemini and known for AlphaGo" }, "uid": "19doxplr3bthf" }, { "id": "pair-meta", "shape": "pair", "tags": [ "us-frontier", "companies" ], "sideA": { "modality": "text", "value": "Meta AI" }, "sideB": { "modality": "text", "value": "Flagship family: Llama (LLaMA, Llama 2, Llama 3); open-weight models released freely for research and commercial use; the driving force in open-source LLMs", "short": "Releases open-weight Llama models for research" }, "uid": "w74n8n11sz9v1" }, { "id": "pair-mistral", "shape": "pair", "tags": [ "intl-other", "companies" ], "sideA": { "modality": "text", "value": "Mistral AI" }, "sideB": { "modality": "text", "value": "Flagship models: Mistral 7B, Mixtral 8×7B, Mistral Large; Paris-based, founded 2023; known for efficiency, open weights under Apache 2.0, and European AI sovereignty", "short": "Efficient AI models with open weights from Paris" }, "uid": "1a7rov814zbgn0" }, { "id": "pair-xai", "shape": "pair", "tags": [ "us-frontier", "companies" ], "sideA": { "modality": "text", "value": "xAI" }, "sideB": { "modality": "text", "value": "Flagship family: Grok; founded by Elon Musk in 2023; integrated with X (formerly Twitter) and positioned as 'maximally truth-seeking'", "short": "Founded by Elon Musk, focuses on truth-seeking" }, "uid": "16rwcgojhuqd4" }, { "id": "pair-cohere", "shape": "pair", "tags": [ "intl-other", "companies" ], "sideA": { "modality": "text", "value": "Cohere" }, "sideB": { "modality": "text", "value": "Flagship products: Command, Embed, and Rerank; an enterprise-focused provider known for RAG and retrieval tooling; co-founded by Aidan Gomez, a co-author of 'Attention Is All You Need'", "short": "Enterprise provider known for RAG tools" }, "uid": "1ub5kob1uvjhe5" }, { "id": "pair-vaswani", "shape": "pair", "tags": [ "transformer-era", "people", "transformer" ], "sideA": { "modality": "text", "value": "Ashish Vaswani" }, "sideB": { "modality": "text", "value": "Lead author of 'Attention Is All You Need' (2017), the Transformer paper, written at Google Brain; later co-founded Adept AI", "short": "Lead author of the Transformer paper" }, "uid": "1dekvaf4o9de1" }, { "id": "pair-hinton", "shape": "pair", "tags": [ "godfathers", "people", "deep-learning" ], "sideA": { "modality": "text", "value": "Geoffrey Hinton" }, "sideB": { "modality": "text", "value": "'Godfather of Deep Learning'; 2018 Turing Award co-recipient; PhD advisor to Ilya Sutskever; pioneered backpropagation; left Google in 2023 to speak freely about AI risk", "short": "Pioneered deep learning, left Google in 2023" }, "uid": "ycmqj61x2w03u" }, { "id": "pair-bengio", "shape": "pair", "tags": [ "godfathers", "people", "deep-learning" ], "sideA": { "modality": "text", "value": "Yoshua Bengio" }, "sideB": { "modality": "text", "value": "'Godfather of Deep Learning'; 2018 Turing Award co-recipient; professor at Université de Montréal and scientific director of Mila; a leading AI-safety advocate", "short": "AI-safety advocate and deep learning pioneer" }, "uid": "1x2mnjb5mb201" }, { "id": "pair-lecun", "shape": "pair", "tags": [ "godfathers", "people", "deep-learning" ], "sideA": { "modality": "text", "value": "Yann LeCun" }, "sideB": { "modality": "text", "value": "'Godfather of Deep Learning'; 2018 Turing Award co-recipient; Chief AI Scientist at Meta; pioneer of convolutional neural networks; notably skeptical of current LLM reasoning", "short": "Pioneer of CNNs, skeptical of LLM reasoning" }, "uid": "l6isvv1h9q8f9" }, { "id": "pair-sutskever-llm", "shape": "pair", "tags": [ "transformer-era", "people" ], "sideA": { "modality": "text", "value": "Ilya Sutskever" }, "sideB": { "modality": "text", "value": "Co-founder and former Chief Scientist of OpenAI; PhD student of Hinton; co-authored AlexNet (2012) and seq2seq (2014); co-founded Safe Superintelligence Inc. in 2024", "short": "Co-founder of OpenAI, known for AlexNet" }, "uid": "uf20v2bar6hm" }, { "id": "pair-karpathy", "shape": "pair", "tags": [ "transformer-era", "people" ], "sideA": { "modality": "text", "value": "Andrej Karpathy" }, "sideB": { "modality": "text", "value": "Founding member of OpenAI and former Director of AI at Tesla; founded Eureka Labs for AI education; a celebrated AI educator behind nanoGPT and popular tutorials", "short": "AI educator and former Tesla AI director" }, "uid": "1pjpl4hvifzg7" }, { "id": "pair-altman-llm", "shape": "pair", "tags": [ "openai-anthropic", "people" ], "sideA": { "modality": "text", "value": "Sam Altman" }, "sideB": { "modality": "text", "value": "CEO of OpenAI since 2019; led ChatGPT's launch and OpenAI's commercial pivot; briefly ousted and reinstated by the board in November 2023", "short": "CEO of OpenAI, led ChatGPT's launch" }, "uid": "1szcq0o11mvei8" }, { "id": "pair-dario", "shape": "pair", "tags": [ "openai-anthropic", "people" ], "sideA": { "modality": "text", "value": "Dario Amodei" }, "sideB": { "modality": "text", "value": "CEO of Anthropic; former VP of Research at OpenAI, where he helped invent RLHF and led GPT-2/GPT-3; left in 2021 over safety concerns to co-found Anthropic", "short": "CEO of Anthropic, focused on AI safety" }, "uid": "mvltahkwl3l7" }, { "id": "pair-daniela", "shape": "pair", "tags": [ "openai-anthropic", "people" ], "sideA": { "modality": "text", "value": "Daniela Amodei" }, "sideB": { "modality": "text", "value": "President of Anthropic; former VP of Safety and Policy at OpenAI; co-founded Anthropic in 2021 with her brother Dario", "short": "President of Anthropic, co-founder with Dario" }, "uid": "1bj3u7r1tk3yd9" }, { "id": "pair-hassabis", "shape": "pair", "tags": [ "deepmind-infra", "people" ], "sideA": { "modality": "text", "value": "Demis Hassabis" }, "sideB": { "modality": "text", "value": "Co-founder and CEO of Google DeepMind; built AlphaGo and AlphaFold; shared the 2024 Nobel Prize in Chemistry for protein-structure prediction and was knighted in 2024", "short": "CEO of DeepMind, built AlphaGo and AlphaFold" }, "uid": "lcclairbju5u" }, { "id": "pair-nvidia", "shape": "pair", "tags": [ "deepmind-infra", "companies", "hardware" ], "sideA": { "modality": "text", "value": "Nvidia" }, "sideB": { "modality": "text", "value": "The dominant maker of the GPUs and the CUDA software that train and run nearly all large models — the hardware backbone of the AI boom", "short": "Dominant GPU maker for AI model training" }, "uid": "1x4d7w11r35xhz" }, { "id": "pair-huggingface", "shape": "pair", "tags": [ "deepmind-infra", "companies", "ecosystem" ], "sideA": { "modality": "text", "value": "Hugging Face" }, "sideB": { "modality": "text", "value": "The leading open hub for sharing models and datasets, and maker of the Transformers library — the de facto commons of the open-source AI ecosystem", "short": "Open hub for models and datasets sharing" }, "uid": "ckidqz16pwbw5" }, { "id": "num-gpt1-llm", "shape": "numeric", "tags": [ "history", "gpt", "numeric" ], "prompt": { "modality": "text", "value": "GPT-1 parameter count (millions)" }, "value": 117, "unit": "million parameters", "tolerance": 0, "uid": "muulj7lfi691" }, { "id": "num-gpt2", "shape": "numeric", "tags": [ "history", "gpt", "numeric" ], "prompt": { "modality": "text", "value": "GPT-2 parameter count, largest version (billions)" }, "value": 1.5, "unit": "billion parameters", "tolerance": 0, "uid": "14lqeib1bb2kqd" }, { "id": "num-gpt3-llm", "shape": "numeric", "tags": [ "history", "gpt", "numeric" ], "prompt": { "modality": "text", "value": "GPT-3 parameter count (billions)" }, "value": 175, "unit": "billion parameters", "tolerance": 0, "uid": "13cvf7dwt2ku3" }, { "id": "num-gpt3-year", "shape": "numeric", "tags": [ "history", "gpt", "numeric" ], "prompt": { "modality": "text", "value": "Year GPT-3 was released" }, "value": 2020, "unit": "year", "tolerance": 0, "uid": "1g4psfk8wjprk" }, { "id": "num-chatgpt-year", "shape": "numeric", "tags": [ "history", "chatgpt", "numeric" ], "prompt": { "modality": "text", "value": "Year ChatGPT launched (November 30)" }, "value": 2022, "unit": "year", "tolerance": 0, "uid": "fo45le1h0mewa" }, { "id": "num-chatgpt-100m", "shape": "numeric", "tags": [ "history", "chatgpt", "numeric" ], "prompt": { "modality": "text", "value": "Months for ChatGPT to reach 100 million monthly users" }, "value": 2, "unit": "months", "tolerance": 1, "uid": "3b9yqh1lzmuqf" }, { "id": "num-gpt4-year", "shape": "numeric", "tags": [ "history", "gpt-4", "numeric" ], "prompt": { "modality": "text", "value": "Year GPT-4 was released (March 14)" }, "value": 2023, "unit": "year", "tolerance": 0, "uid": "ynvq83womj45" }, { "id": "num-gpt3-data", "shape": "numeric", "tags": [ "history", "gpt", "training-data", "numeric" ], "prompt": { "modality": "text", "value": "GPT-3 approximate training-data size (billions of tokens)" }, "value": 400, "unit": "billion tokens", "tolerance": 100, "uid": "1ikv7n314k47sl" }, { "id": "num-chinchilla", "shape": "numeric", "tags": [ "history", "scaling-laws", "numeric" ], "prompt": { "modality": "text", "value": "Chinchilla's compute-optimal training tokens per parameter" }, "value": 20, "unit": "tokens per parameter", "tolerance": 5, "uid": "1v94hom1ayhxh6" }, { "id": "num-kaplan-year", "shape": "numeric", "tags": [ "history", "scaling-laws", "numeric" ], "prompt": { "modality": "text", "value": "Year the Kaplan scaling-laws paper was published" }, "value": 2020, "unit": "year", "tolerance": 0, "uid": "1xx6mt91pfw3fz" }, { "id": "num-ctx-gpt3", "shape": "numeric", "tags": [ "mechanics", "context-window", "numeric" ], "prompt": { "modality": "text", "value": "GPT-3's context window at launch (tokens)" }, "value": 2048, "unit": "tokens", "tolerance": 0, "uid": "pxfm0owyk74" }, { "id": "num-ctx-gpt4", "shape": "numeric", "tags": [ "mechanics", "context-window", "numeric" ], "prompt": { "modality": "text", "value": "GPT-4's context window at launch in 2023 (tokens)" }, "value": 8192, "unit": "tokens", "tolerance": 0, "uid": "1h6ikuj1s6d1rp" }, { "id": "recall-transformer", "shape": "pair", "tags": [ "architecture", "recall" ], "sideA": { "modality": "text", "value": "Transformer" }, "sideB": { "modality": "text", "value": "The 2017 architecture behind every modern LLM" }, "uid": "1ruhvz11bvzx4n" }, { "id": "recall-token", "shape": "pair", "tags": [ "architecture", "recall" ], "sideA": { "modality": "text", "value": "Token" }, "sideB": { "modality": "text", "value": "The unit LLMs process — about ¾ of a word" }, "uid": "1ljlulx1lefgbj" }, { "id": "recall-pretraining", "shape": "pair", "tags": [ "training", "recall" ], "sideA": { "modality": "text", "value": "Pretraining" }, "sideB": { "modality": "text", "value": "Self-supervised next-token training on huge corpora" }, "uid": "g7u7g0d6zsro" }, { "id": "recall-rlhf", "shape": "pair", "tags": [ "training", "recall" ], "sideA": { "modality": "text", "value": "RLHF" }, "sideB": { "modality": "text", "value": "Aligning a model with human preference feedback" }, "uid": "b6r1lb8296n" }, { "id": "recall-rag", "shape": "pair", "tags": [ "inference", "recall" ], "sideA": { "modality": "text", "value": "RAG" }, "sideB": { "modality": "text", "value": "Retrieving documents into the prompt for grounding" }, "uid": "1xqbr7f1qm70rx" }, { "id": "recall-inference", "shape": "pair", "tags": [ "inference", "recall" ], "sideA": { "modality": "text", "value": "Inference" }, "sideB": { "modality": "text", "value": "Running a frozen model — no weights change" }, "uid": "1l1h3yz1ypukxp" }, { "id": "recall-hallucination", "shape": "pair", "tags": [ "inference", "recall" ], "sideA": { "modality": "text", "value": "Hallucination" }, "sideB": { "modality": "text", "value": "Fluent output that is confidently fabricated" }, "uid": "ms9bnoe5hfcs" }, { "id": "recall-context", "shape": "pair", "tags": [ "architecture", "recall" ], "sideA": { "modality": "text", "value": "Context window" }, "sideB": { "modality": "text", "value": "The model's token-limited working memory" }, "uid": "1orlzcu1u9pzbq" }, { "id": "recall-chinchilla", "shape": "pair", "tags": [ "training", "recall" ], "sideA": { "modality": "text", "value": "Chinchilla" }, "sideB": { "modality": "text", "value": "~20 training tokens per parameter is compute-optimal" }, "uid": "8b75l816iphzg" }, { "id": "recall-fewshot", "shape": "pair", "tags": [ "inference", "recall" ], "sideA": { "modality": "text", "value": "Few-shot" }, "sideB": { "modality": "text", "value": "Giving examples in the prompt to teach a task" }, "uid": "dmvyop2wkt7b" }, { "id": "recall-attention", "shape": "pair", "tags": [ "architecture", "recall" ], "sideA": { "modality": "text", "value": "Self-attention" }, "sideB": { "modality": "text", "value": "How tokens weigh one another inside a Transformer layer" }, "uid": "1rtec5216s8l3i" }, { "id": "recall-emergence", "shape": "pair", "tags": [ "inference", "recall" ], "sideA": { "modality": "text", "value": "Emergence" }, "sideB": { "modality": "text", "value": "Capabilities that appear only past a certain model scale" }, "uid": "1kj59sk1kbtp4c" }, { "id": "x-uc-summarization", "shape": "pair", "tags": [ "transform-text", "use-case", "language", "summarization" ], "sideA": { "modality": "text", "value": "Summarization" }, "sideB": { "modality": "text", "value": "Condensing long documents into structured summaries — a top strength. Catch: nuance and exact figures can drop, so verify critical details.", "short": "Condenses documents into structured summaries" }, "uid": "9r2d8jmacxc1" }, { "id": "x-uc-translation", "shape": "pair", "tags": [ "transform-text", "use-case", "language", "translation" ], "sideA": { "modality": "text", "value": "Translation" }, "sideB": { "modality": "text", "value": "Near-professional quality across major languages, including idioms. Catch: weaker on low-resource languages and technical or legal text — get expert review.", "short": "Near-professional quality across major languages" }, "uid": "ph94y2jodb2e" }, { "id": "x-uc-rewriting", "shape": "pair", "tags": [ "improve-classify", "use-case", "language", "rewriting" ], "sideA": { "modality": "text", "value": "Rewriting and tone-shifting" }, "sideB": { "modality": "text", "value": "Reliably shifts register ('make it formal,' 'simplify for a child') while keeping intent — among the most dependable uses, since the output is easy to judge by reading.", "short": "Shifts tone while preserving original intent" }, "uid": "1yi510x77m8m7" }, { "id": "x-uc-editing", "shape": "pair", "tags": [ "improve-classify", "use-case", "language", "editing" ], "sideA": { "modality": "text", "value": "Grammar and copy editing" }, "sideB": { "modality": "text", "value": "Catches errors and awkward phrasing and rewrites inline. Catch: can over-formalize or subtly change meaning without flagging it.", "short": "Catches grammar errors and awkward phrasing" }, "uid": "1s065zn3ulpxh" }, { "id": "x-uc-classification", "shape": "pair", "tags": [ "improve-classify", "use-case", "language", "classification" ], "sideA": { "modality": "text", "value": "Classification and sentiment" }, "sideB": { "modality": "text", "value": "Given a label set, few-shot prompting matches dedicated classifiers. Catch: it's non-deterministic — use low temperature and test on held-out examples.", "short": "Matches labels using few-shot prompting" }, "uid": "qlqmwu1i200mm" }, { "id": "x-uc-accessibility", "shape": "pair", "tags": [ "transform-text", "use-case", "language", "accessibility" ], "sideA": { "modality": "text", "value": "Plain-language simplification" }, "sideB": { "modality": "text", "value": "Rewrites technical, legal, or medical text to a plain reading level for low-literacy or non-native readers — high value, low risk, and easy to verify.", "short": "Simplifies text for low-literacy readers" }, "uid": "i0uxgxq2hmxf" }, { "id": "x-uc-doc-qa", "shape": "fact", "tags": [ "use-case", "knowledge", "rag" ], "title": "Document Q&A over your own files", "body": "Chunk private documents, embed them, and retrieve relevant passages into the prompt (RAG) to get an assistant grounded in your own data — manuals, contracts, wikis — without retraining. It's the most widely deployed enterprise LLM pattern because it's cheap, updatable, and keeps proprietary data out of model weights.", "illustration": { "imagePrompt": "A person at a desk with a laptop, surrounded by stacks of manuals, contracts, and folders, searching through the documents.", "imageSearchTerm": "person searching office documents", "alt": "A person searching documents at a laptop", "depictable": true, "credit": "Pexels · Dziana Hasanbekava · Pexels License", "creditUrl": "https://www.pexels.com/photo/crop-unrecognizable-man-looking-through-folder-in-home-office-7063752/", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-uc-doc-qa.webp" }, "factVariant": "image-heavy", "imageCaption": "RAG turns your own manuals and contracts into an assistant — no retraining, no leaked weights.", "studyGuideAnchor": "document-q-a-over-your-own-files", "uid": "1rgwvwy1e4a8iy" }, { "id": "x-uc-research", "shape": "pair", "tags": [ "knowledge", "use-case", "research" ], "sideA": { "modality": "text", "value": "Research assistance" }, "sideB": { "modality": "text", "value": "Synthesizes sources, drafts summaries, and explains dense papers. Catch: it can't browse live unless given a search tool — always verify citations and that sources say what's claimed.", "short": "Synthesizes sources and drafts summaries" }, "uid": "102ykaj1ne3u6d" }, { "id": "x-uc-semantic-search", "shape": "pair", "tags": [ "knowledge", "use-case", "semantic-search" ], "sideA": { "modality": "text", "value": "Semantic search" }, "sideB": { "modality": "text", "value": "Embeddings retrieve by meaning, not keywords ('contract renewals' finds docs that never use the word). Catch: weaker on niche jargon unless the embedding model is domain-tuned.", "short": "Retrieves documents by meaning, not keywords" }, "uid": "720idvdxf4x9" }, { "id": "x-uc-extraction", "shape": "pair", "tags": [ "knowledge", "use-case", "extraction" ], "sideA": { "modality": "text", "value": "Structured data extraction" }, "sideB": { "modality": "text", "value": "Parses messy text — invoices, emails, notes — into clean JSON or CSV. Catch: rare or ambiguous fields err more often, so validate outputs programmatically.", "short": "Parses text into structured data formats" }, "uid": "1ktulhsbpggnc" }, { "id": "x-uc-codegen", "shape": "fact", "tags": [ "use-case", "coding", "generation" ], "title": "Code generation: the copilot pattern", "body": "Code-trained LLMs suggest lines, whole functions, and boilerplate from a comment or docstring. In a controlled study, developers using GitHub Copilot finished a task about 56% faster, with junior developers gaining most. The value is a useful starting point, not guaranteed correctness — review still matters.", "illustration": { "imagePrompt": "A programmer at a desk typing on a laptop, with lines of code and a suggested line of autocomplete appearing in the editor.", "imageSearchTerm": "programmer typing code editor", "alt": "A developer writing code on a laptop", "depictable": true, "credit": "Pexels · cottonbro studio · Pexels License", "creditUrl": "https://www.pexels.com/photo/hands-typing-on-a-laptop-keyboard-5483077/", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-uc-codegen.webp" }, "factVariant": "image-heavy", "imageCaption": "Developers with an AI copilot finished a coding task 56% faster — but a fast first draft still isn't a correct one.", "studyGuideAnchor": "code-generation-the-copilot-pattern", "uid": "hy47dx1r7948v" }, { "id": "x-uc-num-copilot", "shape": "numeric", "tags": [ "use-case", "coding", "numeric" ], "prompt": { "modality": "text", "value": "In a controlled study, how much faster did developers complete a task using GitHub Copilot (%)?" }, "value": 55.8, "unit": "%", "tolerance": 2, "uid": "ahabjoieca0s" }, { "id": "x-uc-explanation", "shape": "pair", "tags": [ "code", "use-case", "coding", "explanation" ], "sideA": { "modality": "text", "value": "Code explanation and docs" }, "sideB": { "modality": "text", "value": "'Explain this line by line' or 'write a docstring' — very strong, since explanation is text over a constrained domain. Catch: can be plausibly wrong on optimized or legacy code; test the described behavior.", "short": "Explains code line by line or writes docstrings" }, "uid": "1ednm1xabvg7f" }, { "id": "x-uc-debugging", "shape": "pair", "tags": [ "code", "use-case", "coding", "debugging" ], "sideA": { "modality": "text", "value": "Debugging assistance" }, "sideB": { "modality": "text", "value": "Paste an error and the code; LLMs often spot root causes and fixes. Catch: novel bugs or performance issues needing profiler data are harder — treat suggestions as hypotheses to test.", "short": "Identifies root causes and fixes in code" }, "uid": "1c0a3zr12uh7lx" }, { "id": "x-uc-testing", "shape": "pair", "tags": [ "code", "use-case", "coding", "testing" ], "sideA": { "modality": "text", "value": "Test writing" }, "sideB": { "modality": "text", "value": "Generates unit tests covering happy paths and common edge cases. Catch: it won't know your domain's invariants or business rules, so unique edge cases still need a human.", "short": "Generates unit tests for code coverage" }, "uid": "1s67qim2g9hxy" }, { "id": "x-uc-support", "shape": "pair", "tags": [ "writing-comms", "use-case", "business", "customer-support" ], "sideA": { "modality": "text", "value": "Customer support chatbots" }, "sideB": { "modality": "text", "value": "RAG-backed bots answer tier-1 questions 24/7 and escalate ambiguous ones. Catch: out-of-scope questions yield confident wrong answers without good guardrails.", "short": "Chatbots for tier-1 customer support" }, "illustration": { "imagePrompt": "Photograph of a customer-support worker wearing a headset at a desk with a computer, friendly modern office, soft lighting", "imageSearchTerm": "customer service headset support", "alt": "A support worker wearing a headset at a computer", "credit": "aqua.mech (Openverse) · by 2.0", "creditUrl": "https://www.flickr.com/photos/137169575@N04/24704515719", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-uc-support.webp" }, "uid": "1405rgdj7iz2z" }, { "id": "x-uc-drafting", "shape": "pair", "tags": [ "writing-comms", "use-case", "business", "drafting" ], "sideA": { "modality": "text", "value": "Email and report drafting" }, "sideB": { "modality": "text", "value": "Turns bullets or a thread into a polished first draft in seconds. A human edit pass is still needed for tone and accuracy, but starting from a draft beats a blank page.", "short": "Creates polished drafts from bullet points" }, "uid": "1c095j1yuc0tj" }, { "id": "x-uc-meeting-notes", "shape": "pair", "tags": [ "writing-comms", "use-case", "business", "meeting-notes" ], "sideA": { "modality": "text", "value": "Meeting notes and action items" }, "sideB": { "modality": "text", "value": "Summarizes transcripts into decisions and to-dos. Catch: speaker attribution and acronyms get mangled — have attendees skim before distributing.", "short": "Summarizes meetings into decisions and tasks" }, "uid": "751acp1253phn" }, { "id": "x-uc-data-cleaning", "shape": "pair", "tags": [ "thinking-data", "use-case", "business", "data-cleaning" ], "sideA": { "modality": "text", "value": "Data cleaning and normalization" }, "sideB": { "modality": "text", "value": "Normalizes free-text fields ('NY' / 'new york' → 'New York, NY'). Catch: high-volume pipelines need structured-output modes or fine-tuning for consistency.", "short": "Normalizes inconsistent free-text fields" }, "uid": "1r57w7n10awhn9" }, { "id": "x-uc-tutoring", "shape": "pair", "tags": [ "thinking-data", "use-case", "personal", "tutoring" ], "sideA": { "modality": "text", "value": "Tutoring and learning" }, "sideB": { "modality": "text", "value": "Patient, always available, and able to explain a concept ten ways. Catch: factual errors are hard for learners to spot, so it's best paired with a verified curriculum.", "short": "Explains concepts in multiple ways patiently" }, "uid": "16cz8ep1lulbzn" }, { "id": "x-uc-brainstorming", "shape": "pair", "tags": [ "thinking-data", "use-case", "personal", "brainstorming" ], "sideA": { "modality": "text", "value": "Brainstorming and ideation" }, "sideB": { "modality": "text", "value": "Generates many names, angles, or framings fast; the human evaluates and selects. Catch: ideas cluster around training data, so truly novel domains get generic output.", "short": "Generates ideas quickly for human evaluation" }, "uid": "16ysp5xxnydtn" }, { "id": "x-uc-healthcare", "shape": "fact", "tags": [ "use-case", "high-stakes", "healthcare" ], "title": "Healthcare: assist, don't diagnose", "body": "LLMs can summarize clinical notes, draft discharge summaries, and surface literature. But medical hallucinations — a wrong dosage, a misattributed symptom — can harm patients, and research suggests hallucination may be intrinsic to LLMs. Deployment requires clinician review of every output and domain-specific evaluation.", "illustration": { "imagePrompt": "A clinician in a hospital reviewing a discharge summary and patient notes on a tablet.", "imageSearchTerm": "clinician reviewing discharge summary", "alt": "A doctor reviewing information on a tablet in a hospital", "depictable": true, "credit": "AI-generated (gpt-image-1.5)", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-uc-healthcare.webp" }, "factVariant": "image-heavy", "imageCaption": "LLMs can summarize clinical notes, draft discharge summaries, and surface literature.", "studyGuideAnchor": "healthcare-assist-don-t-diagnose", "uid": "m8a9fev8else" }, { "id": "x-uc-legal", "shape": "fact", "tags": [ "use-case", "high-stakes", "legal" ], "title": "Legal: research and drafting, never final", "body": "LLMs are strong at contract summarization, clause extraction, and statute research. But in 2023, two attorneys were sanctioned after ChatGPT fabricated case citations they filed in federal court — a canonical cost of unsupervised use. All LLM legal content must be cite-verified by a qualified attorney.", "illustration": { "imageSearchTerm": "attorney reviewing legal documents", "imagePrompt": "An attorney at a desk in a law office reviewing case files and legal documents, with a laptop open beside stacks of paper.", "alt": "Legal: research and drafting, never final", "depictable": true, "credit": "Pexels · https://kaboompics.com/ · Pexels License", "creditUrl": "https://www.pexels.com/photo/lawyers-looking-at-divorce-paper-7876050/", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-uc-legal.webp" }, "factVariant": "image-heavy", "imageCaption": "Two lawyers got sanctioned in 2023 after the AI invented court cases they filed — fluent, confident, fake.", "studyGuideAnchor": "legal-research-and-drafting-never-final", "uid": "14u7xmfitj0ux" }, { "id": "x-uc-finance", "shape": "fact", "tags": [ "use-case", "high-stakes", "finance" ], "title": "Finance: analysis aid, regulated output", "body": "LLMs help with earnings summaries, filing sentiment, and data extraction. In regulated finance, outputs that drive investment, lending, or compliance carry regulatory exposure, and hallucinated figures can cause real harm. Best practice: RAG over verified data plus mandatory human review before any customer-facing output.", "illustration": { "imageSearchTerm": "financial analyst earnings report", "imagePrompt": "A financial analyst at a desk reviewing earnings reports and financial charts on a computer screen.", "alt": "Finance: analysis aid, regulated output", "depictable": true, "credit": "Pexels · RDNE Stock project · Pexels License", "creditUrl": "https://www.pexels.com/photo/person-holding-white-printer-paper-7580704/", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-uc-finance.webp" }, "factVariant": "image-heavy", "imageCaption": "LLMs help with earnings summaries, filing sentiment, and data extraction.", "studyGuideAnchor": "finance-analysis-aid-regulated-output", "uid": "1j4fchraa4365" }, { "id": "x-uc-copilot-def", "shape": "definition", "tags": [ "use-case", "patterns", "human-in-the-loop" ], "term": { "modality": "text", "value": "copilot pattern" }, "definition": { "modality": "text", "value": "A design where the LLM suggests, drafts, or assists and a human approves before anything takes effect — accelerating the work without finalizing it. It contrasts with autonomous agents that act without per-step review." }, "curatedDistractors": [ "prompt chaining", "retrieval augmentation", "tool-use pattern" ], "uid": "7ei9tv106ev6x" }, { "id": "x-uc-hitl", "shape": "fact", "tags": [ "use-case", "patterns", "human-in-the-loop" ], "title": "Human-in-the-loop: when and why", "body": "Let the cost of verification set the level of autonomy. If checking the output is fast and cheap — reading a draft, running a test — more automation is fine. If verification is expensive or impossible — surgery, a legal filing, a trade — keep a human checkpoint before every consequential action.", "illustration": { "imageSearchTerm": "surgeon reviewing chart before surgery", "imagePrompt": "A surgeon pausing to review a patient's chart before beginning a procedure, illustrating a human checkpoint before a high-stakes action.", "alt": "Human-in-the-loop: when and why", "depictable": true, "credit": "AI-generated (gpt-image-1.5)", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-uc-hitl.webp" }, "studyGuideAnchor": "human-in-the-loop-when-and-why", "uid": "1m6m8la12228t2" }, { "id": "x-uc-proc", "shape": "procedure", "tags": [ "use-case", "patterns", "fit" ], "goal": "Choose the right LLM approach for a task", "steps": [ "Identify the task type: language, code, knowledge retrieval, or taking actions.", "Need real-time or proprietary data? Reach for RAG before fine-tuning.", "Need persistent custom style, persona, or domain behavior? Consider fine-tuning.", "Ask how costly a wrong output is; if high, require human review and prefer a copilot over an autonomous agent.", "Prototype with prompting first; add RAG or fine-tuning only when prompting hits a ceiling." ], "uid": "er5aij32aw4t" }, { "id": "x-uc-good-fit", "shape": "pair", "tags": [ "use-case", "patterns", "fit" ], "sideA": { "modality": "text", "value": "Good LLM fit" }, "sideB": { "modality": "text", "value": "Output is easy to judge, errors are cheap to catch, the task is language-in / language-out, and volume justifies the integration — e.g., drafting, summarizing, classification, reviewed code.", "short": "Tasks easy to judge and verify with LLMs" }, "uid": "1fwzd3utg7ieq" }, { "id": "x-uc-poor-fit", "shape": "pair", "tags": [ "use-case", "patterns", "fit" ], "sideA": { "modality": "text", "value": "Poor LLM fit" }, "sideB": { "modality": "text", "value": "Correctness can't be cheaply verified, exact arithmetic is required without tools, real-time data is needed without retrieval, or accountability demands auditable logic — e.g., live trading, unsupervised clinical decisions.", "short": "Tasks where correctness is hard to verify" }, "uid": "1ytq5hv1g20ngp" }, { "id": "x-lim-halluc-cost", "shape": "fact", "tags": [ "reliability", "hallucination" ], "title": "Why hallucinations cost you", "body": "A hallucinated output ranges from embarrassing (a wrong product name) to costly (fabricated legal citations filed in court, which drew sanctions in a documented 2023 case) to dangerous (a plausible but wrong drug dosage). The damage scales with how hard the claim is to verify and how much the user trusts the model's fluent, confident tone.", "illustration": { "imageSearchTerm": "small error growing larger", "imagePrompt": "A single small crack in a surface that widens and spreads outward into a larger fracture, representing a small error growing into serious consequences.", "alt": "Why hallucinations cost you", "credit": "AI-generated (gpt-image-1.5)", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-halluc-cost.webp" }, "factVariant": "image-heavy", "imageCaption": "One hallucinated legal citation drew real court sanctions in 2023 — fluency isn't truth.", "studyGuideAnchor": "why-hallucinations-cost-you", "uid": "r6zwuv1isdnll" }, { "id": "x-lim-halluc-fix", "shape": "fact", "tags": [ "reliability", "hallucination", "mitigation" ], "title": "Mitigating hallucination: the toolkit", "body": "Four complementary defenses: ground answers in retrieved sources (RAG); require the model to quote and link citations; prompt or fine-tune it to say 'I don't know' rather than confabulate; and for high-stakes output, have a domain expert verify before use. No single measure is enough on its own — they layer.", "illustration": { "imageSearchTerm": "layered hallucination mitigation defenses", "imagePrompt": "Four distinct shields layered and overlapping to form one stronger barrier, representing complementary defenses that work together.", "alt": "Mitigating hallucination: the toolkit", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/x-lim-halluc-fix.jpg", "credit": "Pexels · Google DeepMind", "creditUrl": "https://www.pexels.com/photo/a-black-and-white-diagram-25626449/" }, "studyGuideAnchor": "mitigating-hallucination-the-toolkit", "uid": "vmpkqnclgbr5" }, { "id": "x-lim-confident", "shape": "fact", "tags": [ "reliability", "hallucination" ], "title": "Confident tone is not correctness", "body": "LLMs write authoritative-sounding prose whether or not they have reliable information — it's structural, since the same training that makes output fluent makes it confident. The surface text carries no certainty signal: a low-probability guess and a well-established fact are written in exactly the same register.", "illustration": { "imageSearchTerm": "confident fluent authoritative tone", "imagePrompt": "A confident speaker at a podium addressing an audience with the same assured posture and expression no matter what is being said.", "alt": "Confident tone is not correctness", "depictable": false, "credit": "Pexels · Yusuf Kayode · Pexels License", "creditUrl": "https://www.pexels.com/photo/a-woman-wearing-a-floral-top-and-eyeglasses-11507571/", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-confident.webp" }, "factVariant": "image-heavy", "imageCaption": "A wild guess and a hard fact are written in exactly the same confident register.", "studyGuideAnchor": "confident-tone-is-not-correctness", "uid": "owxplcvmz7ms" }, { "id": "x-lim-cutoff", "shape": "definition", "tags": [ "reliability", "knowledge-cutoff" ], "term": { "modality": "text", "value": "knowledge cutoff" }, "definition": { "modality": "text", "value": "The date after which a model's training data ends; it has no intrinsic knowledge of later events. The real cutoff is fuzzy — a model's effective knowledge of recent events degrades before the stated date, because the web takes time to accumulate coverage of anything new." }, "uid": "g8tcu21mf3i1i" }, { "id": "x-lim-staleness", "shape": "fact", "tags": [ "reliability", "knowledge-cutoff" ], "title": "The staleness gap", "body": "A model is often deployed months or years after its training cutoff, and will confidently discuss outdated facts — prices, laws, office-holders, software versions — as if they were current. Use RAG with live data for time-sensitive tasks, or at least tell users the model's cutoff.", "illustration": { "imageSearchTerm": "old calendar outdated newspaper", "imagePrompt": "A calendar frozen on an old date beside a stack of yellowed, outdated newspapers, symbolizing information that has fallen behind the present.", "alt": "The staleness gap", "credit": "Pexels · Ylanite Koppens · Pexels License", "creditUrl": "https://www.pexels.com/photo/a-close-up-shot-of-newspapers-13081133/", "depictable": true, "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-staleness.webp" }, "factVariant": "image-heavy", "imageCaption": "A model will discuss last year's prices and office-holders as if they were still current.", "studyGuideAnchor": "the-staleness-gap", "uid": "18hzl5jc3ddk9" }, { "id": "x-lim-bias-source", "shape": "fact", "tags": [ "bias", "fairness" ], "title": "Where bias comes from", "body": "LLMs learn from human-written text that over-represents certain languages, demographics, and viewpoints — a model trained mostly on English-language Western web content reflects those perspectives by default. Bias enters at data collection, pretraining, and fine-tuning, and is partly mitigated but rarely eliminated at each stage.", "illustration": { "imagePrompt": "A large pile of books towering over a few scattered books set apart from it, representing an imbalance in whose perspective dominates.", "imageSearchTerm": "training data language imbalance", "alt": "A balance scale beside a laptop representing fairness", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/x-lim-bias-source.jpg", "credit": "Openverse · ccPixs.com · CC BY 2.0", "creditUrl": "https://www.flickr.com/photos/86530412@N02/7953227784" }, "factVariant": "image-heavy", "imageCaption": "Train mostly on English Western web text, and the model adopts those views by default.", "studyGuideAnchor": "where-bias-comes-from", "uid": "2lr84pwnuz6z" }, { "id": "x-lim-bias-shows", "shape": "fact", "tags": [ "bias", "fairness" ], "title": "How bias shows up", "body": "The documented forms are systematic, not random: stereotypical associations (job titles tied to genders), accuracy gaps (worse on African-American Vernacular English than Standard American English), cultural skew (assuming Western norms in advice), and weaker output in low-resource languages. The same groups are disadvantaged repeatedly.", "illustration": { "imageSearchTerm": "systematic stereotypical bias patterns", "imagePrompt": "A set of scales tipped consistently to the same side across several repeated trials, representing a systematic rather than random pattern.", "alt": "How bias shows up", "depictable": false, "credit": "Pexels · Tima Miroshnichenko · Pexels License", "creditUrl": "https://www.pexels.com/photo/a-group-of-women-in-formal-wear-holding-banners-7203387/", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-bias-shows.webp" }, "studyGuideAnchor": "how-bias-shows-up", "uid": "1289zck1u8wbek" }, { "id": "x-lim-bias-hard", "shape": "fact", "tags": [ "bias", "fairness", "nist" ], "title": "Why bias is hard to remove", "body": "Alignment and filters can curb the most obvious bias, but can't re-balance the underlying training distribution — and debiasing one dimension can introduce disparities elsewhere. Evaluating fairness across thousands of groups and tasks is intractable to cover fully; the NIST AI RMF treats data, evaluation, and human factors as distinct, ongoing challenges.", "illustration": { "imageSearchTerm": "bias resistant to removal", "imagePrompt": "A filter straining a cloudy liquid that remains tinted no matter how many times it passes through, representing bias that resists full removal.", "alt": "Why bias is hard to remove", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/x-lim-bias-hard.jpg", "credit": "Pexels · Nicolas Foster", "creditUrl": "https://www.pexels.com/photo/vintage-seagate-hard-disk-drive-on-white-background-38412996/" }, "studyGuideAnchor": "why-bias-is-hard-to-remove", "uid": "18q0tdh3gn3bv" }, { "id": "x-lim-data-paste", "shape": "fact", "tags": [ "privacy", "data-handling" ], "title": "What happens to data you paste", "body": "On a consumer LLM product, your inputs may be logged for abuse monitoring, used to train future models, or seen by support staff. Enterprise API tiers with data-processing agreements typically forbid training on your data. Never paste credentials, patient data, or confidential IP into a consumer chat without reading the provider's data-use terms.", "illustration": { "imageSearchTerm": "person typing laptop chat", "imagePrompt": "A person typing on a laptop keyboard late at night in an office, entering text into a chat window on the screen.", "alt": "What happens to data you paste", "depictable": true, "credit": "homegets.com (Openverse) · by 2.0", "creditUrl": "https://www.flickr.com/photos/141436406@N04/49675580438", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-data-paste.webp" }, "studyGuideAnchor": "what-happens-to-data-you-paste", "uid": "rrgj9z4mtyv9" }, { "id": "x-lim-memorization", "shape": "fact", "tags": [ "privacy", "memorization", "pii" ], "title": "Training-data memorization and PII", "body": "LLMs can memorize and regurgitate fragments of their training data — email addresses, phone numbers, whole passages — when prompted the right way. Research shows multi-query extraction attacks pull out far more PII than simple probes, and fine-tuned models leak more than base models, especially when trained on repeated sensitive data.", "illustration": { "imageSearchTerm": "memorized training data leakage", "imagePrompt": "Small glowing fragments leaking out from a dense, tangled network of light, suggesting private pieces of information escaping from a large dataset.", "alt": "Training-data memorization and PII", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/x-lim-memorization.jpg", "credit": "Pexels · Anna Shvets", "creditUrl": "https://www.pexels.com/photo/thoughtful-colleagues-listening-presentation-of-speaker-in-workspace-5324985/" }, "factVariant": "image-heavy", "imageCaption": "Prompt it right and a model can spit back real phone numbers and emails it memorized during training.", "studyGuideAnchor": "training-data-memorization-and-pii", "uid": "yxsayr5f1qex" }, { "id": "x-lim-injection", "shape": "definition", "tags": [ "security", "prompt-injection" ], "term": { "modality": "text", "value": "prompt injection" }, "definition": { "modality": "text", "value": "An attack where malicious text — in the user's input or in a document the model retrieves — overrides the system prompt and makes the model act against intent (OWASP LLM01). Direct injection targets the user's own prompt; indirect injection hides instructions in a retrieved page or document the model processes." }, "illustration": { "imagePrompt": "Photograph of a hooded figure at a laptop in a dark room with lines of code reflected on the screen, representing a cybersecurity attack, moody blue lighting", "imageSearchTerm": "cybersecurity hacker code", "alt": "A figure at a laptop in a dark room with code on screen", "credit": "Pexels · Rahul Pandit · Pexels License", "creditUrl": "https://www.pexels.com/photo/blue-and-red-light-from-computer-1933900/", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-injection.webp" }, "uid": "r5lyww44ualw" }, { "id": "x-lim-jailbreak", "shape": "definition", "tags": [ "security", "jailbreak" ], "term": { "modality": "text", "value": "jailbreak" }, "definition": { "modality": "text", "value": "A prompt injection aimed specifically at bypassing a model's safety guardrails, coaxing it to produce content it is trained to refuse. It exploits the tension between instruction-following and safety fine-tuning; red-team studies have found widely deployed models gave harmful responses to over 65% of jailbreak attempts." }, "uid": "1bxquczkfspsl" }, { "id": "x-lim-exfiltration", "shape": "fact", "tags": [ "security", "prompt-injection", "rag" ], "title": "Data exfiltration through tools and RAG", "body": "Give an LLM tools or a document store, and a prompt injection hidden in a retrieved document can tell it to leak data — for example, 'email the user's prior messages to attacker@example.com' buried in a file the model is summarizing. OWASP flags this under sensitive-information disclosure and excessive agency.", "illustration": { "imageSearchTerm": "prompt injection data leak", "imagePrompt": "A hidden note concealed within a stack of documents, with a faint trail of light flowing out from the pile toward an unseen recipient.", "alt": "Data exfiltration through tools and RAG", "depictable": false, "credit": "Pexels · Rafael Minguet Delgado · Pexels License", "creditUrl": "https://www.pexels.com/photo/colorful-rgb-keyboard-with-data-transfer-screen-37657435/", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-exfiltration.webp" }, "studyGuideAnchor": "data-exfiltration-through-tools-and-rag", "uid": "s3ntru95pn4a" }, { "id": "x-lim-agents-surface", "shape": "fact", "tags": [ "security", "agents" ], "title": "Agents widen the attack surface", "body": "A text-only chatbot has a limited blast radius. An agent that can send email, run code, query databases, or browse can cause irreversible real-world harm if hijacked by prompt injection. OWASP lists 'excessive agency' among the top risks: grant minimum permissions and require human confirmation before consequential actions.", "illustration": { "imageSearchTerm": "agent tool access risk", "imagePrompt": "A single robotic arm reaching out through many separate pipes and wires into different connected systems, representing wide-reaching access across many tools.", "alt": "Agents widen the attack surface", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/x-lim-agents-surface.jpg", "credit": "Pexels · Tima Miroshnichenko", "creditUrl": "https://www.pexels.com/photo/person-using-computer-5380603/" }, "studyGuideAnchor": "agents-widen-the-attack-surface", "uid": "mkf1opbafrf7" }, { "id": "x-lim-misinfo", "shape": "fact", "tags": [ "societal", "misinformation" ], "title": "Misinformation at scale", "body": "LLMs generate fluent, persuasive text in seconds, slashing the cost of fake news, fake reviews, phishing, and propaganda. The risk is structural — cheap generation, high fluency, and no built-in truth constraint add up to high misinformation throughput — which is why NIST's generative-AI profile names information integrity a core risk.", "illustration": { "imagePrompt": "A person scrolling through a crowded social media news feed on a smartphone, an overwhelming stream of headlines and posts flowing past.", "imageSearchTerm": "smartphone social media newsfeed", "alt": "A smartphone showing a feed of news and social posts", "depictable": true, "credit": "AI-generated (gpt-image-1.5)", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-misinfo.webp" }, "factVariant": "image-heavy", "imageCaption": "LLMs generate fluent, persuasive text in seconds, slashing the cost of fake news, fake reviews, phishing, and propaganda.", "studyGuideAnchor": "misinformation-at-scale", "uid": "wlqal33j3yap" }, { "id": "x-lim-automation-bias", "shape": "definition", "tags": [ "societal", "automation-bias" ], "term": { "modality": "text", "value": "automation bias" }, "definition": { "modality": "text", "value": "The human tendency to over-trust and under-scrutinize automated output — accepting an LLM's confident-sounding answer without applying one's own judgment. Researchers argue that measuring and reducing over-reliance is essential, citing both near-term high-stakes errors and long-term cognitive deskilling." }, "uid": "ijjgiyfb8snq" }, { "id": "x-lim-copyright", "shape": "fact", "tags": [ "societal", "copyright", "ip" ], "title": "Copyright and intellectual property", "body": "Training models on copyrighted text, and generating output that closely echoes it, raises unresolved legal questions. Recent US rulings have split — some treating training as fair use while rejecting the use of pirated sources — and producing output substantially similar to a copyrighted work is a separate risk. The law is still actively evolving.", "illustration": { "imageSearchTerm": "books copyright law gavel", "imagePrompt": "A stack of published books beside a judge's gavel on a wooden desk, representing legal disputes over copyrighted written work.", "alt": "Copyright and intellectual property", "depictable": true, "credit": "Pexels · KATRIN BOLOVTSOVA · Pexels License", "creditUrl": "https://www.pexels.com/photo/justice-figurine-on-table-6077091/", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-copyright.webp" }, "factVariant": "image-heavy", "imageCaption": "Training models on copyrighted text, and generating output that closely echoes it, raises unresolved legal questions.", "studyGuideAnchor": "copyright-and-intellectual-property", "uid": "7yfk7no8jutp" }, { "id": "x-lim-math", "shape": "fact", "tags": [ "limits", "math", "reasoning" ], "title": "Poor at precise math and formal logic", "body": "LLMs predict plausible token sequences; they don't execute algorithms. They make arithmetic errors on multi-digit math, stumble on formal logic, and are unreliable where exact numerical precision matters. The fix is to hand the model a calculator or code tool — never trust raw LLM output for financial or scientific computation.", "illustration": { "imageSearchTerm": "handheld calculator math equations", "imagePrompt": "A hand holding a calculator next to a notebook filled with handwritten math equations, representing tools built for precise arithmetic.", "alt": "Poor at precise math and formal logic", "credit": "Pexels · https://kaboompics.com/ · Pexels License", "creditUrl": "https://www.pexels.com/photo/close-up-photo-of-calculator-display-on-a-smartphone-4386326/", "depictable": true, "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-math.webp" }, "factVariant": "image-heavy", "imageCaption": "It predicts plausible text; it doesn't run the math. Hand it a calculator before you trust a single number.", "studyGuideAnchor": "poor-at-precise-math-and-formal-logic", "uid": "s3ke8j1ipm311" }, { "id": "x-lim-nondeterminism", "shape": "fact", "tags": [ "limits", "non-determinism" ], "title": "Non-determinism: same prompt, different answers", "body": "Above temperature 0, outputs are sampled from a probability distribution, so the same prompt can return meaningfully different answers across calls. That matters for testing, compliance, and any workflow assuming deterministic behavior. Use temperature 0 for reproducibility, and regression-test when upgrading model versions.", "illustration": { "imageSearchTerm": "sampling probability random outcomes", "imagePrompt": "Several identical dice caught mid-roll on a table, each landing on a different face, representing the same starting point producing different outcomes.", "alt": "Non-determinism: same prompt, different answers", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/x-lim-nondeterminism.jpg", "credit": "Pexels · Ron Lach", "creditUrl": "https://www.pexels.com/photo/person-facing-a-big-screen-with-numbers-9783346/" }, "factVariant": "image-heavy", "imageCaption": "Above temperature 0, the same prompt can return a different answer every single call.", "studyGuideAnchor": "non-determinism-same-prompt-different-answers", "uid": "18u71n51vs73gb" }, { "id": "x-lim-cost", "shape": "fact", "tags": [ "limits", "cost", "latency" ], "title": "Cost, latency, and token limits", "body": "Frontier models cost fractions of a cent to cents per thousand tokens — which adds up fast at scale, as in a million-document RAG pipeline. Context windows cap how much text fits per call, and latency of seconds per response matters for real-time use. When cost, speed, or volume dominate, a smaller or self-hosted model may win.", "illustration": { "imageSearchTerm": "token cost latency tradeoff", "imagePrompt": "A stopwatch resting beside a small stack of coins, representing the balance between speed and cost.", "alt": "Cost, latency, and token limits", "credit": "AI-generated (gpt-image-1.5)", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-cost.webp" }, "studyGuideAnchor": "cost-latency-and-token-limits", "uid": "48n0a71iwtfn1" }, { "id": "x-lim-when-not", "shape": "fact", "tags": [ "limits", "fit" ], "title": "When not to use an LLM", "body": "Avoid an LLM as the sole decision-maker when exact computation is required, strict reproducibility is needed, outputs can't be verified by any human or tool, regulatory accountability demands auditable logic, or real-time data is missing and staleness would harm. Rule of thumb: if 'probably right' isn't acceptable, add verification or pick another tool.", "illustration": { "imageSearchTerm": "decision point human judgment", "imagePrompt": "A person standing at a fork in a road, pausing to consider which path to take.", "alt": "When not to use an LLM", "depictable": false, "credit": "Pexels · Yan Krukau · Pexels License", "creditUrl": "https://www.pexels.com/photo/people-hands-pointing-fingers-to-a-man-7640823/", "url": "https://cdn.recurxive.com/packs/llm-101/images/x-lim-when-not.webp" }, "studyGuideAnchor": "when-not-to-use-an-llm", "uid": "g04n86lr515e" }, { "id": "x-lim-redteam", "shape": "definition", "tags": [ "governance", "red-teaming" ], "term": { "modality": "text", "value": "red-teaming" }, "definition": { "modality": "text", "value": "Structured adversarial testing of an LLM system — by internal or external specialists — to find jailbreaks, bias failures, and safety gaps before deployment. The NIST AI Risk Management Framework's generative-AI profile recommends it as a core pre-deployment practice." }, "uid": "1n57uqg1la6wg0" }, { "id": "x-lim-modelcard", "shape": "definition", "tags": [ "governance", "model-card", "transparency" ], "term": { "modality": "text", "value": "model card" }, "definition": { "modality": "text", "value": "A short, standardized document released with a model (proposed by Mitchell and colleagues in 2019) that discloses intended uses, evaluation results across demographic groups, known limitations, training data, and ethical considerations — a kind of nutrition label. Adopted by Hugging Face, Google, and Anthropic." }, "uid": "16ilr7r2tn5z1" }, { "id": "x-lim-guardrails", "shape": "fact", "tags": [ "governance", "guardrails" ], "title": "Guardrails help but aren't enough", "body": "Content filters, system-prompt restrictions, and output classifiers reduce harmful output but can't stop all of it — red-teamers consistently find bypasses. OWASP's LLM Top 10 warns that continuous training and updated safety mechanisms beat static system-prompt guardrails alone. Guardrails lower risk; they don't eliminate it.", "illustration": { "imagePrompt": "A tall protective fence around a property with one small gap where someone has slipped through, representing barriers that reduce but don't eliminate risk.", "imageSearchTerm": "content filter bypass risk", "alt": "A glowing digital shield over a circuit board", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/x-lim-guardrails.jpg", "credit": "Pexels · Dan Nelson", "creditUrl": "https://www.pexels.com/photo/smartphone-and-laptop-with-active-surfshark-vpn-service-4489171/" }, "factVariant": "image-heavy", "imageCaption": "Red-teamers keep finding bypasses — guardrails lower the risk, they never erase it.", "studyGuideAnchor": "guardrails-help-but-aren-t-enough", "uid": "ge6rcsgsr0yk" }, { "id": "x-lim-proc-deploy", "shape": "procedure", "tags": [ "governance", "procedure", "responsible-use" ], "goal": "Deploy an LLM responsibly in production", "steps": [ "Define the task scope and identify the high-risk output types.", "Check the model card for known limitations and demographic performance gaps.", "Add RAG or retrieval for time-sensitive or proprietary knowledge.", "Implement input/output guardrails and prompt-injection defenses.", "Red-team before launch: test jailbreaks, edge cases, and bias scenarios.", "Define a human-review gate for any consequential action.", "Monitor outputs in production and set up incident response for failures." ], "uid": "1lnjpx1vfq0tb" }, { "id": "x-lim-alignment", "shape": "fact", "tags": [ "governance", "alignment" ], "title": "Alignment is an open problem", "body": "Getting LLMs to reliably do what humans intend — across novel situations, adversarial inputs, and large-scale deployment — is an active, unsolved area of AI safety research. RLHF and Constitutional AI reduce misalignment but aren't proven sufficient for highly autonomous, agentic systems, which is why labs like Anthropic, OpenAI, and DeepMind run dedicated safety programs.", "illustration": { "imageSearchTerm": "AI alignment unsolved problem", "imagePrompt": "A compass needle spinning unsteadily without settling on a fixed direction, representing an unresolved search for reliable guidance.", "alt": "Alignment is an open problem", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/x-lim-alignment.jpg", "credit": "Pexels · Diva Plavalaguna", "creditUrl": "https://www.pexels.com/photo/close-up-photo-of-people-holding-puzzle-pieces-6147357/" }, "factVariant": "image-heavy", "imageCaption": "Making AI reliably do what we mean is still unsolved — which is why labs run safety teams.", "studyGuideAnchor": "alignment-is-an-open-problem", "uid": "fyfr6p1b5yd9n" }, { "id": "proc-rag-pipeline", "shape": "procedure", "tags": [ "procedure", "how-to" ], "goal": "Build a retrieval-augmented generation (RAG) pipeline", "steps": [ "Chunk your source documents into 500–1000 token passages with some overlap.", "Embed every chunk using a sentence-embedding model into a vector database.", "At query time, embed the user's question and run k-nearest-neighbor search.", "Concatenate the top-k passages into a context block for the LLM prompt.", "Send the augmented prompt to the LLM and stream the response back to the user.", "Optionally re-rank retrieved passages with a cross-encoder for higher accuracy." ], "uid": "15g1hi01y3vzk8" }, { "id": "proc-prompt-engineer", "shape": "procedure", "tags": [ "procedure", "how-to" ], "goal": "Engineer a robust system prompt", "steps": [ "State the assistant's role and expertise level in one sentence.", "List the specific tasks the assistant should perform and the ones it should refuse.", "Provide the output format (markdown, JSON schema, max length) explicitly.", "Include 1–3 few-shot examples showing the desired behavior on edge cases.", "Add guardrails — what to do if the user asks for restricted content.", "Test the prompt against an adversarial prompt set and iterate." ], "uid": "uucuhrgokoxl" }, { "id": "proc-deploy-llm", "shape": "procedure", "tags": [ "procedure", "how-to" ], "goal": "Deploy an open-source LLM behind an API", "steps": [ "Pick a model weight (LLaMA-2 7B, Mistral 7B) and quantize to int4/int8 for VRAM fit.", "Stand up a GPU instance (a single 24GB card handles 7B comfortably).", "Install an inference server like vLLM or TGI and load the weights.", "Wrap the server with an OpenAI-compatible HTTP layer (chat/completions endpoint).", "Add rate limits, authentication, and request logging in front of the inference layer.", "Set up health checks and autoscaling rules tied to GPU utilization." ], "uid": "1grlph747lrc9" }, { "id": "proc-fine-tune", "shape": "procedure", "tags": [ "procedure", "how-to" ], "goal": "Fine-tune an LLM on instruction data", "steps": [ "Curate a 1–10K example dataset of (prompt, ideal response) pairs.", "Format examples in ChatML or Alpaca style with consistent role markers.", "Use LoRA or QLoRA to train only a small adapter — keeps GPU VRAM manageable.", "Train for 2–4 epochs with a low learning rate (1e-4 or smaller).", "Evaluate on a held-out set, watching for overfitting and instruction drift.", "Merge the adapter back into the base weights for deployment, or serve the adapter." ], "uid": "4w5sio1reddkc" }, { "id": "num-llm-bert-large", "shape": "numeric", "tags": [ "models", "parameters" ], "prompt": { "modality": "text", "value": "Parameter count of BERT-large" }, "value": 340, "unit": "million", "tolerance": 50, "uid": "diw12e119irp6" }, { "id": "num-gpt1-params", "shape": "numeric", "tags": [ "models", "parameters" ], "prompt": { "modality": "text", "value": "Parameter count of GPT-1" }, "value": 117, "unit": "million", "tolerance": 20, "uid": "1mgonrm1tbcy66" }, { "id": "num-gpt2-params", "shape": "numeric", "tags": [ "models", "parameters" ], "prompt": { "modality": "text", "value": "Parameter count of GPT-2" }, "value": 1500, "unit": "million", "tolerance": 200, "uid": "c367visf932u" }, { "id": "num-gpt3-params", "shape": "numeric", "tags": [ "models", "parameters" ], "prompt": { "modality": "text", "value": "Parameter count of GPT-3" }, "value": 175000, "unit": "million", "tolerance": 20000, "uid": "1nyr1z019v9338" }, { "id": "num-llama-7b", "shape": "numeric", "tags": [ "models", "parameters" ], "prompt": { "modality": "text", "value": "Parameter count of LLaMA-7B" }, "value": 7000, "unit": "million", "tolerance": 500, "uid": "1v2jp6q7mie3e" }, { "id": "num-llama-70b", "shape": "numeric", "tags": [ "models", "parameters" ], "prompt": { "modality": "text", "value": "Parameter count of LLaMA-2 70B" }, "value": 70000, "unit": "million", "tolerance": 5000, "uid": "1j9l2ce1hdp8zu" }, { "id": "num-mistral-7b", "shape": "numeric", "tags": [ "models", "parameters" ], "prompt": { "modality": "text", "value": "Parameter count of Mistral 7B" }, "value": 7000, "unit": "million", "tolerance": 500, "uid": "mpjoau1roouxa" }, { "id": "num-palm-540b", "shape": "numeric", "tags": [ "models", "parameters" ], "prompt": { "modality": "text", "value": "Parameter count of PaLM" }, "value": 540000, "unit": "million", "tolerance": 50000, "uid": "1uipllr1o4y23p" }, { "id": "num-bert-released", "shape": "numeric", "tags": [ "dates", "models" ], "prompt": { "modality": "text", "value": "Year BERT was released" }, "value": 2018, "unit": "year", "tolerance": 1, "uid": "r6sypv4piyx5" }, { "id": "num-gpt2-released", "shape": "numeric", "tags": [ "dates", "models" ], "prompt": { "modality": "text", "value": "Year GPT-2 was released" }, "value": 2019, "unit": "year", "tolerance": 1, "uid": "g8cnot1ke9lib" }, { "id": "num-chatgpt-released", "shape": "numeric", "tags": [ "dates", "models" ], "prompt": { "modality": "text", "value": "Year ChatGPT was released" }, "value": 2022, "unit": "year", "tolerance": 1, "uid": "1tm4z7815mcjpc" }, { "id": "num-llama-released", "shape": "numeric", "tags": [ "dates", "models" ], "prompt": { "modality": "text", "value": "Year LLaMA was released by Meta" }, "value": 2023, "unit": "year", "tolerance": 1, "uid": "1sslsuh1diksa3" }, { "id": "num-gpt4-released", "shape": "numeric", "tags": [ "dates", "models" ], "prompt": { "modality": "text", "value": "Year GPT-4 was released" }, "value": 2023, "unit": "year", "tolerance": 1, "uid": "13yn5i01e25o00" }, { "id": "concept-yoshua-bengio", "shape": "concept", "tags": [ "key-concept", "person" ], "conceptKind": "person", "name": "Yoshua Bengio", "clues": [ "A Canadian computer scientist known for his work in deep learning and artificial intelligence.", "A pioneer of deep learning; his group introduced influential neural language models and attention mechanisms.", "He won the Turing Award in 2018 alongside Geoffrey Hinton and Yann LeCun for their work on deep learning." ], "uid": "161oyyqywujl2" }, { "id": "concept-geoffrey-hinton", "shape": "concept", "tags": [ "key-concept", "person" ], "conceptKind": "person", "name": "Geoffrey Hinton", "clues": [ "A British-Canadian cognitive psychologist and computer scientist recognized as a leading figure in AI.", "He is known for his research on neural networks and deep learning techniques that underpin modern LLMs.", "He received the Turing Award in 2018, sharing it with Yoshua Bengio and Yann LeCun for their contributions to AI." ], "uid": "az61m9ppovpj" }, { "id": "concept-yann-lecun", "shape": "concept", "tags": [ "key-concept", "person" ], "conceptKind": "person", "name": "Yann LeCun", "clues": [ "A French-American computer scientist known for his work in machine learning and computer vision.", "He is one of the founding figures of convolutional neural networks, crucial for image processing.", "He was awarded the Turing Award in 2018 alongside Geoffrey Hinton and Yoshua Bengio for advancements in deep learning." ], "uid": "3cni1w3l730o" }, { "id": "concept-ilya-sutskever", "shape": "concept", "tags": [ "key-concept", "person" ], "conceptKind": "person", "name": "Ilya Sutskever", "clues": [ "A co-founder and former chief scientist of OpenAI who later left to found Safe Superintelligence.", "He played a significant role in developing the sequence-to-sequence model and attention mechanisms.", "He is known for his contributions to generative models, including the GPT series of language models." ], "uid": "i6jofl1607n1z" }, { "id": "concept-andrew-ng", "shape": "concept", "tags": [ "key-concept", "person" ], "conceptKind": "person", "name": "Andrew Ng", "clues": [ "An influential computer scientist and entrepreneur in the field of AI and machine learning.", "He co-founded Google Brain and has significantly impacted online education through Coursera.", "He is known for his work on deep learning and has taught many foundational courses in AI." ], "uid": "ejh5vv1o1w8qp" }, { "id": "concept-alec-radford", "shape": "concept", "tags": [ "key-concept", "person" ], "conceptKind": "person", "name": "Alec Radford", "clues": [ "An AI researcher known for his work at OpenAI, particularly in natural language processing.", "He was a key figure in developing the original GPT model and its successors.", "He has contributed to advancements in generative models and their applications in AI." ], "uid": "1jegogl16xrn7v" }, { "id": "concept-word2vec", "shape": "concept", "tags": [ "key-concept", "thing" ], "conceptKind": "thing", "name": "word2vec", "clues": [ "A technique for representing words as vectors in a high-dimensional space.", "It captures semantic relationships through vector arithmetic.", "Introduced by a team led by Tomas Mikolov in 2013." ], "uid": "1kgithp1udzmgn" }, { "id": "concept-bert", "shape": "concept", "tags": [ "key-concept", "thing" ], "conceptKind": "thing", "name": "BERT", "clues": [ "A model designed for understanding the context of words in a sentence.", "It uses bidirectional training to predict masked words.", "Released by Google in October 2018, it set new benchmarks in NLP." ], "uid": "rqa5c618yqmc6" }, { "id": "concept-gpt-3", "shape": "concept", "tags": [ "key-concept", "thing" ], "conceptKind": "thing", "name": "GPT-3", "clues": [ "A large language model known for its few-shot learning capabilities.", "It has 175 billion parameters, making it one of the largest models to date.", "Released by OpenAI in 2020, it can perform tasks without explicit training." ], "uid": "1brbo1h15dut2r" }, { "id": "concept-transformer", "shape": "concept", "tags": [ "key-concept", "thing" ], "conceptKind": "thing", "name": "Transformer", "clues": [ "An architecture that processes sequences of data in parallel.", "It replaces recurrent layers with self-attention mechanisms.", "Introduced in the 2017 paper 'Attention Is All You Need' by Vaswani et al." ], "uid": "z3dzrd1fdbehf" }, { "id": "concept-lstm", "shape": "concept", "tags": [ "key-concept", "thing" ], "conceptKind": "thing", "name": "LSTM", "clues": [ "A type of recurrent neural network designed to remember information over long sequences.", "It addresses the vanishing gradient problem of standard RNNs.", "Developed in the 1990s, it became popular for NLP tasks around 2014." ], "uid": "3d6rfui5fv9u" }, { "id": "concept-chatgpt", "shape": "concept", "tags": [ "key-concept", "thing" ], "conceptKind": "thing", "name": "ChatGPT", "clues": [ "An AI model designed for conversational interactions.", "It utilizes reinforcement learning from human feedback to improve responses.", "Launched by OpenAI in November 2022, it quickly gained widespread use." ], "uid": "1elpz6z1j0efr5" }, { "id": "scrapbook-llm-101-1", "shape": "fact", "factVariant": "scrapbook", "title": "The Evolution of NLP", "body": "A visual timeline of NLP advancements.", "tags": [ "scrapbook", "overview" ], "illustration": { "kind": "scrapbook", "alt": "A visual timeline of NLP advancements.", "imagePrompt": "An abstract flowing timeline of glowing nodes connected in sequence, evolving from simple shapes into increasingly complex interconnected forms.", "imageSearchTerm": "NLP timeline advancement evolution", "depictable": false, "credit": "AI-generated (gpt-image-1.5)", "url": "https://cdn.recurxive.com/packs/llm-101/images/scrapbook-llm-101-1.webp" }, "uid": "1czr61qfrr8u" }, { "id": "scrapbook-llm-101-2", "shape": "fact", "factVariant": "scrapbook", "title": "Inside Large Language Models", "body": "An overview of how LLMs function.", "tags": [ "scrapbook", "overview" ], "illustration": { "kind": "scrapbook", "alt": "An overview of how LLMs function.", "imagePrompt": "Design a cheerful scrapbook collage that illustrates how large language models (LLMs) work. Feature a central doodle of a brain made from tokens, surrounded by elements like attention mechanisms, context windows, and fine-tuning processes. Use labels to highlight terms like 'Pretraining', 'RLHF', and 'Few-shot Learning'. Incorporate playful stickers and arrows to guide the viewer's eye and convey information in a light-hearted way. The title 'Inside Large Language Models' should be prominently hand-lettered at the top, using a cohesive warm color palette.", "imageSearchTerm": "llm-101 scrapbook poster", "credit": "AI-generated (gpt-image-1)", "url": "https://cdn.recurxive.com/packs/llm-101/generated/scrapbook-2.webp" }, "uid": "1asmtb9q9wz4j" }, { "id": "q-llm-bias-stages", "shape": "pair", "sideA": { "modality": "text", "value": "Stages where bias enters an LLM" }, "sideB": { "modality": "text", "value": "Data collection, pretraining, and fine-tuning", "short": "collection, pretraining, fine-tuning" }, "tags": [ "bias", "fairness" ], "uid": "5qft4rwzxll1" }, { "id": "q-llm-bias-aave-gap", "shape": "pair", "sideA": { "modality": "text", "value": "A documented LLM accuracy gap" }, "sideB": { "modality": "text", "value": "Worse on African-American Vernacular English than Standard American English", "short": "worse on AAVE than SAE" }, "tags": [ "bias", "fairness" ], "uid": "1qa48ub1ricyvp" }, { "id": "q-llm-paste-consumer", "shape": "pair", "sideA": { "modality": "text", "value": "What can happen to inputs on a consumer LLM product" }, "sideB": { "modality": "text", "value": "Logged for abuse monitoring, used to train future models, or seen by support staff", "short": "logged, trained on, or reviewed" }, "tags": [ "privacy", "data-handling" ], "uid": "1ejy74t1y4qpf7" }, { "id": "mcq-rw-l1-p1-before", "shape": "mcq", "tags": [ "seq2seq", "attention", "nlp-history" ], "prompt": { "modality": "text", "value": "In sequence-to-sequence (seq2seq) translation models, what problem did Bahdanau's attention mechanism solve?" }, "options": [ { "modality": "text", "value": "The single fixed-length vector between encoder and decoder had to carry all information about the source, losing detail on long inputs", "short": "Fixed-length vector bottleneck" }, { "modality": "text", "value": "The inability to train the model in parallel across a sequence on GPUs", "short": "No parallel GPU training" }, { "modality": "text", "value": "N-gram models' blindness to any word more than two positions back", "short": "N-grams miss distant words" }, { "modality": "text", "value": "The vanishing gradient that caused RNNs to lose signal over long sequences", "short": "Vanishing gradient in RNNs" } ], "correctIndex": 0, "explanation": "Seq2seq squeezed the whole source sentence into one fixed vector — a bottleneck. Attention let the decoder look back at all encoder states each step, weighting them by relevance. LSTMs addressed vanishing gradients; the Transformer later added parallelism.", "uid": "35el94hp2440" }, { "id": "mcq-rw-l1-p2-gpt", "shape": "mcq", "tags": [ "bert", "pretraining", "gpt-era" ], "prompt": { "modality": "text", "value": "How did BERT's (2018) training objective differ from GPT's?" }, "options": [ { "modality": "text", "value": "It predicted the next token strictly left-to-right, one at a time", "short": "Next token, left-to-right" }, { "modality": "text", "value": "It masked random words and predicted them using context from both directions (masked language modeling)", "short": "Masked words, bidirectional" }, { "modality": "text", "value": "It ranked human-preferred outputs to train a reward model", "short": "Ranked outputs for reward model" }, { "modality": "text", "value": "It learned new tasks from a few examples in the prompt, with no weight updates", "short": "Few-shot from prompt examples" } ], "correctIndex": 1, "explanation": "BERT masks random tokens and predicts them from context on both sides at once — bidirectional — unlike GPT's left-to-right next-token objective. It hit state-of-the-art on 11 benchmarks. (In-context learning is GPT-3; reward models are RLHF.)", "uid": "19xknm21ctwwtq" }, { "id": "mcq-rw-l1-p3-modern", "shape": "mcq", "tags": [ "llama", "open-weights", "scaling" ], "prompt": { "modality": "text", "value": "What did Meta's LLaMA (2023) demonstrate about model size versus performance?" }, "options": [ { "modality": "text", "value": "LLaMA needed roughly the same 175B parameters as GPT-3 to reach parity", "short": "Needed ~175B like GPT-3" }, { "modality": "text", "value": "LLaMA's 65B model was the smallest that could beat GPT-3", "short": "65B was smallest to beat GPT-3" }, { "modality": "text", "value": "LLaMA matched GPT-3 only after fine-tuning on private medical and legal data", "short": "Matched only after fine-tuning" }, { "modality": "text", "value": "A 13B-parameter LLaMA outperformed the 175B-parameter GPT-3 on most standard benchmarks", "short": "13B LLaMA beat 175B GPT-3" } ], "correctIndex": 3, "explanation": "LLaMA-13B beat GPT-3 (175B) on most benchmarks, showing careful data curation and training compute can outweigh raw parameter count. LLaMA trained only on publicly available data, and its released weights sparked an open fine-tuning ecosystem.", "uid": "v59lo7ndjpc1" }, { "id": "mcq-rw-l2-p1-tokens", "shape": "mcq", "tags": [ "tokens", "tokenization" ], "prompt": { "modality": "text", "value": "On average, roughly how much of a word does a single token represent?" }, "options": [ { "modality": "text", "value": "About two words", "short": "About two words" }, { "modality": "text", "value": "About three-quarters of a word", "short": "About 3/4 of a word" }, { "modality": "text", "value": "Exactly one word", "short": "Exactly one word" }, { "modality": "text", "value": "About one-third of a word", "short": "About 1/3 of a word" } ], "correctIndex": 1, "explanation": "A token is the basic unit an LLM processes — on average about three-quarters of a word. Common words like 'ChatGPT' are one token, while 'unhappy' may split into 'un' + 'happy'. Tokens (not words) are what context limits, pricing, and speed are measured in.", "uid": "10iwrjfpjajlp" }, { "id": "mcq-rw-l2-p2-generate", "shape": "mcq", "tags": [ "hallucination", "generation", "rag" ], "prompt": { "modality": "text", "value": "Why do large language models 'hallucinate' — state false things with confidence?" }, "options": [ { "modality": "text", "value": "The temperature setting is always too high, forcing random errors", "short": "Temperature always too high" }, { "modality": "text", "value": "They are built to produce plausible-sounding text, not to retrieve verified facts, and have no internal fact-checker", "short": "Plausible text, no fact-checker" }, { "modality": "text", "value": "Their training data is always out of date, so every answer is stale", "short": "Training data always stale" }, { "modality": "text", "value": "They deliberately invent details to fill up the context window", "short": "Invent to fill context window" } ], "correctIndex": 1, "explanation": "A hallucination is fluent, confident text that's wrong. It happens because the model fills in plausible continuations the same way whether they're true or false — it has no fact-checker. Grounding it with real sources (RAG) is the main fix.", "uid": "12y28lyr45os2" }, { "id": "mcq-rw-l3-p1-pretrain", "shape": "mcq", "tags": [ "pretraining", "training-pipeline" ], "prompt": { "modality": "text", "value": "Which statement about the pretraining stage is correct?" }, "options": [ { "modality": "text", "value": "It trains on human-written (prompt, ideal-response) pairs to teach instruction-following", "short": "Trains on prompt-response pairs" }, { "modality": "text", "value": "It is the cheapest and fastest stage of the pipeline", "short": "Cheapest, fastest stage" }, { "modality": "text", "value": "It uses self-supervised next-token prediction with no human labels, and consumes most of the total training compute", "short": "Self-supervised, most compute" }, { "modality": "text", "value": "It uses human preference rankings to train a reward model", "short": "Preference-ranked reward model" } ], "correctIndex": 2, "explanation": "Pretraining runs self-supervised next-token prediction over trillions of tokens with no human labels, giving the model its broad world knowledge — and it's where most compute is spent. Learning from labeled pairs is SFT; preference rankings feed RLHF.", "uid": "3wbo351jdav8z" }, { "id": "mcq-rw-l3-p2-align", "shape": "mcq", "tags": [ "inference", "training-vs-inference" ], "prompt": { "modality": "text", "value": "During a normal chat, does an LLM learn from your messages and update itself?" }, "options": [ { "modality": "text", "value": "No — inference is read-only; the weights stay frozen and any 'memory' is just text in the current context window", "short": "No — weights frozen at inference" }, { "modality": "text", "value": "No — the model never actually processes your messages; it only replays memorized training text", "short": "No — just replays memorized text" }, { "modality": "text", "value": "Yes — each message updates the model's weights through gradient descent", "short": "Yes — updates weights per msg" }, { "modality": "text", "value": "Yes — the model fine-tunes itself on new facts after every conversation", "short": "Yes — self-fine-tunes each chat" } ], "correctIndex": 0, "explanation": "Training updates weights (rare, costly); inference just runs the frozen model and is read-only. The model isn't learning as you chat — its knowledge is fixed at the training cutoff, and any 'memory' is only the text sitting in the context window.", "uid": "z4lsy21mlt9qy" }, { "id": "mcq-rw-l4-p1-prompt", "shape": "mcq", "tags": [ "prompting", "limitations" ], "prompt": { "modality": "text", "value": "According to the guide, what is the key limitation of prompting alone?" }, "options": [ { "modality": "text", "value": "It is limited to knowledge the model already has", "short": "Limited to existing knowledge" }, { "modality": "text", "value": "It requires a vector database to function", "short": "Requires a vector database" }, { "modality": "text", "value": "It costs far more than fine-tuning", "short": "Costs more than fine-tuning" }, { "modality": "text", "value": "It permanently rewrites the model's weights", "short": "Permanently rewrites weights" } ], "correctIndex": 0, "explanation": "Prompting costs nothing but is bounded by context and the model's existing weights, so it can't supply knowledge the model never learned. That gap is exactly why RAG, function calling, and agents exist.", "uid": "fxzv5x1v4dl7b" }, { "id": "mcq-rw-l4-p2-rag", "shape": "mcq", "tags": [ "agents", "tool-use" ], "prompt": { "modality": "text", "value": "What makes an LLM agent different from a standard single-prompt interaction?" }, "options": [ { "modality": "text", "value": "It permanently updates the model's weights after each task", "short": "Updates weights each task" }, { "modality": "text", "value": "It runs a planning loop — selecting tools, acting, and observing results until a goal is met", "short": "Plans, uses tools in a loop" }, { "modality": "text", "value": "It inserts retrieved documents into the prompt before answering", "short": "Inserts retrieved docs in prompt" }, { "modality": "text", "value": "It splits documents into chunks before embedding them", "short": "Chunks docs before embedding" } ], "correctIndex": 1, "explanation": "An agent wraps tool use in a planning loop: the model acts as a controller that plans, picks tools, acts, and observes until the goal is met. Weight updates are fine-tuning, inserting retrieved docs is RAG, and splitting docs is chunking.", "uid": "1yzs0btjjpnx7" }, { "id": "mcq-rw-l5-p1-companies", "shape": "mcq", "tags": [ "companies", "open-source", "meta" ], "prompt": { "modality": "text", "value": "Which major AI lab became the driving force of open-source LLMs by releasing its model weights freely?" }, "options": [ { "modality": "text", "value": "OpenAI" }, { "modality": "text", "value": "Anthropic" }, { "modality": "text", "value": "Meta AI" }, { "modality": "text", "value": "Google DeepMind" } ], "correctIndex": 2, "explanation": "Meta AI makes the Llama family and, by releasing its weights freely, became the driving force of open-source LLMs. OpenAI, Anthropic, and Google DeepMind keep their frontier models closed.", "uid": "ndc9t2uftmuu" }, { "id": "mcq-rw-l5-p2-researchers", "shape": "mcq", "tags": [ "researchers", "turing-award" ], "prompt": { "modality": "text", "value": "Which trio are the 'godfathers of deep learning' who jointly received the 2018 Turing Award?" }, "options": [ { "modality": "text", "value": "Ashish Vaswani, Ilya Sutskever, and Demis Hassabis", "short": "Vaswani, Sutskever, Hassabis" }, { "modality": "text", "value": "Geoffrey Hinton, Ilya Sutskever, and Ashish Vaswani", "short": "Hinton, Sutskever, Vaswani" }, { "modality": "text", "value": "Yoshua Bengio, Yann LeCun, and Andrej Karpathy", "short": "Bengio, LeCun, Karpathy" }, { "modality": "text", "value": "Geoffrey Hinton, Yoshua Bengio, and Yann LeCun", "short": "Hinton, Bengio, LeCun" } ], "correctIndex": 3, "explanation": "Hinton, Bengio, and LeCun are the three 'godfathers of deep learning' and joint 2018 Turing Award recipients. Vaswani, Sutskever, Karpathy, and Hassabis are pivotal figures but not part of that trio.", "uid": "1dgqj9v7mbi1x" }, { "id": "mcq-rw-l5-p3-founders", "shape": "mcq", "tags": [ "founders", "anthropic", "openai" ], "prompt": { "modality": "text", "value": "Before co-founding Anthropic, what role did Dario Amodei hold — where he helped invent RLHF and led GPT-2 and GPT-3?" }, "options": [ { "modality": "text", "value": "CEO of Google DeepMind", "short": "CEO, Google DeepMind" }, { "modality": "text", "value": "Director of AI at Tesla", "short": "Director of AI, Tesla" }, { "modality": "text", "value": "VP of Research at OpenAI", "short": "VP of Research, OpenAI" }, { "modality": "text", "value": "Chief AI Scientist at Meta", "short": "Chief AI Scientist, Meta" } ], "correctIndex": 2, "explanation": "Dario Amodei was OpenAI's VP of Research — helping invent RLHF and leading GPT-2 and GPT-3 — before leaving in 2021 over safety concerns to co-found Anthropic with his sister Daniela.", "uid": "hp6qkc2n9slk" }, { "id": "mcq-rw-l6-p1-sizes", "shape": "mcq", "tags": [ "timeline", "gpt", "milestones" ], "prompt": { "modality": "text", "value": "Which sequence correctly orders these milestones in time?" }, "options": [ { "modality": "text", "value": "ChatGPT (2020) → GPT-3 (2022) → GPT-4 (2023)", "short": "ChatGPT → GPT-3 → GPT-4" }, { "modality": "text", "value": "GPT-4 (2020) → GPT-3 (2021) → ChatGPT (2022)", "short": "GPT-4 → GPT-3 → ChatGPT" }, { "modality": "text", "value": "GPT-3 (2020) → ChatGPT (late 2022) → GPT-4 (early 2023)", "short": "GPT-3 → ChatGPT → GPT-4" }, { "modality": "text", "value": "GPT-3 (2020) → GPT-4 (2022) → ChatGPT (2023)", "short": "GPT-3 → GPT-4 → ChatGPT" } ], "correctIndex": 2, "explanation": "The spine of the modern timeline: GPT-3 in 2020, ChatGPT on Nov 30, 2022, and GPT-4 on March 14, 2023 — the roughly eighteen months in which LLMs went from research artifact to mass technology.", "uid": "1gsm5sr1teu5t5" }, { "id": "mcq-rw-l6-p2-scaling", "shape": "mcq", "tags": [ "scaling", "misconceptions", "alignment" ], "prompt": { "modality": "text", "value": "Which finding best supports the claim that 'a bigger model is not always better'?" }, "options": [ { "modality": "text", "value": "Kaplan's scaling laws showed loss falls predictably as models grow", "short": "Scaling laws: loss falls w/ size" }, { "modality": "text", "value": "GPT-3 had far more parameters than GPT-2", "short": "GPT-3 bigger than GPT-2" }, { "modality": "text", "value": "GPT-4 launched with a larger context window than GPT-3", "short": "GPT-4's bigger context window" }, { "modality": "text", "value": "InstructGPT's 1.3-billion-parameter model was preferred by humans over the 175-billion GPT-3", "short": "1.3B InstructGPT beat 175B GPT-3" } ], "correctIndex": 3, "explanation": "Chinchilla and InstructGPT showed data quality and alignment can beat raw size — a 1.3B InstructGPT model was preferred over 175B GPT-3. The other statements are true but don't undercut model size.", "uid": "9s3s1tskohdn" }, { "id": "mcq-rw-l8-p1-language", "shape": "mcq", "tags": [ "translation", "language-tasks" ], "prompt": { "modality": "text", "value": "LLM translation reaches near-professional quality across major languages, but the section warns it is weakest on which inputs?" }, "options": [ { "modality": "text", "value": "Short marketing taglines", "short": "Short marketing taglines" }, { "modality": "text", "value": "Casual dialogue between two speakers", "short": "Casual two-speaker dialogue" }, { "modality": "text", "value": "Low-resource languages and technical or legal text", "short": "Low-resource langs, legal text" }, { "modality": "text", "value": "Idioms and common expressions in major languages", "short": "Idioms in major languages" } ], "correctIndex": 2, "explanation": "Translation handles major languages well and even manages idioms, but quality drops on low-resource languages and technical or legal text — get expert review for those.", "uid": "11ei2lv1a59r15" }, { "id": "mcq-rw-l8-p2-knowledge-code", "shape": "mcq", "tags": [ "rag", "enterprise" ], "prompt": { "modality": "text", "value": "The guide says RAG (Retrieval-Augmented Generation) beats fine-tuning for grounding an LLM in private company knowledge. Why?" }, "options": [ { "modality": "text", "value": "It's cheaper, the knowledge base updates without retraining, and proprietary data never enters the model's weights", "short": "Cheap, updatable, data stays out" }, { "modality": "text", "value": "It lets the model memorize the documents during training, so no storage is needed", "short": "Memorizes docs in training" }, { "modality": "text", "value": "It permanently bakes the company's documents into the model's weights for faster recall", "short": "Bakes docs into weights" }, { "modality": "text", "value": "It guarantees the model can browse the live web at query time", "short": "Browses live web at query time" } ], "correctIndex": 0, "explanation": "RAG retrieves relevant chunks at query time and injects them into the prompt, so the model answers from context — cheaper than retraining, updatable without it, and proprietary data stays out of the weights.", "uid": "36cti0cxgkn0" }, { "id": "mcq-rw-l8-p3-work", "shape": "mcq", "tags": [ "brainstorming", "limitations" ], "prompt": { "modality": "text", "value": "The section praises LLMs for brainstorming and ideation, but what limitation does it note?" }, "options": [ { "modality": "text", "value": "It always produces ideas that are too radical to be useful", "short": "Ideas too radical to use" }, { "modality": "text", "value": "It refuses to generate creative content without a fine-tuned model", "short": "Refuses without fine-tuning" }, { "modality": "text", "value": "Ideas cluster around training data, so truly novel domains get generic output", "short": "Clusters near training data" }, { "modality": "text", "value": "It can only generate one idea per prompt, making it slow", "short": "Only one idea per prompt" } ], "correctIndex": 2, "explanation": "LLMs generate many names, angles, and framings fast for a human to select — but because output reflects patterns in the training data, genuinely novel domains tend to get generic ideas.", "uid": "1iv1ef011f7t18" }, { "id": "mcq-rw-l8-p4-highstakes", "shape": "mcq", "tags": [ "human-in-the-loop", "high-stakes" ], "prompt": { "modality": "text", "value": "In the section's human-in-the-loop framework, what should determine how much automation is appropriate for a task?" }, "options": [ { "modality": "text", "value": "The length of the model's context window", "short": "Context window length" }, { "modality": "text", "value": "The size of the model being used", "short": "Size of the model" }, { "modality": "text", "value": "The cost of verifying the output", "short": "Cost of verifying output" }, { "modality": "text", "value": "The number of users the system serves", "short": "Number of users served" } ], "correctIndex": 2, "explanation": "Let the cost of verification set the checkpoint: cheap-to-check outputs (a draft, a unit test) allow more automation; expensive or impossible-to-verify actions (a trade, a court filing) require human approval first.", "uid": "1ne4a3i16lfu9e" }, { "id": "mcq-rw-l9-p1-reliability", "shape": "mcq", "tags": [ "hallucination", "reliability" ], "prompt": { "modality": "text", "value": "According to the guide, why do LLMs hallucinate?" }, "options": [ { "modality": "text", "value": "A rare software bug corrupts the model's stored facts", "short": "A software bug corrupts facts" }, { "modality": "text", "value": "They predict plausible text with no built-in mechanism to check whether it is factually grounded", "short": "Predict text, no fact-checking" }, { "modality": "text", "value": "They intentionally fabricate answers to seem more helpful", "short": "Fabricate to seem more helpful" }, { "modality": "text", "value": "Their training data was too small to cover the topic", "short": "Training data was too small" } ], "correctIndex": 1, "explanation": "Hallucination isn't a glitch — it's the model doing exactly what it was trained to do, predicting plausible text, with nothing that verifies factual grounding. That's why layered defenses like RAG and citations are needed.", "uid": "ilsiwt1u2l2yj" }, { "id": "mcq-rw-l9-p2-bias-privacy", "shape": "mcq", "tags": [ "privacy", "memorization", "pii" ], "prompt": { "modality": "text", "value": "The guide warns that LLMs can memorize and regurgitate fragments of training data. Which finding about extracting that data does it cite?" }, "options": [ { "modality": "text", "value": "Only base models memorize data; fine-tuning fully eliminates the leak", "short": "Fine-tuning eliminates leaks" }, { "modality": "text", "value": "Memorization affects only images, never text such as email addresses", "short": "Affects only images, not text" }, { "modality": "text", "value": "Multi-query extraction attacks surface far more PII than simple single-query probes, and fine-tuned models leak more than base models", "short": "Multi-query extracts more PII" }, { "modality": "text", "value": "Single-query probes extract the most PII of any method", "short": "Single-query extracts most PII" } ], "correctIndex": 2, "explanation": "Research shows multi-query extraction attacks surface far more personally identifiable information than simple probes, and fine-tuned models — especially those repeatedly trained on sensitive data — leak more than base models.", "uid": "gokf9du4pjdf" }, { "id": "mcq-rw-l9-p3-security", "shape": "mcq", "tags": [ "agents", "prompt-injection", "security" ], "prompt": { "modality": "text", "value": "Why does the guide say an agent (a model that can send email, run code, or browse) widens the attack surface compared with a text-only chatbot?" }, "options": [ { "modality": "text", "value": "It stores user data permanently, unlike a chatbot", "short": "Stores user data permanently" }, { "modality": "text", "value": "It has a longer context window, so more malicious instructions fit", "short": "Longer context fits attacks" }, { "modality": "text", "value": "If hijacked by prompt injection, it can take irreversible real-world actions, not just produce bad text", "short": "Can take real-world actions" }, { "modality": "text", "value": "It runs a larger model that is inherently easier to jailbreak", "short": "Larger model, easy to jailbreak" } ], "correctIndex": 2, "explanation": "A text-only chatbot has a limited blast radius; an agent that can act can cause irreversible harm if hijacked. The fix is structural: minimum necessary permissions plus human confirmation before consequential actions.", "uid": "1vtoz6c1d14vcc" }, { "id": "mcq-rw-l9-p4-governance", "shape": "mcq", "tags": [ "safety", "guardrails", "red-teaming" ], "prompt": { "modality": "text", "value": "Why does the guide say safeguards like content filters and output classifiers are not enough to make an LLM safe for high-stakes decisions?" }, "options": [ { "modality": "text", "value": "They only function correctly at temperature 0", "short": "Only work at temperature 0" }, { "modality": "text", "value": "Red-teamers consistently find ways to bypass them", "short": "Red-teamers bypass them" }, { "modality": "text", "value": "They multiply token costs beyond most budgets", "short": "Multiply token costs" }, { "modality": "text", "value": "They cannot run without RLHF fine-tuning", "short": "Need RLHF fine-tuning" } ], "correctIndex": 1, "explanation": "Guardrails reduce harmful output but don't eliminate it — red-teamers keep finding bypasses. OWASP's LLM Top 10 favors continuous training over static system-prompt guardrails, and human oversight remains necessary for high-stakes use.", "uid": "kxv3ye1i33ygm" }, { "id": "mcq-rw-l7-p1-recall", "shape": "mcq", "tags": [ "transformer", "attention", "architecture" ], "prompt": { "modality": "text", "value": "The 2017 Transformer architecture unlocked scale in modern LLMs by replacing what with attention?" }, "options": [ { "modality": "text", "value": "Pooling layers" }, { "modality": "text", "value": "Word embeddings" }, { "modality": "text", "value": "Convolution" }, { "modality": "text", "value": "Recurrence" } ], "correctIndex": 3, "explanation": "The Transformer replaced recurrence — the sequential, step-by-step processing of RNNs — with attention, letting whole sequences be processed in parallel. That shift is what unlocked the scale behind every modern LLM.", "uid": "9mhcdjh9u04h" }, { "id": "mcq-rw-lc-people", "shape": "mcq", "tags": [ "turing-award", "deep-learning", "researchers" ], "prompt": { "modality": "text", "value": "Yoshua Bengio and Geoffrey Hinton shared the 2018 Turing Award for deep learning with which third researcher?" }, "options": [ { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Andrew Ng" }, { "modality": "text", "value": "Alec Radford" }, { "modality": "text", "value": "Ilya Sutskever" } ], "correctIndex": 0, "explanation": "The 2018 Turing Award went jointly to Bengio, Hinton, and Yann LeCun for their foundational deep-learning work. LeCun is also known as a founding figure of convolutional neural networks.", "uid": "fmvaims2bdri" }, { "id": "mcq-rw-lc-things", "shape": "mcq", "tags": [ "gpt-3", "parameters", "openai" ], "prompt": { "modality": "text", "value": "According to the guide, how many parameters does GPT-3 have?" }, "options": [ { "modality": "text", "value": "13 billion" }, { "modality": "text", "value": "175 million" }, { "modality": "text", "value": "1.75 billion" }, { "modality": "text", "value": "175 billion" } ], "correctIndex": 3, "explanation": "GPT-3, released by OpenAI in 2020, has 175 billion parameters — one of the largest models of its time — and is known for strong few-shot learning, performing tasks without explicit training.", "uid": "1qfvjvqato0xa" }, { "id": "cz-llm-101-def-token", "shape": "cloze", "tags": [ "text-units", "mechanics", "tokenization" ], "template": "The basic unit of text an LLM processes — on average about three-quarters of a word; 'ChatGPT' is one ___, while 'unhappy' may split into 'un' + 'happy'. It matters because everything — context limits, pricing, and speed — is measured in ___, not words or characters.", "answer": "token", "distractors": [ "embedding", "tokenization", "hallucination" ], "derivedFrom": "def-token", "explanation": "Compare: embedding — A list of numbers (a vector) that represents a token's meaning in a high-dimensional space, where tokens with similar meanings sit close together. It matters because embeddings are how a model turns discrete text symbols into continuous math it can compute over.", "uid": "q9bv2f9sf029" }, { "id": "cz-llm-101-def-embedding-llm", "shape": "cloze", "tags": [ "text-units", "mechanics", "embeddings" ], "template": "A list of numbers (a vector) that represents a token's meaning in a high-dimensional space, where tokens with similar meanings sit close together. It matters because ___ are how a model turns discrete text symbols into continuous math it can compute over.", "answer": "embedding", "distractors": [ "token", "tokenization", "hallucination" ], "derivedFrom": "def-embedding-llm", "explanation": "Compare: token — The basic unit of text an LLM processes — on average about three-quarters of a word; 'ChatGPT' is one token, while 'unhappy' may split into 'un' + 'happy'. It matters because everything — context limits, pricing, and speed — is measured in tokens, not words or characters.", "uid": "mgbatykh0vxm" }, { "id": "cz-llm-101-def-Transformer", "shape": "cloze", "tags": [ "model-internals", "mechanics", "transformer" ], "template": "A neural network architecture that processes all tokens in a sequence at once, in parallel, using self-attention layers, rather than one token at a time like an RNN. It matters because that parallelism makes ___ vastly faster to train on GPUs, enabling a scale RNNs could never reach.", "answer": "Transformer", "distractors": [ "parameters (weights)", "self-attention", "context window" ], "derivedFrom": "def-Transformer", "uid": "1mindcs1aps97o" }, { "id": "cz-llm-101-def-logits", "shape": "cloze", "tags": [ "generation", "mechanics", "sampling" ], "template": "The raw, unnormalized scores a model assigns to every possible next token, converted into a probability distribution by the softmax function. It matters because understanding ___ clarifies what 'temperature' and other sampling knobs are actually adjusting.", "answer": "logits", "distractors": [ "next-token prediction", "autoregressive generation", "Code-trained LLMs" ], "derivedFrom": "def-logits", "explanation": "Compare: next-token prediction — The core objective: given all previous tokens, predict the single most probable next token, repeated over and over to generate text. It matters because this deceptively simple task, applied at massive scale, is what gives LLMs their broad language ability.", "uid": "yfs89b4n1gdl" }, { "id": "cz-llm-101-def-temperature", "shape": "cloze", "tags": [ "behavior", "mechanics", "sampling" ], "template": "A sampling knob (typically 0–2) that scales the logits: low ___ makes output more deterministic and focused; high ___ makes it more random and creative. It matters because it is the primary dial for controlling how predictable or varied a model's output is.", "answer": "temperature", "distractors": [ "hallucination", "emergent abilities", "context window" ], "derivedFrom": "def-temperature", "explanation": "Compare: hallucination — When an LLM generates fluent, confident-sounding text that is factually wrong or entirely fabricated, grounded in no real source. It matters because users often can't tell a hallucination from an accurate answer, making verification essential for high-stakes use.", "uid": "3e48s0qytv4w" }, { "id": "cz-llm-101-def-hallucination", "shape": "cloze", "tags": [ "behavior", "mechanics", "hallucination" ], "template": "When an LLM generates fluent, confident-sounding text that is factually wrong or entirely fabricated, grounded in no real source. It matters because users often can't tell a ___ from an accurate answer, making verification essential for high-stakes use.", "answer": "hallucination", "distractors": [ "emergent abilities", "temperature", "tokenization" ], "derivedFrom": "def-hallucination", "uid": "sbfjkh14n3pxz" }, { "id": "cz-llm-101-def-inference-llm", "shape": "cloze", "tags": [ "training", "inference" ], "template": "Running a trained, frozen model on a new input to produce an output, with no weights changed. It matters because ___ is what happens every time you send a message; the model's knowledge is locked at training time unless supplemented by tools or RAG.", "answer": "inference", "distractors": [ "pretraining", "fine-tuning", "RLHF" ], "derivedFrom": "def-inference-llm", "uid": "1prjdpr1c3kpp9" }, { "id": "cz-llm-101-def-prompt", "shape": "cloze", "tags": [ "prompt-basics", "prompting" ], "template": "The input sent to an LLM, typically a system message (behavior instructions), zero or more prior turns, and the current user message. It matters because the ___ is the only way to steer a frozen model, which makes ___ engineering a high-leverage skill.", "answer": "prompt", "distractors": [ "zero-shot prompting", "system prompt", "in-context learning" ], "derivedFrom": "def-prompt", "explanation": "Compare: zero-shot prompting — Asking a model to perform a task with no examples — just a plain instruction, such as 'Translate this to French.' It matters because it tests a model's general capability; large models handle zero-shot well, while smaller ones often need examples.", "uid": "1weq7sb1cycwhp" }, { "id": "czr-llm-101-x-uc-copilot-def", "shape": "cloze", "tags": [ "use-case", "patterns", "human-in-the-loop" ], "template": "___ — A design where the LLM suggests, drafts, or assists and a human approves before anything takes effect — accelerating the work without finalizing it. It contrasts with autonomous agents that act without per-step review", "answer": "copilot pattern", "distractors": [ "prompt chaining", "retrieval augmentation", "tool-use pattern" ], "derivedFrom": "x-uc-copilot-def", "uid": "zthsu41y78xps" }, { "id": "cz-llm-101-f-rnn-lstm", "shape": "cloze", "tags": [ "history", "pre-transformer" ], "template": "___ and their improved variant, Long Short-Term Memory (LSTM) networks, processed text sequentially, keeping a hidden 'memory' of earlier words. They dominated NLP from roughly 2014-2017 but struggled with long sequences due to vanishing gradients.", "answer": "Recurrent Neural Networks", "distractors": [ "Convolutional Neural Networks", "Feedforward Neural Networks", "Generative Adversarial Networks" ], "derivedFrom": "f-rnn-lstm", "uid": "1rpuqgktwyblk" }, { "id": "cz-llm-101-f-bert", "shape": "cloze", "tags": [ "history", "bert" ], "template": "Devlin and colleagues at Google released BERT in October ___. Unlike left-to-right models, BERT was pretrained to predict masked words using context from both sides of a sentence, setting new state-of-the-art results on eleven NLP benchmarks.", "answer": "2018", "distractors": [ "2011", "2027", "1994" ], "derivedFrom": "f-bert", "uid": "1a627vv12l1jkt" }, { "id": "cz-llm-101-f-gpt3", "shape": "cloze", "tags": [ "history", "gpt", "few-shot" ], "template": "OpenAI released GPT-3 in ___ with 175 billion parameters — over 10× larger than any prior dense model. Its key finding: given only a few examples in the prompt ('few-shot'), it could perform tasks it was never explicitly trained on, demonstrating in-context learning at scale.", "answer": "2020", "distractors": [ "2013", "2029", "1996" ], "derivedFrom": "f-gpt3", "uid": "hpxglr1u5xtad" }, { "id": "cz-llm-101-f-instructgpt", "shape": "cloze", "tags": [ "history", "rlhf" ], "template": "OpenAI showed that fine-tuning GPT-3 with ___ from Human Feedback sharply improved helpfulness and reduced harmful output. Remarkably, a 1.3B InstructGPT was preferred by human raters over the raw 175B GPT-3 — bigger is not automatically better.", "answer": "Reinforcement Learning", "distractors": [ "reward modeling", "Constitutional AI", "self-supervised pretraining" ], "derivedFrom": "f-instructgpt", "explanation": "Compare: Constitutional AI — Anthropic's alignment approach, built on top of the AI-feedback idea.", "uid": "z4vlimos8gtu" }, { "id": "cz-llm-101-f-chatgpt-launch", "shape": "cloze", "tags": [ "history", "chatgpt" ], "template": "OpenAI released ChatGPT as a free public demo on November 30, ___, powered by a GPT-3.5-class model fine-tuned with RLHF. It reached 1 million users in five days and about 100 million monthly users within two months — the fastest-growing consumer app in history at the time.", "answer": "2022", "distractors": [ "2015", "2031", "1998" ], "derivedFrom": "f-chatgpt-launch", "uid": "vqqj5k17zraro" }, { "id": "cz-llm-101-f-gpt4", "shape": "cloze", "tags": [ "history", "gpt-4" ], "template": "OpenAI released GPT-4 on March 14, ___. It was multimodal (accepting image and text input), scored at or above human level on many professional exams, and launched with an 8,000-token context window, with a 32K variant following soon after.", "answer": "2023", "distractors": [ "2016", "2032", "1999" ], "derivedFrom": "f-gpt4", "uid": "19cksyr1y4njlh" }, { "id": "cz-llm-101-f-mistral", "shape": "cloze", "tags": [ "history", "open-weights", "mistral" ], "template": "___ and Meta researchers founded Mistral AI in Paris in 2023. Its Mistral 7B (September 2023), released under Apache 2.0, became the most-downloaded open model within weeks, and Mixtral 8×7B matched GPT-3.5 using a Mixture-of-Experts design.", "answer": "Former DeepMind", "distractors": [ "Former OpenAI", "Former Anthropic", "Former Microsoft" ], "derivedFrom": "f-mistral", "uid": "10sj35j15mnf7d" }, { "id": "cz-llm-101-f-claude-gemini", "shape": "cloze", "tags": [ "history", "anthropic", "google" ], "template": "Anthropic released Claude (2023), built with ___ and RLAIF alignment, while Google released Gemini (December 2023) as its multimodal flagship, replacing Bard and developed by Google DeepMind. Both are major non-OpenAI frontier model families.", "answer": "Constitutional AI", "distractors": [ "supervised fine-tuning", "reward modeling", "instruction tuning" ], "derivedFrom": "f-claude-gemini", "explanation": "Compare: Supervised fine-tuning — The pipeline stage that first turns a raw text-predictor into something that answers questions.", "uid": "mud70ttlq1q3" }, { "id": "tf-t-llm-101-recall-transformer", "shape": "trueFalse", "tags": [ "architecture", "recall" ], "statement": "\"Transformer\" refers to \"The 2017 architecture behind every modern LLM\".", "isTrue": true, "why": "Transformer refers to The 2017 architecture behind every modern LLM.", "derivedFrom": "recall-transformer", "uid": "veewx7mvkc51" }, { "id": "tf-t-llm-101-recall-pretraining", "shape": "trueFalse", "tags": [ "training", "recall" ], "statement": "\"Pretraining\" refers to \"Self-supervised next-token training on huge corpora\".", "isTrue": true, "why": "Pretraining refers to Self-supervised next-token training on huge corpora.", "derivedFrom": "recall-pretraining", "uid": "1m99dx0featng" }, { "id": "tf-f-llm-101-recall-rlhf", "shape": "trueFalse", "tags": [ "training", "recall" ], "statement": "\"RLHF\" refers to \"Self-supervised next-token training on huge corpora\".", "isTrue": false, "why": "RLHF actually refers to Aligning a model with human preference feedback.", "derivedFrom": "recall-rlhf", "uid": "1oh5f5urn4ox2" }, { "id": "tf-f-llm-101-recall-rag", "shape": "trueFalse", "tags": [ "inference", "recall" ], "statement": "\"RAG\" refers to \"Capabilities that appear only past a certain model scale\".", "isTrue": false, "why": "RAG actually refers to Retrieving documents into the prompt for grounding.", "derivedFrom": "recall-rag", "uid": "cakh9injq20m" }, { "id": "tf-t-llm-101-recall-hallucination", "shape": "trueFalse", "tags": [ "inference", "recall" ], "statement": "\"Hallucination\" refers to \"Fluent output that is confidently fabricated\".", "isTrue": true, "why": "Hallucination refers to Fluent output that is confidently fabricated.", "derivedFrom": "recall-hallucination", "uid": "mf1e38vair74" }, { "id": "tf-t-llm-101-recall-context", "shape": "trueFalse", "tags": [ "architecture", "recall" ], "statement": "\"Context window\" refers to \"The model's token-limited working memory\".", "isTrue": true, "why": "Context window refers to The model's token-limited working memory.", "derivedFrom": "recall-context", "uid": "1ft042q18y399i" }, { "id": "tf-t-llm-101-q-llm-bias-stages", "shape": "trueFalse", "tags": [ "bias", "fairness" ], "statement": "\"Stages where bias enters an LLM\" refers to \"Data collection, pretraining, and fine-tuning\".", "isTrue": true, "why": "Stages where bias enters an LLM refers to Data collection, pretraining, and fine-tuning.", "derivedFrom": "q-llm-bias-stages", "uid": "1vfjj651l8mgtr" }, { "id": "tf-f-llm-101-q-llm-bias-aave-gap", "shape": "trueFalse", "tags": [ "bias", "fairness" ], "statement": "\"A documented LLM accuracy gap\" refers to \"Data collection, pretraining, and fine-tuning\".", "isTrue": false, "why": "A documented LLM accuracy gap actually refers to Worse on African-American Vernacular English than Standard American English.", "derivedFrom": "q-llm-bias-aave-gap", "uid": "15fy82vcjqhyp" }, { "id": "tf-d-llm-101-def-token", "shape": "trueFalse", "tags": [ "text-units", "mechanics", "tokenization" ], "statement": "token: The basic unit of text an LLM processes — on average about three-quarters of a word; 'ChatGPT' is one token, while 'unhappy' may split into 'un' + 'happy'.", "isTrue": true, "why": "That definition is correct for \"token\".", "derivedFrom": "def-token", "uid": "ztby09rfhejr" }, { "id": "tf-d-llm-101-def-tokenization", "shape": "trueFalse", "tags": [ "text-units", "mechanics", "tokenization" ], "statement": "tokenization: The process of splitting raw text into tokens before feeding it to a model, using algorithms such as Byte-Pair Encoding (BPE).", "isTrue": true, "why": "That definition is correct for \"tokenization\".", "derivedFrom": "def-tokenization", "uid": "11yk0q183bs9q" }, { "id": "tf-d-llm-101-def-embedding-llm", "shape": "trueFalse", "tags": [ "text-units", "mechanics", "embeddings" ], "statement": "embedding: A list of numbers (a vector) that represents a token's meaning in a high-dimensional space, where tokens with similar meanings sit close together.", "isTrue": true, "why": "That definition is correct for \"embedding\".", "derivedFrom": "def-embedding-llm", "uid": "1hyo0817e2jzw" }, { "id": "tf-d-llm-101-def-Transformer", "shape": "trueFalse", "tags": [ "model-internals", "mechanics", "transformer" ], "statement": "Transformer: A neural network architecture that processes all tokens in a sequence at once, in parallel, using self-attention layers, rather than one token at a time like an RNN.", "isTrue": true, "why": "That definition is correct for \"Transformer\".", "derivedFrom": "def-Transformer", "uid": "1o98shpuljjj7" }, { "id": "tf-d-llm-101-def-self-attention", "shape": "trueFalse", "tags": [ "model-internals", "mechanics", "attention" ], "statement": "self-attention: A mechanism where each token looks at every other token in the context and learns how much to attend to (weight) each one when forming its own representation.", "isTrue": true, "why": "That definition is correct for \"self-attention\".", "derivedFrom": "def-self-attention", "uid": "wieiez5j6355" }, { "id": "tf-d-llm-101-def-next-token", "shape": "trueFalse", "tags": [ "generation", "mechanics", "pretraining" ], "statement": "next-token prediction: The core objective: given all previous tokens, predict the single most probable next token, repeated over and over to generate text.", "isTrue": true, "why": "That definition is correct for \"next-token prediction\".", "derivedFrom": "def-next-token", "uid": "f06a0betnmdp" }, { "id": "tf-df-llm-101-def-autoregressive", "shape": "trueFalse", "tags": [ "generation", "mechanics" ], "statement": "next-token prediction: Producing text one token at a time, where each new token is chosen based on all prior tokens — including the ones the model just generated.", "isTrue": false, "why": "That's the definition of \"autoregressive generation\", not \"next-token prediction\".", "derivedFrom": "def-autoregressive", "uid": "1ti2jh2bbzt3u" }, { "id": "tf-df-llm-101-def-logits", "shape": "trueFalse", "tags": [ "generation", "mechanics", "sampling" ], "statement": "next-token prediction: The raw, unnormalized scores a model assigns to every possible next token, converted into a probability distribution by the softmax function.", "isTrue": false, "why": "That's the definition of \"logits\", not \"next-token prediction\".", "derivedFrom": "def-logits", "uid": "9ouj7l8jmad3" }, { "id": "tf-df-llm-101-def-temperature", "shape": "trueFalse", "tags": [ "behavior", "mechanics", "sampling" ], "statement": "hallucination: A sampling knob (typically 0–2) that scales the logits: low temperature makes output more deterministic and focused; high temperature makes it more random and creative.", "isTrue": false, "why": "That's the definition of \"temperature\", not \"hallucination\".", "derivedFrom": "def-temperature", "uid": "cybh4zwt9fxt" }, { "id": "tf-d-llm-101-def-hallucination", "shape": "trueFalse", "tags": [ "behavior", "mechanics", "hallucination" ], "statement": "hallucination: When an LLM generates fluent, confident-sounding text that is factually wrong or entirely fabricated, grounded in no real source.", "isTrue": true, "why": "That definition is correct for \"hallucination\".", "derivedFrom": "def-hallucination", "uid": "skduod267ouv" }, { "id": "tf-d-llm-101-def-pretraining", "shape": "trueFalse", "tags": [ "before-alignment", "training", "pretraining" ], "statement": "pretraining: Stage one: the model trains on a massive, diverse text corpus (trillions of tokens from the web, books, and code) using self-supervised next-token prediction, with no human labels.", "isTrue": true, "why": "That definition is correct for \"pretraining\".", "derivedFrom": "def-pretraining", "uid": "iyy3stg7lmwv" }, { "id": "tf-d-llm-101-def-base-model", "shape": "trueFalse", "tags": [ "before-alignment", "training", "pretraining" ], "statement": "base model: A pretrained-only LLM that has learned to predict text but hasn't been tuned to follow instructions or be helpful; it simply continues a prompt, often in unexpected ways.", "isTrue": true, "why": "That definition is correct for \"base model\".", "derivedFrom": "def-base-model", "uid": "18mc30rgp12ot" }, { "id": "tf-d-llm-101-def-sft", "shape": "trueFalse", "tags": [ "before-alignment", "training", "sft" ], "statement": "supervised fine-tuning (SFT): Stage two: the base model is trained on a curated set of (prompt, ideal-response) pairs written or selected by humans, teaching it to follow instructions rather than just continue text.", "isTrue": true, "why": "That definition is correct for \"supervised fine-tuning (SFT)\".", "derivedFrom": "def-sft", "uid": "4vj5jhj693c7" }, { "id": "tf-d-llm-101-def-reward-model", "shape": "trueFalse", "tags": [ "alignment", "training", "rlhf" ], "statement": "reward model: A separate neural network trained on human preference data that scores how good a model response is, used as the optimization target during the RL phase of RLHF.", "isTrue": true, "why": "That definition is correct for \"reward model\".", "derivedFrom": "def-reward-model", "uid": "aj7jbaksbery" }, { "id": "tf-df-llm-101-def-instruct-model", "shape": "trueFalse", "tags": [ "training", "sft", "rlhf" ], "statement": "supervised fine-tuning (SFT): An LLM that has gone through SFT and RLHF (or equivalent) on top of a base model, making it follow instructions, refuse harmful requests, and converse naturally — for example, ChatGPT.", "isTrue": false, "why": "That's the definition of \"instruct / chat model\", not \"supervised fine-tuning (SFT)\".", "derivedFrom": "def-instruct-model", "uid": "d7f8gq177bgea" }, { "id": "tf-d-llm-101-def-inference-llm", "shape": "trueFalse", "tags": [ "training", "inference" ], "statement": "inference: Running a trained, frozen model on a new input to produce an output, with no weights changed.", "isTrue": true, "why": "That definition is correct for \"inference\".", "derivedFrom": "def-inference-llm", "uid": "1y2i09b13jvfeh" }, { "id": "tf-df-llm-101-def-flops", "shape": "trueFalse", "tags": [ "training", "compute" ], "statement": "Constitutional AI (CAI): Floating Point Operations — a measure of computational work.", "isTrue": false, "why": "That's the definition of \"FLOPs\", not \"Constitutional AI (CAI)\".", "derivedFrom": "def-flops", "uid": "mc4t6f1ue87n9" }, { "id": "tf-df-llm-101-def-prompt", "shape": "trueFalse", "tags": [ "prompt-basics", "prompting" ], "statement": "zero-shot prompting: The input sent to an LLM, typically a system message (behavior instructions), zero or more prior turns, and the current user message.", "isTrue": false, "why": "That's the definition of \"prompt\", not \"zero-shot prompting\".", "derivedFrom": "def-prompt", "uid": "16rrusa89vcm2" }, { "id": "tf-df-llm-101-def-system-prompt", "shape": "trueFalse", "tags": [ "prompt-basics", "prompting" ], "statement": "zero-shot prompting: The special instruction block, usually hidden from the end user, that sets the model's persona, constraints, and task context at the start of a conversation.", "isTrue": false, "why": "That's the definition of \"system prompt\", not \"zero-shot prompting\".", "derivedFrom": "def-system-prompt", "uid": "1ol2r1c1wvxow8" }, { "id": "tf-d-llm-101-def-few-shot", "shape": "trueFalse", "tags": [ "prompt-techniques", "prompting", "few-shot" ], "statement": "few-shot prompting: Providing one to several worked examples in the prompt before asking the model to handle a new case.", "isTrue": true, "why": "That definition is correct for \"few-shot prompting\".", "derivedFrom": "def-few-shot", "uid": "pjgjm21uipqde" }, { "id": "tf-df-llm-101-def-icl", "shape": "trueFalse", "tags": [ "prompt-techniques", "prompting", "in-context-learning" ], "statement": "few-shot prompting: An LLM's ability to adapt its behavior from examples or instructions in the prompt, without changing any weights.", "isTrue": false, "why": "That's the definition of \"in-context learning\", not \"few-shot prompting\".", "derivedFrom": "def-icl", "uid": "74svxf4jqaj1" }, { "id": "tf-df-llm-101-def-rag", "shape": "trueFalse", "tags": [ "rag" ], "statement": "vector database: A pattern in which relevant documents are retrieved from an external store and inserted into the prompt as context before the model answers.", "isTrue": false, "why": "That's the definition of \"RAG (Retrieval-Augmented Generation)\", not \"vector database\".", "derivedFrom": "def-rag", "uid": "4k605o2r8djc" }, { "id": "tf-df-llm-101-def-vector-db", "shape": "trueFalse", "tags": [ "rag", "embeddings" ], "statement": "chunking: A specialized database that stores document embeddings and supports fast similarity search, returning the most semantically relevant chunks for a query.", "isTrue": false, "why": "That's the definition of \"vector database\", not \"chunking\".", "derivedFrom": "def-vector-db", "uid": "4yteo3mqsn05" }, { "id": "tf-d-llm-101-def-chunking", "shape": "trueFalse", "tags": [ "rag" ], "statement": "chunking: Splitting source documents into smaller segments before embedding them for RAG, so retrieved passages fit the context window and stay semantically focused.", "isTrue": true, "why": "That definition is correct for \"chunking\".", "derivedFrom": "def-chunking", "uid": "1h8e29r1gwa9b1" }, { "id": "tf-df-llm-101-def-tool-use", "shape": "trueFalse", "tags": [ "agents", "tool-use" ], "statement": "LLM agent: A structured way to let an LLM invoke external tools — APIs, calculators, databases — by emitting a well-formed call the host application runs, then feeding the result back to the model.", "isTrue": false, "why": "That's the definition of \"function calling / tool use\", not \"LLM agent\".", "derivedFrom": "def-tool-use", "uid": "10342s8hut2og" }, { "id": "tf-df-llm-101-x-uc-copilot-def", "shape": "trueFalse", "tags": [ "use-case", "patterns", "human-in-the-loop" ], "statement": "few-shot prompting: A design where the LLM suggests, drafts, or assists and a human approves before anything takes effect — accelerating the work without finalizing it.", "isTrue": false, "why": "That's the definition of \"copilot pattern\", not \"few-shot prompting\".", "derivedFrom": "x-uc-copilot-def", "uid": "191b3u6151bfx6" }, { "id": "tf-d-llm-101-x-lim-cutoff", "shape": "trueFalse", "tags": [ "reliability", "knowledge-cutoff" ], "statement": "knowledge cutoff: The date after which a model's training data ends; it has no intrinsic knowledge of later events.", "isTrue": true, "why": "That definition is correct for \"knowledge cutoff\".", "derivedFrom": "x-lim-cutoff", "uid": "zuqm6i1fo41pi" }, { "id": "tf-d-llm-101-x-lim-injection", "shape": "trueFalse", "tags": [ "security", "prompt-injection" ], "statement": "prompt injection: An attack where malicious text — in the user's input or in a document the model retrieves — overrides the system prompt and makes the model act against intent (OWASP LLM01).", "isTrue": true, "why": "That definition is correct for \"prompt injection\".", "derivedFrom": "x-lim-injection", "uid": "1ujzuw18hwmu8" }, { "id": "tf-d-llm-101-x-lim-jailbreak", "shape": "trueFalse", "tags": [ "security", "jailbreak" ], "statement": "jailbreak: A prompt injection aimed specifically at bypassing a model's safety guardrails, coaxing it to produce content it is trained to refuse.", "isTrue": true, "why": "That definition is correct for \"jailbreak\".", "derivedFrom": "x-lim-jailbreak", "uid": "107lh6s1bgrnf8" }, { "id": "tf-d-llm-101-x-lim-automation-bias", "shape": "trueFalse", "tags": [ "societal", "automation-bias" ], "statement": "automation bias: The human tendency to over-trust and under-scrutinize automated output — accepting an LLM's confident-sounding answer without applying one's own judgment.", "isTrue": true, "why": "That definition is correct for \"automation bias\".", "derivedFrom": "x-lim-automation-bias", "uid": "gd47eaed3vdi" }, { "id": "tf-df-llm-101-x-lim-redteam", "shape": "trueFalse", "tags": [ "governance", "red-teaming" ], "statement": "model card: Structured adversarial testing of an LLM system — by internal or external specialists — to find jailbreaks, bias failures, and safety gaps before deployment.", "isTrue": false, "why": "That's the definition of \"red-teaming\", not \"model card\".", "derivedFrom": "x-lim-redteam", "uid": "1mbvjx26x22wu" }, { "id": "concept-rw-llm-101-gpt-1", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "gpt", "llm-history" ], "name": "GPT-1", "clues": [ "OpenAI's 2018 debut in what became a famous numbered family of models.", "With just 117 million parameters, trained on BooksCorpus, it proved one pretrained model could be cheaply adapted to match purpose-built systems.", "The original and smallest of its line, it established the 'pretrain once, adapt cheaply' recipe every successor inherited." ], "uid": "1oz1k6v19nq3ft" }, { "id": "concept-rw-llm-101-gpt-2", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "gpt", "llm-history" ], "name": "GPT-2", "clues": [ "A 2019 OpenAI model with 1.5 billion parameters, trained on 40 GB of web text called WebText.", "Its paragraphs were coherent enough that OpenAI staged a phased release rather than publishing it all at once.", "It was initially withheld over fears it could mass-produce disinformation — the field's first big public capability-risk debate." ], "uid": "ujtu3ktdh9b8" }, { "id": "concept-rw-llm-101-gpt-4", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "gpt", "modern-wave" ], "name": "GPT-4", "clues": [ "Launched March 14, 2023, this OpenAI model added multimodality, accepting image as well as text input.", "It scored above the human average on bar exams and medical licensing tests.", "OpenAI published no parameter count for it, and its context window began at 8,000 tokens." ], "uid": "x77z9q11us8q6" }, { "id": "concept-rw-llm-101-llama", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "open-model", "modern-wave" ], "name": "LLaMA", "clues": [ "A family of models Meta released in February 2023, ranging from 7B to 65B parameters.", "It was trained exclusively on publicly available data like Common Crawl, GitHub, and Wikipedia.", "Its 13B version outperformed a 175B model, and a leaked copy ignited an open-source fine-tuning ecosystem." ], "uid": "ux2g2y1p5dmtu" }, { "id": "concept-rw-llm-101-mistral-7b", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "open-model", "modern-wave" ], "name": "Mistral 7B", "clues": [ "A September 2023 open model released under the permissive Apache 2.0 license.", "Within weeks it became the most-downloaded open model.", "The first release from a Paris startup founded by former DeepMind and Meta researchers." ], "uid": "ideej6qcgifi" }, { "id": "concept-rw-llm-101-embedding", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "mechanics" ], "name": "Embedding", "clues": [ "The step that converts a discrete symbol into continuous math a model can actually compute over.", "It's a long list of numbers — a vector — placing a token's meaning in a high-dimensional space where similar meanings sit close together.", "Every major NLP model since word2vec relies on some learned form of these to represent meaning geometrically." ], "uid": "1bkyh1zy0xj8p" }, { "id": "concept-rw-llm-101-self-attention", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "mechanics" ], "name": "Self-attention", "clues": [ "The Transformer's key trick, and the mechanism that let the architecture drop recurrence entirely.", "Every token in a sequence looks at every other token and learns how much to weight each one.", "It's how a model links a pronoun to the noun it refers to, no matter how far apart they sit." ], "uid": "1r6weu11wm637r" }, { "id": "concept-rw-llm-101-context-window", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "mechanics" ], "name": "Context window", "clues": [ "A model's working memory: anything outside it is simply invisible to the model.", "It's the maximum number of tokens the model can see at once.", "For GPT-4 this began at 8,000 tokens; exceed it and the earliest text drops out of view." ], "uid": "1r8wvxib4ks8u" }, { "id": "concept-rw-llm-101-temperature", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "generation" ], "name": "Temperature", "clues": [ "The main dial for controlling how predictable a model's responses are.", "Set it low for focused, deterministic output; set it high to flatten the odds and invite surprise.", "It reshapes the probabilities derived from the model's logits before the next token is chosen." ], "uid": "ezswcq1tcl07q" }, { "id": "concept-rw-llm-101-hallucination", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "generation" ], "name": "Hallucination", "clues": [ "The technology's single most-discussed flaw.", "Fluent, confident text that happens to be simply wrong or fabricated.", "It occurs because the model produces plausible text, not verified facts; grounding it with real sources via RAG is the main fix." ], "uid": "z8ojzzaf5c9" }, { "id": "concept-rw-llm-101-pretraining", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "training" ], "name": "Pretraining", "clues": [ "The stage where the overwhelming majority of a model's compute is spent.", "It trains on trillions of tokens from web, books, and code using self-supervised next-token prediction, with no human labels.", "Its output is a base model with broad world knowledge that hasn't yet been taught to follow instructions." ], "uid": "b1vu591i840fn" }, { "id": "concept-rw-llm-101-supervised-fine-tuning", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "training" ], "name": "Supervised fine-tuning", "clues": [ "The pipeline stage that first turns a raw text-predictor into something that answers questions.", "It trains a base model on curated pairs of a prompt and its ideal response, written or selected by humans.", "It comes right after pretraining and just before the RLHF stage, teaching the model to follow directions rather than merely continue text." ], "uid": "z06wtz1noytpt" }, { "id": "concept-rw-llm-101-rlhf", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "training", "alignment" ], "name": "RLHF", "clues": [ "The training stage that made modern assistants finally feel aligned.", "Human raters compare pairs of outputs; those preferences train a reward model the LLM is then tuned to satisfy.", "Dario Amodei helped invent this preference-based alignment method at OpenAI; a variant swaps in AI-generated labels." ], "uid": "raeqz913xw3r7" }, { "id": "concept-rw-llm-101-constitutional-ai", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "alignment" ], "name": "Constitutional AI", "clues": [ "Anthropic's alignment approach, built on top of the AI-feedback idea.", "The model critiques and revises its own outputs against an explicit, written set of guiding principles.", "The payoff is far less reliance on human harm-labeling, and a value set that's auditable rather than buried in opaque preference data." ], "uid": "1sav9i6hl33l2" }, { "id": "concept-rw-llm-101-instruct-or-chat-model", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "alignment" ], "name": "Instruct or chat model", "clues": [ "The finished product of the whole pipeline: base model, plus SFT, plus RLHF or its equivalent.", "It follows directions, refuses harmful requests, and converses naturally.", "It's the kind of model nearly everyone actually interacts with, behaving very differently from the raw base model underneath." ], "uid": "1lax1wl1dcg1pz" }, { "id": "concept-rw-llm-101-rag", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "rag-tools" ], "name": "RAG", "clues": [ "A pattern that gives a model knowledge it wasn't trained on, without any retraining.", "It fetches relevant documents from an external store and inserts them as context before the model answers.", "Ideal for fresh or proprietary knowledge, it depends on a vector database and on chunking documents to fit the context window." ], "uid": "bkhuqj1b9vwvp" }, { "id": "concept-rw-llm-101-function-calling", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "rag-tools" ], "name": "Function calling", "clues": [ "One of three patterns for real-world capability, also known simply as tool use.", "The model emits a structured request to an external API.", "The host executes that request and feeds the result back into the model." ], "uid": "1sshp421my7e7i" }, { "id": "concept-rw-llm-101-llm-agent", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "rag-tools" ], "name": "LLM agent", "clues": [ "The most autonomous of the three patterns for extending a model.", "It wraps tool use in a planning loop.", "The model becomes a controller that plans, selects tools, takes actions, and observes results until a goal is met." ], "uid": "127tb38yscwko" }, { "id": "concept-rw-llm-101-vector-database", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "rag-tools" ], "name": "Vector database", "clues": [ "The storage engine that makes RAG possible.", "It holds document embeddings and runs fast similarity search over them.", "RAG queries it to find the most relevant chunks before the model answers." ], "uid": "abps5ghpdva0" }, { "id": "concept-rw-llm-101-fine-tuning", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "rag-tools" ], "name": "Fine-tuning", "clues": [ "The one approach here that actually changes a model's weights rather than just its context.", "It bakes persistent behavior into the model — expensive, but durable across calls.", "Unlike prompting or RAG, its effect survives from one call to the next because it lives in the weights themselves." ], "uid": "1eopz6g16ihsuo" }, { "id": "concept-rw-llm-101-openai", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "company" ], "name": "OpenAI", "clues": [ "An AI lab founded in 2015, long the industry's capability frontier-setter.", "It makes the GPT family and created ChatGPT.", "Sam Altman has been its CEO since 2019, and Dario Amodei once served as its VP of Research." ], "uid": "1t9h9v11w60tzb" }, { "id": "concept-rw-llm-101-anthropic", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "company" ], "name": "Anthropic", "clues": [ "An AI lab founded in 2021 by ex-OpenAI researchers, with a safety-first research focus.", "It makes the Claude family and pioneered Constitutional AI.", "Its founders are the siblings Dario and Daniela Amodei." ], "uid": "1lyb2wf1tmzg3d" }, { "id": "concept-rw-llm-101-google-deepmind", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "company" ], "name": "Google DeepMind", "clues": [ "The lab behind AlphaGo and AlphaFold.", "Led by Demis Hassabis, it makes the Gemini family of models.", "Gemini, its multimodal flagship, is the successor to Bard and PaLM." ], "uid": "o5fscg1jy71t0" }, { "id": "concept-rw-llm-101-meta-ai", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "company" ], "name": "Meta AI", "clues": [ "By releasing its model weights freely, this lab became the driving force of open-source LLMs.", "It makes the Llama family of models.", "Its Chief AI Scientist is Yann LeCun, inventor of convolutional networks." ], "uid": "180wg4jeqnoqh" }, { "id": "concept-rw-llm-101-mistral-ai", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "company" ], "name": "Mistral AI", "clues": [ "A Paris-based startup founded in 2023, waving a banner of European AI sovereignty.", "It's known for efficient, Apache-licensed open-weight models.", "Its 7B model became the most-downloaded open model within weeks of release." ], "uid": "1p2hosme3u4ye" }, { "id": "concept-rw-llm-101-instructgpt", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "model", "alignment" ], "name": "InstructGPT", "clues": [ "A model used to puncture the myth that a bigger model is always better.", "Its 1.3-billion-parameter version was preferred by humans over a model more than a hundred times larger.", "Human raters favored its output over the 175-billion-parameter flagship, showing alignment can beat raw size." ], "uid": "1ad16av1p7kytt" }, { "id": "concept-rw-llm-101-healthcare", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "high-stakes", "medicine" ], "name": "Healthcare", "clues": [ "A high-stakes domain where LLMs help but must never be the final decision-maker.", "Useful for summarizing clinical notes and drafting discharge summaries, where the bottleneck is reading and writing volume.", "Its signature danger is a wrong dosage or invented drug interaction, so every output needs clinician review." ], "uid": "1f0r0duuflvmu" }, { "id": "concept-rw-llm-101-legal", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "high-stakes", "law" ], "name": "Legal", "clues": [ "A high-stakes domain where an LLM's research is a starting point, never the finished product.", "Strong at contract summarization, clause extraction, and statute research.", "Its cautionary tale: in 2023 two attorneys were sanctioned for filing a brief citing cases ChatGPT had fabricated." ], "uid": "12egns9179b35n" }, { "id": "concept-rw-llm-101-finance", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "high-stakes", "regulation" ], "name": "Finance", "clues": [ "A high-stakes domain where LLM outputs fall squarely under existing regulation.", "Useful for earnings-call summarization, filing sentiment analysis, and extracting structured data from dense reports.", "Best practice here is RAG over verified data sources plus mandatory human review before anything reaches a customer." ], "uid": "11y2h23hm8jql" }, { "id": "concept-rw-llm-101-knowledge-cutoff", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "risk", "staleness" ], "name": "Knowledge cutoff", "clues": [ "A source of staleness baked into every model at training time.", "It's the date after which a model's training data ends, leaving it blind to later events.", "Because models ship months later, they'll confidently discuss outdated prices, laws, and office-holders as if current — the fix is live data via RAG." ], "uid": "fkezkyl237r2" }, { "id": "concept-rw-llm-101-prompt-injection", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "risk", "security" ], "name": "Prompt injection", "clues": [ "A security attack that turns tool access or a document store into an exfiltration vector.", "Malicious instructions hidden inside a retrieved file that override the model's original instructions and hijack its actions.", "OWASP's LLM Top 10 flags it as LLM01 under excessive agency — think buried text telling the model to forward your prior messages to an attacker." ], "uid": "1fsh420ydqcvo" }, { "id": "concept-rw-llm-101-automation-bias", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "risk", "human-factors" ], "name": "Automation bias", "clues": [ "A human-side failure mode that makes copyright and other LLM risks worse.", "The tendency to over-trust and under-scrutinize confident-sounding model output.", "Named alongside evolving copyright law, it's why users reproduce protected content without realizing it." ], "uid": "v39s7hmhw8xv" }, { "id": "concept-rw-llm-101-non-determinism", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "risk", "deployment" ], "name": "Non-determinism", "clues": [ "A deployment property that breaks any workflow assuming reproducible behavior.", "Above temperature 0, outputs are sampled from a probability distribution, so the same prompt can return different answers across calls.", "The fix is to use temperature 0 for reproducibility and regression-test outputs whenever you upgrade model versions." ], "uid": "1lkbnjviyzwa5" }, { "id": "concept-rw-llm-101-token", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "core-term", "text-unit" ], "name": "Token", "clues": [ "The basic unit these models actually process.", "Roughly three-quarters of a word.", "It's the currency in which context limits and pricing are measured." ], "uid": "fgarfp1ym2x3f" }, { "id": "concept-rw-llm-101-inference", "shape": "concept", "conceptKind": "thing", "tags": [ "thing", "core-term", "usage" ], "name": "Inference", "clues": [ "What happens every single time you send a message to a model.", "Running the frozen model with no weights changing at all.", "It's the opposite of the pipeline stages that update parameters — the model just computes an answer." ], "uid": "1luaryk192j9y4" }, { "id": "ot-pre-transformer-1", "shape": "mcq", "tags": [ "lstm", "rnn" ], "prompt": { "modality": "text", "value": "LSTMs added gating mechanisms that regulated what to remember and what to forget. Which problem did this partially address?" }, "options": [ { "modality": "text", "value": "The vanishing gradient that caused RNNs to lose signal over long sequences", "short": "Vanishing gradient in RNNs" }, { "modality": "text", "value": "The inability to parallelize training across the tokens of a sequence", "short": "Can't parallelize training" }, { "modality": "text", "value": "The fixed-length vector bottleneck between a seq2seq encoder and decoder", "short": "Seq2seq vector bottleneck" }, { "modality": "text", "value": "The sparsity of n-gram counts for rare word sequences", "short": "N-gram count sparsity" } ], "correctIndex": 0, "explanation": "LSTM gating partially fixed vanishing gradients, letting signal survive longer sequences. The fixed-vector bottleneck was seq2seq's problem (solved by attention), and parallel training only arrived with the Transformer.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "ixivnz1w3fv35" }, { "id": "ot-pre-transformer-2", "shape": "mcq", "tags": [ "lstm", "limitations" ], "prompt": { "modality": "text", "value": "Which statement about LSTM-based NLP models (roughly 2014–2017) is NOT accurate?" }, "options": [ { "modality": "text", "value": "Training could be parallelized across every token in a sequence at once", "short": "Parallel training over tokens" }, { "modality": "text", "value": "They dominated NLP benchmarks for several years before the Transformer", "short": "Dominated NLP pre-Transformer" }, { "modality": "text", "value": "Gating mechanisms regulated what the model remembered and forgot", "short": "Gates control remember/forget" }, { "modality": "text", "value": "An internal hidden state carried information forward through the sequence", "short": "Hidden state carries info" } ], "correctIndex": 0, "explanation": "Sequential processing was a hard constraint: each token waited for the previous one, so training could not be parallelized across a sequence. The other three describe LSTMs accurately.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "m6v21a30for" }, { "id": "ot-pre-transformer-3", "shape": "mcq", "tags": [ "transformer", "attention" ], "prompt": { "modality": "text", "value": "What did the 2017 Transformer paper from Vaswani et al. prove about model architecture?" }, "options": [ { "modality": "text", "value": "Attention layers worked best when bolted onto an LSTM encoder–decoder", "short": "Attention bolted onto LSTM" }, { "modality": "text", "value": "Predicting masked words beat predicting the next word left-to-right", "short": "Masked words beat next-word" }, { "modality": "text", "value": "Recurrence could be discarded entirely and a model built from attention alone", "short": "Attention alone was enough" }, { "modality": "text", "value": "Learned embeddings could replace sequence modeling altogether", "short": "Embeddings replace sequences" } ], "correctIndex": 2, "explanation": "The Transformer paper showed recurrence was unnecessary — attention alone sufficed. Attention-on-LSTM was the earlier Bahdanau step; embeddings were word2vec; masked prediction was BERT in 2018.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1w1qzvdtubgc7" }, { "id": "ot-pre-transformer-4", "shape": "mcq", "tags": [ "timeline", "chronology" ], "prompt": { "modality": "text", "value": "Which sequence puts these pre-Transformer milestones in correct chronological order?" }, "options": [ { "modality": "text", "value": "The Transformer paper, then word2vec, then seq2seq with attention", "short": "Transformer → word2vec → seq2seq" }, { "modality": "text", "value": "seq2seq with attention, then word2vec, then the Transformer paper", "short": "seq2seq → word2vec → Transformer" }, { "modality": "text", "value": "word2vec, then seq2seq with attention, then the Transformer paper", "short": "word2vec → seq2seq → Transformer" }, { "modality": "text", "value": "word2vec, then the Transformer paper, then seq2seq with attention", "short": "word2vec → Transformer → seq2seq" } ], "correctIndex": 2, "explanation": "word2vec landed in 2013, seq2seq and Bahdanau's attention in 2014–2015, and the Transformer paper in 2017. Attention was the decisive conceptual move that made the Transformer possible.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1ixsg9e1jhb7qm" }, { "id": "ot-scaling-1", "shape": "mcq", "tags": [ "scaling-laws", "kaplan" ], "prompt": { "modality": "text", "value": "What did Kaplan et al.'s 2020 scaling laws describe?" }, "options": [ { "modality": "text", "value": "Loss improves only in discrete jumps once a model crosses size thresholds", "short": "Loss improves in discrete jumps" }, { "modality": "text", "value": "Compute-optimal training needs roughly 20 tokens of data per parameter", "short": "~20 tokens per parameter" }, { "modality": "text", "value": "Loss stops falling with added compute beyond about 10 billion parameters", "short": "Loss plateaus past 10B params" }, { "modality": "text", "value": "Loss falls predictably as a power function of compute, data, and parameters", "short": "Loss: power law of compute/data" } ], "correctIndex": 3, "explanation": "Kaplan's laws gave a quantitative power-law map of loss versus compute, data, and parameters, justifying ever-larger models. The 20-tokens-per-parameter rule is Chinchilla (2022); the other two contradict the smooth power-law finding.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "v3y4cgnj3ll8" }, { "id": "ot-scaling-2", "shape": "mcq", "tags": [ "scaling-laws", "chinchilla" ], "prompt": { "modality": "text", "value": "What did the Chinchilla (2022) result revise about GPT-3?" }, "options": [ { "modality": "text", "value": "GPT-3 had far too many training tokens for its size", "short": "GPT-3 had too many tokens" }, { "modality": "text", "value": "GPT-3 was significantly undertrained for its parameter count", "short": "GPT-3 was undertrained" }, { "modality": "text", "value": "GPT-3's architecture, not its data, capped its performance", "short": "Architecture capped GPT-3" }, { "modality": "text", "value": "GPT-3's loss did not follow a power law at all", "short": "GPT-3 loss: no power law" } ], "correctIndex": 1, "explanation": "Chinchilla showed GPT-3 was undertrained given its 175B parameters — an optimally data-scaled model of similar compute cost could match it at roughly one-seventh the size. Chinchilla revised the power laws rather than rejecting them.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1dkw6tjpcc1b5" }, { "id": "ot-scaling-3", "shape": "mcq", "tags": [ "chinchilla", "tokens-per-parameter" ], "prompt": { "modality": "text", "value": "By the Chinchilla scaling laws, roughly how many tokens of training data does compute-optimal training call for per model parameter?" }, "options": [ { "modality": "text", "value": "About 2" }, { "modality": "text", "value": "About 100" }, { "modality": "text", "value": "About 20" }, { "modality": "text", "value": "About 300" } ], "correctIndex": 2, "explanation": "Chinchilla put compute-optimal training at roughly 20 tokens per parameter. GPT-3 fell far short: 175B parameters trained on only about 300B tokens, which is under 2 tokens per parameter.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "7adxug4gg01o" }, { "id": "ot-scaling-4", "shape": "mcq", "tags": [ "scaling-laws", "compare" ], "prompt": { "modality": "text", "value": "Which statement about the Kaplan (2020) and Chinchilla (2022) scaling laws is NOT accurate?" }, "options": [ { "modality": "text", "value": "Chinchilla came out of DeepMind", "short": "Chinchilla came from DeepMind" }, { "modality": "text", "value": "Kaplan's laws helped justify the strategy of building ever-larger models", "short": "Kaplan justified bigger models" }, { "modality": "text", "value": "Chinchilla concluded GPT-3 had too few parameters for its training data", "short": "GPT-3 had too few parameters" }, { "modality": "text", "value": "Chinchilla implied a model near one-seventh of GPT-3's size could match it", "short": "1/7 GPT-3 size could match it" } ], "correctIndex": 2, "explanation": "Chinchilla found the opposite: GPT-3 had too little data for its parameter count, not too few parameters. The other three statements are accurate.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "oni9txvab8lb" }, { "id": "ot-scaling-5", "shape": "mcq", "tags": [ "chinchilla", "apply" ], "prompt": { "modality": "text", "value": "A lab has a fixed compute budget for one training run. What does the Chinchilla finding recommend?" }, "options": [ { "modality": "text", "value": "Add parameters and cut the token count to fit the budget" }, { "modality": "text", "value": "Split the budget across many tiny models and ensemble them" }, { "modality": "text", "value": "Train the largest model the budget allows on the same data" }, { "modality": "text", "value": "Train a smaller model on substantially more data" } ], "correctIndex": 3, "explanation": "Chinchilla's compute-optimal point trades parameters for tokens: a smaller, more thoroughly trained model wins at equal compute. Maximizing parameter count was the Kaplan-era instinct that Chinchilla corrected.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "9mpbb6rqi9pm" }, { "id": "ot-bert-gpt-1", "shape": "mcq", "tags": [ "bert", "masked-lm" ], "prompt": { "modality": "text", "value": "Which statement about BERT (2018) is NOT accurate?" }, "options": [ { "modality": "text", "value": "It predicted randomly masked words using context from both sides at once", "short": "Masked words, both-side context" }, { "modality": "text", "value": "Google released it in October 2018", "short": "Google released it Oct 2018" }, { "modality": "text", "value": "It was pretrained to predict each next token using only left-side context", "short": "Next-token, left context only" }, { "modality": "text", "value": "It set new state-of-the-art results on eleven NLP benchmarks", "short": "SOTA on 11 NLP benchmarks" } ], "correctIndex": 2, "explanation": "Left-to-right next-token prediction was GPT's objective; BERT's whole departure was masked language modeling, which forced bidirectional representations. The release date and the eleven-benchmark result are accurate.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "x30vx41wspehc" }, { "id": "ot-bert-gpt-2", "shape": "mcq", "tags": [ "gpt-1", "parameters" ], "prompt": { "modality": "text", "value": "What were GPT-1's parameter count and training corpus?" }, "options": [ { "modality": "text", "value": "175 billion parameters, trained on ~300 billion tokens", "short": "175B params, ~300B tokens" }, { "modality": "text", "value": "340 million parameters, trained on Wikipedia", "short": "340M params, Wikipedia" }, { "modality": "text", "value": "117 million parameters, trained on BooksCorpus", "short": "117M params, BooksCorpus" }, { "modality": "text", "value": "1.5 billion parameters, trained on WebText", "short": "1.5B params, WebText" } ], "correctIndex": 2, "explanation": "GPT-1 had 117M parameters and trained on BooksCorpus. 1.5B on WebText is GPT-2; 175B on ~300B tokens is GPT-3. GPT-1's contribution was the paradigm — pretrain once, adapt cheaply — more than its scale.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "jafnwm1woklia" }, { "id": "ot-bert-gpt-3", "shape": "mcq", "tags": [ "gpt-3", "in-context-learning" ], "prompt": { "modality": "text", "value": "What was GPT-3's defining discovery?" }, "options": [ { "modality": "text", "value": "Human preference rankings could be distilled into a reward model", "short": "Preferences → reward model" }, { "modality": "text", "value": "Multimodality — the model accepted both image and text input", "short": "Multimodality: image + text" }, { "modality": "text", "value": "In-context learning — a few prompt examples sufficed, with no gradient updates", "short": "In-context learning, no updates" }, { "modality": "text", "value": "Masked language modeling produced bidirectional representations", "short": "Masked LM, bidirectional" } ], "correctIndex": 2, "explanation": "GPT-3 showed in-context (few-shot) learning: examples in the prompt alone let it handle tasks it was never trained on. Multimodality arrived with GPT-4, masked LM is BERT, and reward models belong to RLHF.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1coo2q31qjd5d9" }, { "id": "ot-bert-gpt-4", "shape": "mcq", "tags": [ "gpt-progression", "rank" ], "prompt": { "modality": "text", "value": "Which option lists GPT-1, GPT-2, and GPT-3 with their correct parameter counts in order?" }, "options": [ { "modality": "text", "value": "117 million, then 175 billion, then 1.5 billion", "short": "117M, 175B, 1.5B" }, { "modality": "text", "value": "117 million, then 1.5 billion, then 175 billion", "short": "117M, 1.5B, 175B" }, { "modality": "text", "value": "117 million, then 1.5 billion, then 17.5 billion", "short": "117M, 1.5B, 17.5B" }, { "modality": "text", "value": "1.5 billion, then 117 million, then 175 billion", "short": "1.5B, 117M, 175B" } ], "correctIndex": 1, "explanation": "The progression ran 117M (GPT-1, 2018) to 1.5B (GPT-2, 2019) to 175B (GPT-3, 2020) — GPT-3 was more than 100x larger than GPT-2 while keeping the same pretraining objective.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1uupnw111zzykn" }, { "id": "ot-align-1", "shape": "mcq", "tags": [ "rlhf", "instructgpt" ], "prompt": { "modality": "text", "value": "Which sequence correctly describes the RLHF pipeline used for InstructGPT?" }, "options": [ { "modality": "text", "value": "Human raters rank outputs, the rankings train a reward model, the policy is fine-tuned against it", "short": "Rank→reward model→fine-tune" }, { "modality": "text", "value": "A reward model is trained first, then raters rank its outputs, then the model is pretrained", "short": "Reward model→rank→pretrain" }, { "modality": "text", "value": "The policy is fine-tuned on raw web text, then raters rank the pretraining data", "short": "Fine-tune web→rank data" }, { "modality": "text", "value": "Raters write ideal answers, the model memorizes them, no reward model is used", "short": "Write answers→memorize, no RM" } ], "correctIndex": 0, "explanation": "RLHF runs in that order: rank outputs, train a reward model on the rankings, fine-tune the policy against the reward model. Pretraining comes before RLHF, not after, and the reward model is a learned output of the rankings.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "z4tupk19lc8ho" }, { "id": "ot-align-2", "shape": "mcq", "tags": [ "instructgpt", "alignment" ], "prompt": { "modality": "text", "value": "What was the striking finding when InstructGPT was compared against raw GPT-3?" }, "options": [ { "modality": "text", "value": "A 1.3B InstructGPT was preferred by raters over the 175B GPT-3", "short": "1.3B preferred over 175B GPT-3" }, { "modality": "text", "value": "InstructGPT won only after being scaled past 175B parameters", "short": "Won only past 175B params" }, { "modality": "text", "value": "Raters preferred GPT-3 for helpfulness but InstructGPT for safety", "short": "GPT-3 helpful, InstructGPT safe" }, { "modality": "text", "value": "A 13B InstructGPT tied with the 175B GPT-3 on rater preference", "short": "13B tied with 175B GPT-3" } ], "correctIndex": 0, "explanation": "Raters preferred the 1.3B InstructGPT over the untuned 175B GPT-3 — alignment training, not raw scale, determined practical usefulness. The 13B figure belongs to LLaMA's comparison with GPT-3, not InstructGPT's.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1yyspic1uik56g" }, { "id": "ot-align-3", "shape": "mcq", "tags": [ "chatgpt", "launch-date" ], "prompt": { "modality": "text", "value": "On what date did OpenAI release ChatGPT as a free public research preview?" }, "options": [ { "modality": "text", "value": "November 30, 2022" }, { "modality": "text", "value": "February 24, 2023" }, { "modality": "text", "value": "December 6, 2023" }, { "modality": "text", "value": "March 14, 2023" } ], "correctIndex": 0, "explanation": "ChatGPT launched November 30, 2022 on a GPT-3.5-class RLHF model. March 14, 2023 is GPT-4's launch; February 2023 is LLaMA; December 2023 is Gemini.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1gvroyeg8juhi" }, { "id": "ot-align-4", "shape": "mcq", "tags": [ "chatgpt", "adoption" ], "prompt": { "modality": "text", "value": "How fast did ChatGPT reach its early adoption milestones?" }, "options": [ { "modality": "text", "value": "1 million users in five weeks and about 10 million monthly users within two months", "short": "1M in 5 wks, 10M/mo in 2 mo" }, { "modality": "text", "value": "100 million users in five days and 1 billion monthly users within two months", "short": "100M in 5 days, 1B/mo in 2 mo" }, { "modality": "text", "value": "1 million users in five days and about 100 million monthly users within two months", "short": "1M in 5 days, 100M/mo in 2 mo" }, { "modality": "text", "value": "1 million users in two months and about 100 million monthly users within a year", "short": "1M in 2 mo, 100M/mo in 1 yr" } ], "correctIndex": 2, "explanation": "ChatGPT hit 1 million users in five days and roughly 100 million monthly active users within two months — the fastest consumer-product adoption curve recorded to that point.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "14xywh8w5v4ck" }, { "id": "ot-align-5", "shape": "mcq", "tags": [ "gpt-4", "launch" ], "prompt": { "modality": "text", "value": "Which statement about the GPT-4 launch is NOT accurate?" }, "options": [ { "modality": "text", "value": "OpenAI published GPT-4's parameter count alongside the release", "short": "Published GPT-4 param count" }, { "modality": "text", "value": "GPT-4 was multimodal, accepting image and text input", "short": "Multimodal: image + text input" }, { "modality": "text", "value": "GPT-4 scored above human average on bar exams and medical licensing tests", "short": "Above human avg on bar/medical" }, { "modality": "text", "value": "GPT-4 launched with an 8,000-token context window, with a 32K variant following", "short": "8K context, 32K variant later" } ], "correctIndex": 0, "explanation": "OpenAI gave no public parameter count for GPT-4, beginning an era of less transparent frontier releases. The multimodality, 8K/32K context, and professional-exam results are all accurate.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1ebbl5f17zd0xd" }, { "id": "ot-open-gemini", "shape": "mcq", "tags": [ "gemini", "frontier" ], "prompt": { "modality": "text", "value": "What was Gemini, which Google released in December 2023?" }, "options": [ { "modality": "text", "value": "A chat preview aligned with Constitutional AI and RLAIF", "short": "Chat preview, Constitutional AI" }, { "modality": "text", "value": "A Mixture-of-Experts model that matched GPT-3.5 at low inference cost", "short": "MoE matching GPT-3.5, low cost" }, { "modality": "text", "value": "Its multimodal flagship, developed by Google DeepMind and replacing Bard", "short": "DeepMind flagship, replaced Bard" }, { "modality": "text", "value": "An open-weight family from 7B to 65B trained only on public data", "short": "Open-weight 7B–65B, public data" } ], "correctIndex": 2, "explanation": "Gemini was Google DeepMind's multimodal flagship, replacing Bard. The open 7B–65B family is Meta's LLaMA, the MoE model is Mixtral 8x7B, and Constitutional AI plus RLAIF describes Anthropic's Claude.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1czz9ljjbbxxp" }, { "id": "ot-open-2", "shape": "mcq", "tags": [ "mistral", "license" ], "prompt": { "modality": "text", "value": "Under which license did Mistral AI release Mistral 7B in September 2023, helping it become the most-downloaded open model within weeks?" }, "options": [ { "modality": "text", "value": "A research-only community license", "short": "Research-only community license" }, { "modality": "text", "value": "Creative Commons BY-NC 4.0", "short": "Creative Commons BY-NC 4.0" }, { "modality": "text", "value": "Apache 2.0", "short": "Apache 2.0" }, { "modality": "text", "value": "GPL-3.0", "short": "GPL-3.0" } ], "correctIndex": 2, "explanation": "Mistral 7B shipped under the permissive Apache 2.0 license, which allowed unrestricted commercial use and helped drive its rapid adoption. A copyleft, non-commercial, or research-only license would have limited exactly that.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1j5nt2iyg7uvi" }, { "id": "ot-open-3", "shape": "mcq", "tags": [ "mixtral", "mixture-of-experts" ], "prompt": { "modality": "text", "value": "How does the Mixture-of-Experts design in Mixtral 8x7B keep inference cost below its nominal parameter count?" }, "options": [ { "modality": "text", "value": "Parameters are compressed to 4 bits before every forward pass", "short": "Params compressed to 4 bits" }, { "modality": "text", "value": "Only a subset of the model's parameters activate for each token", "short": "Only some params fire per token" }, { "modality": "text", "value": "The model discards its earliest layers once a prompt exceeds 8K tokens", "short": "Drops early layers past 8K" }, { "modality": "text", "value": "Eight separate 7B models vote and the majority answer is returned", "short": "Eight 7B models vote, majority" } ], "correctIndex": 1, "explanation": "MoE routes each token to a subset of experts, so only some parameters activate per token and inference stays well below the nominal count. That is routing, not quantization, layer-dropping, or an ensemble vote.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "4h5iav10fe2l" }, { "id": "ot-open-4", "shape": "mcq", "tags": [ "claude", "rlaif" ], "prompt": { "modality": "text", "value": "Anthropic built Claude with Constitutional AI plus which technique, designed to reduce reliance on human labelers?" }, "options": [ { "modality": "text", "value": "RLHF — reinforcement learning from human feedback", "short": "RLHF (RL from human feedback)" }, { "modality": "text", "value": "Mixture-of-Experts routing", "short": "Mixture-of-Experts routing" }, { "modality": "text", "value": "RLAIF — reinforcement learning from AI feedback", "short": "RLAIF (RL from AI feedback)" }, { "modality": "text", "value": "Masked language modeling", "short": "Masked language modeling" } ], "correctIndex": 2, "explanation": "Claude pairs Constitutional AI with RLAIF, which draws feedback from AI rather than human labelers while encoding explicit values. RLHF is the human-labeler method RLAIF was meant to lean on less; the other two are unrelated architectures.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1f79t21rxnyhj" }, { "id": "ot-text-to-numbers-1", "shape": "mcq", "tags": [ "tokenization", "bpe" ], "prompt": { "modality": "text", "value": "Which algorithm is a common way to split raw text into tokens?" }, "options": [ { "modality": "text", "value": "Byte-Pair Encoding (BPE)", "short": "Byte-Pair Encoding (BPE)" }, { "modality": "text", "value": "Gradient descent", "short": "Gradient descent" }, { "modality": "text", "value": "Backpropagation", "short": "Backpropagation" }, { "modality": "text", "value": "Softmax normalization", "short": "Softmax normalization" } ], "correctIndex": 0, "explanation": "Tokenization splits raw text into tokens using algorithms such as Byte-Pair Encoding. Softmax converts raw next-token scores into probabilities, and gradient descent and backpropagation are training procedures — none of them cut text into pieces.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1f4mg9zhspop" }, { "id": "ot-text-to-numbers-2", "shape": "mcq", "tags": [ "tokenization", "vocabulary" ], "prompt": { "modality": "text", "value": "A tokenizer works against a fixed vocabulary. Roughly how large is that vocabulary typically?" }, "options": [ { "modality": "text", "value": "About 10 million tokens", "short": "About 10 million tokens" }, { "modality": "text", "value": "About 32,000 to 100,000 tokens", "short": "32,000–100,000 tokens" }, { "modality": "text", "value": "About 3,000 to 5,000 tokens", "short": "3,000–5,000 tokens" }, { "modality": "text", "value": "About 500,000 to 1,000,000 tokens", "short": "500,000–1,000,000 tokens" } ], "correctIndex": 1, "explanation": "The fixed vocabulary is roughly 32K–100K tokens. That size determines what the model 'sees' and how it handles rare or foreign words; a few thousand entries would be far too few, and hundreds of thousands or millions far too many.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1c7c2ks153dx1c" }, { "id": "ot-text-to-numbers-3", "shape": "mcq", "tags": [ "embeddings", "vectors" ], "prompt": { "modality": "text", "value": "What is an embedding, and what does it accomplish for the model?" }, "options": [ { "modality": "text", "value": "A compressed copy of the training text stored for later lookup", "short": "Compressed copy of training text" }, { "modality": "text", "value": "A limit on how many tokens the model can read at once", "short": "A limit on tokens read at once" }, { "modality": "text", "value": "A vector of numbers placing a token's meaning in a space where similar meanings sit close together", "short": "Vector placing meaning in space" }, { "modality": "text", "value": "A raw score assigned to each candidate next token", "short": "Raw score for each next token" } ], "correctIndex": 2, "explanation": "An embedding is a vector representing a token's meaning in high-dimensional space, turning discrete symbols into continuous math. The limit on tokens read at once is the context window; raw next-token scores are logits.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "9tc5jvuyeowp" }, { "id": "ot-text-to-numbers-4", "shape": "mcq", "tags": [ "embeddings", "vectors" ], "prompt": { "modality": "text", "value": "Two tokens' embedding vectors sit very close together in the embedding space. What does that closeness indicate?" }, "options": [ { "modality": "text", "value": "Their meanings are similar", "short": "Their meanings are similar" }, { "modality": "text", "value": "They are equally likely to be chosen as the next token", "short": "Equally likely as next token" }, { "modality": "text", "value": "They appear next to each other in the sentence being processed", "short": "Adjacent in the sentence" }, { "modality": "text", "value": "They are spelled with a similar number of characters", "short": "Similar spelling length" } ], "correctIndex": 0, "explanation": "An embedding places a token's meaning in a high-dimensional space where tokens with similar meanings sit close together. Closeness encodes meaning — not spelling, not position in the sentence, and not next-token probability.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1kicbvb1oviplh" }, { "id": "ot-model-internals-1", "shape": "mcq", "tags": [ "transformer", "architecture" ], "prompt": { "modality": "text", "value": "What distinguishes the Transformer from an RNN, and why does that matter?" }, "options": [ { "modality": "text", "value": "It processes all tokens in a sequence at once in parallel, making it fast enough to train at massive scale on GPUs", "short": "Processes all tokens in parallel" }, { "modality": "text", "value": "It stores training text verbatim, so it never needs to compute embeddings", "short": "Stores training text verbatim" }, { "modality": "text", "value": "It processes tokens strictly one at a time, which improves accuracy on long documents", "short": "Processes tokens one at a time" }, { "modality": "text", "value": "It removes the need for a fixed context window, letting it read unlimited text", "short": "Removes fixed context window" } ], "correctIndex": 0, "explanation": "The Transformer's parallel processing of the whole sequence — unlike an RNN's one-at-a-time approach — is what makes large-scale GPU training feasible. It still has a context window, and one-at-a-time is the RNN, not the Transformer.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1gfywff1bpcp5p" }, { "id": "ot-model-internals-2", "shape": "mcq", "tags": [ "self-attention", "transformer" ], "prompt": { "modality": "text", "value": "Self-attention lets a model connect a pronoun to the noun it refers to. How does the mechanism achieve this?" }, "options": [ { "modality": "text", "value": "A separate lookup table maps every pronoun to its most common noun", "short": "Lookup table: pronoun→noun" }, { "modality": "text", "value": "Each token is compared only with the token immediately before it", "short": "Compares only to prior token" }, { "modality": "text", "value": "Tokens are re-sorted so related words end up adjacent in the sequence", "short": "Re-sorts related words adjacent" }, { "modality": "text", "value": "Every token looks at every other token and learns how much to attend to each one", "short": "Each token attends to all others" } ], "correctIndex": 3, "explanation": "In self-attention each token attends to every other token and learns a weight for each, which is why distance in the sequence doesn't break the link. Comparing only with the previous token or reordering the sequence is not what happens.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1p9xoqj18xmkdl" }, { "id": "ot-model-internals-3", "shape": "mcq", "tags": [ "context-window", "limits" ], "prompt": { "modality": "text", "value": "A document is longer than the model's context window. What happens to the portion that doesn't fit?" }, "options": [ { "modality": "text", "value": "It is automatically converted into extra embeddings that bypass the limit", "short": "Converted to extra embeddings" }, { "modality": "text", "value": "It is processed first, then the in-window text is discarded", "short": "Processed first, rest discarded" }, { "modality": "text", "value": "It is invisible to the model — the window is the maximum it can see at once", "short": "Invisible; window is the max" }, { "modality": "text", "value": "It is stored in the parameters and recalled as needed", "short": "Stored in params, recalled" } ], "correctIndex": 2, "explanation": "The context window is the maximum number of tokens the model can see at once — its working memory — and anything outside it is simply invisible. Parameters encode learned knowledge from training, not text you paste in now.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1355668jxuvr4" }, { "id": "ot-model-internals-4", "shape": "mcq", "tags": [ "parameters", "weights" ], "prompt": { "modality": "text", "value": "In an LLM, what are the parameters (weights)?" }, "options": [ { "modality": "text", "value": "The billions of learned numbers inside the model that encode everything it 'knows'", "short": "Billions of learned numbers" }, { "modality": "text", "value": "The tokens currently held in the model's working memory", "short": "Tokens in working memory" }, { "modality": "text", "value": "The settings a user adjusts at request time, such as temperature", "short": "User settings like temperature" }, { "modality": "text", "value": "The vocabulary entries the tokenizer matches text against", "short": "Tokenizer vocabulary entries" } ], "correctIndex": 0, "explanation": "Parameters are the billions of numerical values learned during training that encode the model's knowledge, and their count is the main proxy for capacity. Temperature is a sampling knob; the working memory is the context window.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "sbn1tv9qvyr1" }, { "id": "ot-model-internals-5", "shape": "mcq", "tags": [ "internals", "review" ], "prompt": { "modality": "text", "value": "Which pairing of a model internal with its role is INCORRECT?" }, "options": [ { "modality": "text", "value": "Self-attention — every token weighs how much to attend to every other token", "short": "Self-attention — tokens attend" }, { "modality": "text", "value": "Parameters — the maximum number of tokens the model can see at once", "short": "Parameters — max tokens seen" }, { "modality": "text", "value": "Context window — the model's working memory for a single conversation", "short": "Context window — working memory" }, { "modality": "text", "value": "Transformer — an architecture that processes a sequence in parallel", "short": "Transformer — parallel arch" } ], "correctIndex": 1, "explanation": "The maximum number of tokens visible at once is the context window, not the parameters. Parameters are the learned weights that encode what the model knows. The other three pairings are correct.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "69ue6182j8tn" }, { "id": "ot-generation-loop-1", "shape": "mcq", "tags": [ "next-token", "training" ], "prompt": { "modality": "text", "value": "What is the core skill an LLM learns in training, from which its broad language ability follows?" }, "options": [ { "modality": "text", "value": "Retrieving the closest matching passage from a database of verified facts", "short": "Retrieve a verified-fact passage" }, { "modality": "text", "value": "Ranking whole candidate answers and returning the highest-scoring one", "short": "Rank answers, return the best" }, { "modality": "text", "value": "Next-token prediction — given all previous tokens, predict the most likely next one", "short": "Next-token prediction each step" }, { "modality": "text", "value": "Translating the prompt into a formal logical representation before answering", "short": "Translate prompt into logic" } ], "correctIndex": 2, "explanation": "The model's core learned objective is next-token prediction: given all previous tokens, predict the single most likely next one. Retrieval from a verified database is what grounding/RAG adds — it is not the base skill.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "o5ixcr19h26zp" }, { "id": "ot-generation-loop-2", "shape": "mcq", "tags": [ "autoregressive", "generation" ], "prompt": { "modality": "text", "value": "In autoregressive generation, what does the model condition each new token on?" }, "options": [ { "modality": "text", "value": "All the tokens before it, including the ones the model itself just produced", "short": "All prior tokens, incl. its own" }, { "modality": "text", "value": "Only the single token immediately preceding it", "short": "Only the preceding token" }, { "modality": "text", "value": "Only the user's original prompt, ignoring its own output so far", "short": "Only the original prompt" }, { "modality": "text", "value": "A draft of the full response written before any token is emitted", "short": "A pre-written full draft" } ], "correctIndex": 0, "explanation": "Autoregressive generation chooses each new token based on all prior tokens — the prompt plus the model's own just-generated output. It never drafts the full answer first, and it uses far more than just the previous token.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "meq3np1kbdch7" }, { "id": "ot-generation-loop-3", "shape": "mcq", "tags": [ "autoregressive", "error-compounding" ], "prompt": { "modality": "text", "value": "A model makes a small factual slip early in a long answer. Why can that slip grow into a bigger problem?" }, "options": [ { "modality": "text", "value": "Each mistake is written into the model's parameters and reused later", "short": "It's written into parameters" }, { "modality": "text", "value": "The mistake pushes the earlier correct text outside the context window", "short": "Pushes text out of context" }, { "modality": "text", "value": "The error raises the temperature setting for the rest of the response", "short": "It raises the temperature" }, { "modality": "text", "value": "It can't go back and edit — it commits to each token in sequence, so later tokens build on the mistake", "short": "Commits each token; builds on it" } ], "correctIndex": 3, "explanation": "Because generation is autoregressive and committed token by token, the model can't revise; subsequent tokens condition on the error, so it compounds. Parameters are fixed after training, and errors don't change temperature.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1vi317414ho98c" }, { "id": "ot-generation-loop-4", "shape": "mcq", "tags": [ "autoregressive", "generation" ], "prompt": { "modality": "text", "value": "A model is midway through a sentence and has produced three tokens. When is the fourth token decided?" }, "options": [ { "modality": "text", "value": "At training time, when the phrase was memorized from the data", "short": "At training time, memorized" }, { "modality": "text", "value": "At the same moment as the others, since the Transformer works in parallel", "short": "Same moment, in parallel" }, { "modality": "text", "value": "Only once the first three exist — it is chosen conditioned on them", "short": "Once the first three exist" }, { "modality": "text", "value": "Before generation started, when the model planned out the whole sentence", "short": "Before generation, all planned" } ], "correctIndex": 2, "explanation": "Generation is autoregressive: tokens come out one at a time, each conditioned on everything before it. The Transformer reads a sequence in parallel, but it still emits tokens sequentially and never plans the full sentence up front.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1xrn11l4st3tz" }, { "id": "ot-generation-loop-5", "shape": "mcq", "tags": [ "next-token", "autoregressive" ], "prompt": { "modality": "text", "value": "How do next-token prediction and autoregressive generation relate to each other?" }, "options": [ { "modality": "text", "value": "Next-token prediction runs only at training time; generation uses a different objective", "short": "NTP only at training, not gen" }, { "modality": "text", "value": "Autoregressive generation predicts the whole sequence at once, then next-token prediction checks it", "short": "AR gen predicts all; NTP checks" }, { "modality": "text", "value": "Autoregressive generation is next-token prediction applied repeatedly, feeding each new token back in", "short": "AR gen = NTP applied repeatedly" }, { "modality": "text", "value": "They are competing architectures; a model uses one or the other, never both", "short": "Rival methods; one or the other" } ], "correctIndex": 2, "explanation": "Generation is simply next-token prediction done repeatedly, each new token conditioned on everything before it. The same learned objective drives both training and generation — they aren't rival mechanisms.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "rfi785pkwy1v" }, { "id": "ot-sampling-1", "shape": "mcq", "tags": [ "logits", "sampling" ], "prompt": { "modality": "text", "value": "What are logits in an LLM?" }, "options": [ { "modality": "text", "value": "The normalized probabilities the model finally assigns to each candidate token", "short": "Normalized token probabilities" }, { "modality": "text", "value": "The vectors that encode each token's meaning in high-dimensional space", "short": "High-dim token meaning vectors" }, { "modality": "text", "value": "The learned weights stored inside the model's layers", "short": "Learned weights in the layers" }, { "modality": "text", "value": "The raw, unnormalized scores assigned to every possible next token", "short": "Raw, unnormalized token scores" } ], "correctIndex": 3, "explanation": "Logits are the raw, unnormalized scores for every candidate next token; they only become probabilities after normalization, so the finished probability values are not the logits. Meaning vectors are embeddings.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "6re4hz1ionfz9" }, { "id": "ot-sampling-2", "shape": "mcq", "tags": [ "logits", "softmax" ], "prompt": { "modality": "text", "value": "Which function converts a model's raw next-token scores into a probability distribution?" }, "options": [ { "modality": "text", "value": "Embedding lookup" }, { "modality": "text", "value": "Self-attention" }, { "modality": "text", "value": "Byte-Pair Encoding" }, { "modality": "text", "value": "Softmax" } ], "correctIndex": 3, "explanation": "Softmax normalizes the raw logits into a probability distribution over the vocabulary. BPE and tokenization split text into tokens; self-attention weighs tokens against one another inside the Transformer.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "16p2ric1j1667o" }, { "id": "ot-sampling-3", "shape": "mcq", "tags": [ "temperature", "sampling" ], "prompt": { "modality": "text", "value": "You need output that is as focused and deterministic as possible. How should you set the temperature?" }, "options": [ { "modality": "text", "value": "Low — it makes the model pick the safe, high-probability token", "short": "Low — picks safe high-prob token" }, { "modality": "text", "value": "High — it widens the search so the model finds the correct answer", "short": "High — widens the search" }, { "modality": "text", "value": "High — it sharpens the odds toward the single best token", "short": "High — sharpens to best token" }, { "modality": "text", "value": "It makes no difference; temperature only affects generation speed", "short": "No effect; only changes speed" } ], "correctIndex": 0, "explanation": "Low temperature makes the model favor the safe, high-probability token, giving focused, deterministic output. High temperature does the opposite — it flattens the odds and invites surprise.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "90kfu91qupr93" }, { "id": "ot-sampling-4", "shape": "mcq", "tags": [ "temperature", "logits" ], "prompt": { "modality": "text", "value": "Where in the generation process does the temperature setting take effect?" }, "options": [ { "modality": "text", "value": "It edits the embeddings of the prompt tokens before they enter the model", "short": "Edits prompt embeddings" }, { "modality": "text", "value": "It expands the context window so more candidates are considered", "short": "Expands the context window" }, { "modality": "text", "value": "It updates a subset of the model's parameters for the current request", "short": "Updates some parameters" }, { "modality": "text", "value": "It scales the logits before the next token is sampled", "short": "Scales logits before sampling" } ], "correctIndex": 3, "explanation": "Temperature is a sampling knob: it scales the logits — the raw scores — so the token draw is more or less random. It never touches the embeddings, the trained parameters, or the size of the context window.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1lzwp321rk4z12" }, { "id": "ot-sampling-5", "shape": "mcq", "tags": [ "temperature", "review" ], "prompt": { "modality": "text", "value": "Which statement about the temperature knob is NOT true?" }, "options": [ { "modality": "text", "value": "It is the main dial for controlling how predictable a model's responses are", "short": "Main dial for predictability" }, { "modality": "text", "value": "Setting it to 0 guarantees the answer will be factually correct", "short": "0 guarantees correct answers" }, { "modality": "text", "value": "It typically ranges from 0 to 2", "short": "Typically ranges 0 to 2" }, { "modality": "text", "value": "High values flatten the odds and invite surprising word choices", "short": "High flattens odds, surprises" } ], "correctIndex": 1, "explanation": "Temperature shapes how random the sampling is, not whether the content is true — at 0 the model just reliably picks its highest-scoring token, right or wrong. The other three statements describe it accurately.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1wgn0bm1h8iq2y" }, { "id": "ot-halluc-emerge-1", "shape": "mcq", "tags": [ "hallucination", "definition" ], "prompt": { "modality": "text", "value": "What is the defining description of a hallucination?" }, "options": [ { "modality": "text", "value": "Fluent, confident text that is factually wrong or entirely fabricated", "short": "Fluent but false/fabricated text" }, { "modality": "text", "value": "Text that exceeds the model's context window and gets truncated", "short": "Text truncated past context" }, { "modality": "text", "value": "Any answer the model refuses to give", "short": "Any answer it refuses to give" }, { "modality": "text", "value": "Output that is obviously garbled and easy for a user to spot", "short": "Obviously garbled output" } ], "correctIndex": 0, "explanation": "A hallucination is fluent, confident-sounding text that is simply wrong or fabricated. The danger is precisely that it is NOT obviously garbled — users often can't tell it from an accurate answer.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1gjr9et1c3x4k7" }, { "id": "ot-halluc-emerge-2", "shape": "mcq", "tags": [ "hallucination", "rag" ], "prompt": { "modality": "text", "value": "What is the main mitigation for hallucination?" }, "options": [ { "modality": "text", "value": "Reducing the model's parameter count", "short": "Reduce parameter count" }, { "modality": "text", "value": "Lowering the temperature to zero on every request", "short": "Set temperature to zero" }, { "modality": "text", "value": "Grounding the model with real sources — the job of RAG", "short": "Ground in real sources (RAG)" }, { "modality": "text", "value": "Enlarging the tokenizer's vocabulary", "short": "Enlarge tokenizer vocabulary" } ], "correctIndex": 2, "explanation": "Grounding the model in real sources, which is what RAG does, is the main mitigation. Temperature shapes randomness, not truth, and vocabulary or parameter count don't add a fact-checker.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "b1y5wpool7u7" }, { "id": "ot-halluc-emerge-3", "shape": "mcq", "tags": [ "emergence", "scale" ], "prompt": { "modality": "text", "value": "What makes an ability 'emergent'?" }, "options": [ { "modality": "text", "value": "It was explicitly programmed in by researchers before training", "short": "Explicitly programmed in" }, { "modality": "text", "value": "It appears in large models but is absent in smaller ones of the same type", "short": "Present only in large models" }, { "modality": "text", "value": "It fades away as a model is scaled up", "short": "Fades as the model scales up" }, { "modality": "text", "value": "It shows up only when temperature is set high", "short": "Shows up only at high temp" } ], "correctIndex": 1, "explanation": "Emergent abilities appear in large models but are absent in smaller ones — capability jumps researchers don't fully predict. They arise from scale, not from explicit programming or a sampling setting.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1uo263reovdr9" }, { "id": "ot-halluc-emerge-4", "shape": "mcq", "tags": [ "emergence", "examples" ], "prompt": { "modality": "text", "value": "Which pair of capabilities are examples of emergent abilities?" }, "options": [ { "modality": "text", "value": "Self-attention and parallel processing", "short": "Self-attention & parallelism" }, { "modality": "text", "value": "Tokenization and embedding lookup", "short": "Tokenization & embedding lookup" }, { "modality": "text", "value": "Softmax normalization and logit scaling", "short": "Softmax norm & logit scaling" }, { "modality": "text", "value": "Multi-step arithmetic and step-by-step (chain-of-thought) reasoning", "short": "Multi-step math & CoT reasoning" } ], "correctIndex": 3, "explanation": "Multi-step arithmetic and chain-of-thought reasoning are the named examples of abilities that appear only at scale. Tokenization, self-attention, and softmax are built-in mechanics present at any model size.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1pkzwix1g5xjmr" }, { "id": "ot-pretrain-1", "shape": "mcq", "tags": [ "pretraining", "self-supervised" ], "prompt": { "modality": "text", "value": "Pretraining is described as \"self-supervised.\" What does that mean in this context?" }, "options": [ { "modality": "text", "value": "Human raters supervise the model by ranking its outputs against each other", "short": "Raters rank the outputs" }, { "modality": "text", "value": "The model supervises a smaller student model and distills its knowledge into it", "short": "Model teaches a student model" }, { "modality": "text", "value": "Humans write ideal responses that the model is trained to copy", "short": "Humans write ideal answers" }, { "modality": "text", "value": "The training signal comes from the text itself — predicting the next token — so no human labels are needed", "short": "Signal from text; no labels" } ], "correctIndex": 3, "explanation": "Self-supervised means the corpus supplies its own labels: the next token is the answer. Ranking outputs is RLHF; writing ideal responses is SFT.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1tzaniv1jjszj9" }, { "id": "ot-pretrain-2", "shape": "mcq", "tags": [ "base-model", "instruction-following" ], "prompt": { "modality": "text", "value": "You hand a raw base model the question \"What causes tides?\" What is the most likely behavior described in the guide?" }, "options": [ { "modality": "text", "value": "It refuses, because base models are trained to decline factual questions", "short": "Refuses the factual question" }, { "modality": "text", "value": "It may continue with more questions, since all it knows how to do is extend text", "short": "Keeps generating more questions" }, { "modality": "text", "value": "It answers helpfully and concisely, then offers a follow-up", "short": "Answers, then offers follow-up" }, { "modality": "text", "value": "It returns an error, because base models cannot process prompts", "short": "Returns a processing error" } ], "correctIndex": 1, "explanation": "A base model only predicts continuations — a question can plausibly be followed by more questions. Helpful answering comes from SFT; refusal behavior comes from RLHF.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1lfa0e81lvyzx4" }, { "id": "ot-pretrain-3", "shape": "mcq", "tags": [ "base-model", "pipeline" ], "prompt": { "modality": "text", "value": "Which is NOT true of the base model produced by pretraining?" }, "options": [ { "modality": "text", "value": "It has soaked up broad world knowledge", "short": "Absorbed broad world knowledge" }, { "modality": "text", "value": "It is unsafe to hand to general users as a finished product", "short": "Unsafe as a finished product" }, { "modality": "text", "value": "It has been taught to follow instructions and be helpful", "short": "Taught to follow instructions" }, { "modality": "text", "value": "It has learned to predict text", "short": "Learned to predict text" } ], "correctIndex": 2, "explanation": "Pretraining yields broad world knowledge and text prediction, but explicitly does NOT teach instruction-following or helpfulness — that requires SFT and RLHF. Base models are raw material, not safe for general users.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "a7q4exk3ootb" }, { "id": "ot-pretrain-4", "shape": "mcq", "tags": [ "pretraining", "compute" ], "prompt": { "modality": "text", "value": "What is the pretraining corpus described as, and how much of the pipeline's compute does the stage consume?" }, "options": [ { "modality": "text", "value": "Trillions of tokens from the web, books, and code; the overwhelming majority of compute", "short": "Web/book/code; most compute" }, { "modality": "text", "value": "Thousands of curated prompt/response pairs; a small fraction of compute", "short": "Curated pairs; little compute" }, { "modality": "text", "value": "Billions of tokens of code only; the smallest share of compute", "short": "Code only; least compute" }, { "modality": "text", "value": "Millions of human preference comparisons; roughly half the compute", "short": "Preference pairs; half compute" } ], "correctIndex": 0, "explanation": "The guide specifies trillions of tokens from web, books, and code, and that pretraining is where the overwhelming majority of compute is spent. Curated pairs are SFT; preference comparisons are RLHF.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1yuq3ip1v38en" }, { "id": "ot-sft-1", "shape": "mcq", "tags": [ "sft", "training-data" ], "prompt": { "modality": "text", "value": "What data does supervised fine-tuning train on?" }, "options": [ { "modality": "text", "value": "A massive unlabeled corpus of web text scraped at random", "short": "Unlabeled web text corpus" }, { "modality": "text", "value": "Curated (prompt, ideal-response) pairs written or selected by humans", "short": "Curated prompt/response pairs" }, { "modality": "text", "value": "AI-generated critiques of the model's own responses", "short": "AI critiques of responses" }, { "modality": "text", "value": "Pairs of model outputs ranked against each other by human raters", "short": "Human-ranked output pairs" } ], "correctIndex": 1, "explanation": "SFT uses curated prompt/ideal-response pairs. Unlabeled web text is pretraining, ranked output pairs are RLHF, and AI self-critiques are Constitutional AI.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1btcxo81mcluxc" }, { "id": "ot-sft-2", "shape": "mcq", "tags": [ "sft", "instruction-following" ], "prompt": { "modality": "text", "value": "According to the guide, what capability does SFT add that pretraining alone does not?" }, "options": [ { "modality": "text", "value": "Running cheaply at inference time without changing weights", "short": "Running cheaply at inference" }, { "modality": "text", "value": "Following instructions rather than merely continuing text", "short": "Following user instructions" }, { "modality": "text", "value": "Absorbing broad world knowledge from a large corpus", "short": "Absorbing world knowledge" }, { "modality": "text", "value": "Predicting the next token in a sequence", "short": "Predicting the next token" } ], "correctIndex": 1, "explanation": "SFT teaches instruction-following — it transforms a text-predictor into something that answers questions. Next-token prediction and world knowledge already come from pretraining.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "si8j7r1ten3od" }, { "id": "ot-sft-3", "shape": "mcq", "tags": [ "sft", "pipeline" ], "prompt": { "modality": "text", "value": "Where does SFT sit in the training pipeline?" }, "options": [ { "modality": "text", "value": "Stage three — after RLHF has already aligned the model", "short": "Stage three: after RLHF" }, { "modality": "text", "value": "It runs in parallel with pretraining on the same corpus", "short": "Parallel with pretraining" }, { "modality": "text", "value": "Stage two — after pretraining produces the base model, before RLHF", "short": "Stage two: after pretraining" }, { "modality": "text", "value": "Stage one — it is the very first training the model receives", "short": "Stage one: first training" } ], "correctIndex": 2, "explanation": "The guide orders the pipeline pretraining (stage one) → SFT (stage two) → RLHF (stage three). SFT starts from the base model pretraining produced.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1vsrzs92uuvhz" }, { "id": "ot-sft-4", "shape": "mcq", "tags": [ "sft", "apply" ], "prompt": { "modality": "text", "value": "A team has a base model that rambles instead of answering. Which stage is designed to fix exactly that, and why?" }, "options": [ { "modality": "text", "value": "Inference tuning — adjusting the prompt at runtime rewrites the weights", "short": "Inference-time prompt tuning" }, { "modality": "text", "value": "Reward modeling alone — a scoring network by itself changes the model's behavior", "short": "Reward modeling by itself" }, { "modality": "text", "value": "Pretraining — more tokens will eventually teach it to answer directly", "short": "Pretraining: more tokens" }, { "modality": "text", "value": "SFT — training on human-written ideal responses teaches it to answer the prompt", "short": "SFT: human ideal responses" } ], "correctIndex": 3, "explanation": "SFT is the stage that turns a rambling text-predictor into something that follows instructions. More pretraining just improves prediction; a reward model only scores — the RL phase is what updates the LLM.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "qmabla1iuxr7i" }, { "id": "ot-sft-5", "shape": "mcq", "tags": [ "sft", "contrast" ], "prompt": { "modality": "text", "value": "Which is NOT part of the description of supervised fine-tuning?" }, "options": [ { "modality": "text", "value": "It teaches the model to follow instructions", "short": "Teaches instruction-following" }, { "modality": "text", "value": "Its training pairs are written or selected by humans", "short": "Human-written training pairs" }, { "modality": "text", "value": "Human raters compare pairs of outputs to express which they prefer", "short": "Raters compare output pairs" }, { "modality": "text", "value": "It starts from the base model", "short": "Starts from the base model" } ], "correctIndex": 2, "explanation": "Comparing pairs of outputs is the human-preference step of RLHF, not SFT. SFT trains on human-authored prompt/ideal-response pairs starting from the base model.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "smzxa01eyao1g" }, { "id": "ot-rlhf-1", "shape": "mcq", "tags": [ "rlhf", "sequence" ], "prompt": { "modality": "text", "value": "Put the RLHF procedure in the correct order." }, "options": [ { "modality": "text", "value": "A reward model is trained first → raters check its scores → the LLM is frozen", "short": "Reward model→raters→freeze LLM" }, { "modality": "text", "value": "Raters compare output pairs → preferences train a reward model → RL fine-tunes the LLM to maximize its score", "short": "Compare→reward model→RL-tune" }, { "modality": "text", "value": "Raters write ideal responses → the LLM copies them → a reward model scores the copies", "short": "Write ideal→LLM copies→score" }, { "modality": "text", "value": "The LLM is fine-tuned with RL → raters compare the results → a reward model is discarded", "short": "RL-tune LLM→compare→drop model" } ], "correctIndex": 1, "explanation": "RLHF runs: human pairwise comparisons → those preferences train a separate reward model → reinforcement learning fine-tunes the LLM to maximize that reward model's score.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "jt9xtb1rrezm5" }, { "id": "ot-rlhf-2", "shape": "mcq", "tags": [ "reward-model", "rlhf" ], "prompt": { "modality": "text", "value": "What is the reward model, and what limit does the guide place on it?" }, "options": [ { "modality": "text", "value": "A rule-based filter of banned words; it is capped by the size of the block list", "short": "Word filter; blocklist cap" }, { "modality": "text", "value": "A copy of the base model; it is capped by the training cutoff date", "short": "Base-model copy; cutoff cap" }, { "modality": "text", "value": "A separate network scoring response quality; its quality caps the quality of alignment", "short": "Separate scorer; alignment cap" }, { "modality": "text", "value": "The LLM itself, rescoring its own outputs; it is capped by the context window", "short": "LLM rescores self; context cap" } ], "correctIndex": 2, "explanation": "The reward model is a separate network trained on human preference data, and its quality effectively caps alignment quality — a flawed one just teaches the model to game it.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "5almi1e7g53b" }, { "id": "ot-rlhf-3", "shape": "mcq", "tags": [ "rlaif", "scaling" ], "prompt": { "modality": "text", "value": "RLAIF differs from RLHF in which single respect?" }, "options": [ { "modality": "text", "value": "An AI generates the preference labels instead of human raters", "short": "AI makes the preference labels" }, { "modality": "text", "value": "It skips the reward model and optimizes the LLM directly", "short": "Skips the reward model entirely" }, { "modality": "text", "value": "It replaces reinforcement learning with supervised fine-tuning", "short": "Swaps RL for supervised tuning" }, { "modality": "text", "value": "It runs before pretraining rather than after it", "short": "Runs before pretraining begins" } ], "correctIndex": 0, "explanation": "RLAIF is RLHF with AI-generated preference labels, which removes the human-labeling bottleneck and enables alignment at larger scale. The reward model and RL phase remain.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "jzy7nzdxxznt" }, { "id": "ot-rlhf-4", "shape": "mcq", "tags": [ "constitutional-ai", "alignment" ], "prompt": { "modality": "text", "value": "In Constitutional AI, how does the model produce the preferences it then trains on?" }, "options": [ { "modality": "text", "value": "It critiques and revises its own outputs against a written set of principles", "short": "Revises self against principles" }, { "modality": "text", "value": "It retrieves rulings from a legal database at inference time", "short": "Retrieves legal-DB rulings" }, { "modality": "text", "value": "It compares its outputs to a frozen copy of the base model", "short": "Compares to frozen base model" }, { "modality": "text", "value": "It polls a panel of human harm-labelers for every response", "short": "Polls human harm-labelers" } ], "correctIndex": 0, "explanation": "In Anthropic's Constitutional AI the model self-critiques and revises against a written constitution, then trains on those AI-generated preferences — reducing reliance on human harm-labeling.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "bvwmzsymeoc" }, { "id": "ot-rlhf-5", "shape": "mcq", "tags": [ "constitutional-ai", "auditability" ], "prompt": { "modality": "text", "value": "What payoff does the guide credit to Constitutional AI's written constitution?" }, "options": [ { "modality": "text", "value": "A guarantee that the model can never produce a harmful output", "short": "Guarantees no harmful output" }, { "modality": "text", "value": "Guiding principles that are explicit and auditable rather than buried in opaque preference data", "short": "Explicit, auditable principles" }, { "modality": "text", "value": "Elimination of the reward model from the alignment pipeline", "short": "Removes the reward model" }, { "modality": "text", "value": "Lower inference cost per API call for the finished model", "short": "Lower per-call inference cost" } ], "correctIndex": 1, "explanation": "The payoff is less reliance on human harm-labeling plus principles that are explicit and auditable, unlike opaque preference data. CAI is an alignment method — it does not change per-call inference cost.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1mwxm7lnosn0f" }, { "id": "ot-instruct-1", "shape": "mcq", "tags": [ "instruct-model", "pipeline" ], "prompt": { "modality": "text", "value": "Which combination produces an instruct/chat model?" }, "options": [ { "modality": "text", "value": "Base model + SFT + RLHF (or equivalent)", "short": "Base model + SFT + RLHF" }, { "modality": "text", "value": "Base model + pretraining on a second corpus", "short": "Base + second pretraining" }, { "modality": "text", "value": "A base model given a longer context window", "short": "Base + longer context window" }, { "modality": "text", "value": "SFT + a reward model, with no pretraining stage", "short": "SFT + reward, no pretraining" } ], "correctIndex": 0, "explanation": "The instruct/chat model is the product of the full pipeline: a pretrained base model, plus SFT, plus RLHF or an equivalent alignment step. A bigger context window changes nothing about alignment.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "13cguxn3z866p" }, { "id": "ot-instruct-2", "shape": "mcq", "tags": [ "instruct-model", "contrast" ], "prompt": { "modality": "text", "value": "Which behavior is characteristic of an instruct/chat model but NOT of the base model underneath it?" }, "options": [ { "modality": "text", "value": "Refusing harmful requests", "short": "Refusing harmful requests" }, { "modality": "text", "value": "Running with frozen weights at inference", "short": "Frozen weights at inference" }, { "modality": "text", "value": "Predicting likely next tokens", "short": "Predicting likely next tokens" }, { "modality": "text", "value": "Drawing on broad world knowledge from its corpus", "short": "Drawing on world knowledge" } ], "correctIndex": 0, "explanation": "Refusal behavior is installed by alignment (RLHF or equivalent). Token prediction, world knowledge, and frozen-weight inference are all true of the base model too.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "142fo9o1l4redo" }, { "id": "ot-instruct-3", "shape": "mcq", "tags": [ "instruct-model", "definition" ], "prompt": { "modality": "text", "value": "Which trio of behaviors does the guide use to characterize an instruct/chat model such as ChatGPT?" }, "options": [ { "modality": "text", "value": "Follows instructions, refuses harmful requests, converses naturally", "short": "Follows, refuses, converses" }, { "modality": "text", "value": "Scores responses, ranks output pairs, serves as an optimization target", "short": "Scores, ranks, optimizes" }, { "modality": "text", "value": "Predicts next tokens, extends text, continues prompts in unexpected ways", "short": "Predicts, extends, continues" }, { "modality": "text", "value": "Critiques its own outputs, revises them, trains on its own preferences", "short": "Critiques, revises, self-trains" } ], "correctIndex": 0, "explanation": "The guide defines the instruct/chat model by exactly those three behaviors. Extending text unexpectedly describes the base model; scoring and ranking describe the reward model; self-critique describes Constitutional AI.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "x57rnn10y5yi5" }, { "id": "ot-instruct-4", "shape": "mcq", "tags": [ "instruct-model", "deployment" ], "prompt": { "modality": "text", "value": "Which kind of model does the guide say nearly everyone actually interacts with?" }, "options": [ { "modality": "text", "value": "The reward model, since it decides what counts as a good answer", "short": "Reward model (scores answers)" }, { "modality": "text", "value": "The base model, since it holds all the world knowledge", "short": "Base model (world knowledge)" }, { "modality": "text", "value": "A base model steered by a well-crafted system prompt", "short": "Base model + system prompt" }, { "modality": "text", "value": "The instruct/chat model, which behaves very differently from the base model underneath", "short": "Instruct/chat model (tuned)" } ], "correctIndex": 3, "explanation": "The instruct/chat model is what nearly everyone interacts with, and it behaves very differently from the base model it is built on. Base models aren't safe to hand to general users.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "v87kp1fzi7kz" }, { "id": "ot-instruct-5", "shape": "mcq", "tags": [ "base-model", "apply" ], "prompt": { "modality": "text", "value": "A startup downloads a base model and ships it to consumers as a chatbot with no further training. What does the guide predict?" }, "options": [ { "modality": "text", "value": "It won't reliably follow instructions or refuse harm — base models are raw material, not a finished product", "short": "Won't follow/refuse reliably" }, { "modality": "text", "value": "It will be unable to generate any text at all without SFT", "short": "Can't generate text w/o SFT" }, { "modality": "text", "value": "It will keep learning from user chats and align itself over time", "short": "Self-aligns from user chats" }, { "modality": "text", "value": "It will behave identically to ChatGPT, since the world knowledge is already there", "short": "Acts identical to ChatGPT" } ], "correctIndex": 0, "explanation": "Base models are the raw material, not a finished product, and aren't safe for general users — they lack instruction-following and refusal. They also don't learn during chats; inference is read-only.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1s8h7ytubj3wf" }, { "id": "ot-infer-1", "shape": "mcq", "tags": [ "training", "inference" ], "prompt": { "modality": "text", "value": "Which pairing correctly matches each process to its description?" }, "options": [ { "modality": "text", "value": "Training runs the frozen model on a prompt; inference updates weights via gradient descent", "short": "Train=run frozen; infer=update" }, { "modality": "text", "value": "Training updates weights via gradient descent; inference runs the frozen model on a prompt", "short": "Train=update; infer=run frozen" }, { "modality": "text", "value": "Neither changes weights; they differ only in how much text they output", "short": "Neither updates; differ output" }, { "modality": "text", "value": "Both training and inference update weights, differing only in scale", "short": "Both update; differ in scale" } ], "correctIndex": 1, "explanation": "Training updates weights through gradient descent over billions of examples; inference runs the finished, frozen model. That asymmetry is the guide's single most clarifying distinction.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "133otq719dehh1" }, { "id": "ot-infer-2", "shape": "mcq", "tags": [ "cost", "training-vs-inference" ], "prompt": { "modality": "text", "value": "Why is fine-tuning expensive while querying an API is cheap?" }, "options": [ { "modality": "text", "value": "Fine-tuning runs billions of times a day, while queries run only once", "short": "FT billions/day; query once" }, { "modality": "text", "value": "Fine-tuning must re-download the training corpus on every request", "short": "Re-downloads corpus each time" }, { "modality": "text", "value": "API queries are subsidized, while fine-tuning is billed at true cost", "short": "API subsidized; FT true cost" }, { "modality": "text", "value": "Training runs on massive GPU clusters for weeks or months at a cost of millions; inference is cheap per call", "short": "GPU clusters, weeks, millions" } ], "correctIndex": 3, "explanation": "The cost asymmetry is structural: training is done once or rarely on huge clusters over weeks or months for millions of dollars, while inference is cheap per call — and it's inference that runs billions of times a day.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "xvtxov14hxgt9" }, { "id": "ot-infer-3", "shape": "mcq", "tags": [ "flops", "scale" ], "prompt": { "modality": "text", "value": "Roughly how many FLOPs does the guide say training a large LLM takes?" }, "options": [ { "modality": "text", "value": "On the order of 10^12 to 10^14", "short": "10^12 to 10^14" }, { "modality": "text", "value": "On the order of 10^6 to 10^9", "short": "10^6 to 10^9" }, { "modality": "text", "value": "On the order of 10^23 to 10^25", "short": "10^23 to 10^25" }, { "modality": "text", "value": "On the order of 10^40 to 10^45", "short": "10^40 to 10^45" } ], "correctIndex": 2, "explanation": "The guide puts large-model training at roughly 10^23 to 10^25 FLOPs — thousands of GPU-hours and millions of dollars of compute.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "15i4jmbd8zikd" }, { "id": "ot-infer-4", "shape": "mcq", "tags": [ "flops", "scaling-laws" ], "prompt": { "modality": "text", "value": "What does FLOPs stand for, and why do FLOPs budgets matter?" }, "options": [ { "modality": "text", "value": "Floating Point Operations — tied to scaling laws, they determine what an org can afford to build", "short": "Floating Point Operations" }, { "modality": "text", "value": "Full-Length Output Predictions — they cap how long a response can be", "short": "Full-Length Output Predictions" }, { "modality": "text", "value": "Frozen Layer Operations — they count the layers skipped during inference", "short": "Frozen Layer Operations" }, { "modality": "text", "value": "Fast Local Optimization Passes — they set how quickly a model answers a prompt", "short": "Fast Local Optimization Passes" } ], "correctIndex": 0, "explanation": "FLOPs are Floating Point Operations, a measure of computational work. Because they tie to scaling laws, FLOPs budgets determine what model sizes an organization can afford to train.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "5tdyg11hzi7bz" }, { "id": "ot-prompt-basics-1", "shape": "mcq", "tags": [ "prompt", "basics" ], "prompt": { "modality": "text", "value": "What does the input sent to an LLM typically consist of?" }, "options": [ { "modality": "text", "value": "A system message, zero or more prior turns, and the current user message", "short": "System msg + turns + user msg" }, { "modality": "text", "value": "The model's weight matrices plus its tokenizer vocabulary", "short": "Weight matrices + tokenizer" }, { "modality": "text", "value": "A set of document embeddings pulled from a vector database", "short": "Embeddings from vector DB" }, { "modality": "text", "value": "A batch of labeled training examples with gradient updates", "short": "Labeled training examples" } ], "correctIndex": 0, "explanation": "A prompt is the input to the model: system message (behavior instructions), any prior turns, and the current user message. Weights, embeddings, and training batches are not part of the prompt itself.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "l0k3y515y4vlj" }, { "id": "ot-prompt-basics-2", "shape": "mcq", "tags": [ "system-prompt" ], "prompt": { "modality": "text", "value": "Which part of the input sets the model's persona, constraints, and task context at the start of a conversation and is usually hidden from the end user?" }, "options": [ { "modality": "text", "value": "The system prompt", "short": "System prompt" }, { "modality": "text", "value": "The context window", "short": "Context window" }, { "modality": "text", "value": "The current user message", "short": "Current user message" }, { "modality": "text", "value": "The embedding layer", "short": "Embedding layer" } ], "correctIndex": 0, "explanation": "The system prompt is the special instruction block that configures persona, constraints, and task context. The user message is visible and per-turn; the context window is a size limit, not an instruction block.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1qotxj7597svd" }, { "id": "ot-prompt-basics-3", "shape": "mcq", "tags": [ "system-prompt", "limits" ], "prompt": { "modality": "text", "value": "Which statement about the system prompt is NOT true?" }, "options": [ { "modality": "text", "value": "It is where developers configure a model's behavior for their application", "short": "Devs configure app behavior" }, { "modality": "text", "value": "It is usually hidden from the end user", "short": "Usually hidden from user" }, { "modality": "text", "value": "It rewrites the model's weights so the persona persists across all future calls", "short": "Rewrites weights permanently" }, { "modality": "text", "value": "It sets the model's persona, constraints, and task context", "short": "Sets persona/constraints/task" } ], "correctIndex": 2, "explanation": "System prompts live in the context, not the weights — baking behavior into weights is fine-tuning. The other three are exactly how the guide describes the system prompt.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1x1prmxd9nz4f" }, { "id": "ot-prompt-basics-4", "shape": "mcq", "tags": [ "prompt", "prompt-engineering" ], "prompt": { "modality": "text", "value": "Why does the guide call prompt engineering a high-leverage skill?" }, "options": [ { "modality": "text", "value": "Prompts can supply knowledge the model was never trained on", "short": "Supply untrained knowledge" }, { "modality": "text", "value": "Prompts persist across calls the way fine-tuned behavior does", "short": "Persist across calls like FT" }, { "modality": "text", "value": "The prompt is the only way to steer a frozen model", "short": "Only way to steer frozen model" }, { "modality": "text", "value": "Prompts permanently update the model's weights", "short": "Permanently update weights" } ], "correctIndex": 2, "explanation": "The model's weights are frozen at inference, so the prompt is the sole steering lever. Prompting never updates weights, never persists past the context, and cannot add knowledge the model was never trained on.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "xcr4uc1qub074" }, { "id": "ot-shots-1", "shape": "mcq", "tags": [ "zero-shot" ], "prompt": { "modality": "text", "value": "Asking a model to perform a task with a plain instruction and no examples at all is called what?" }, "options": [ { "modality": "text", "value": "Chain-of-thought prompting", "short": "Chain-of-thought prompting" }, { "modality": "text", "value": "Fine-tuning", "short": "Fine-tuning" }, { "modality": "text", "value": "Zero-shot prompting", "short": "Zero-shot prompting" }, { "modality": "text", "value": "Few-shot prompting", "short": "Few-shot prompting" } ], "correctIndex": 2, "explanation": "Zero-shot prompting gives an instruction with no examples, e.g. 'Translate this to French.' Few-shot supplies worked examples, chain-of-thought asks for step-by-step reasoning, and fine-tuning changes weights instead of prompting.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "vvbrwr4igbq5" }, { "id": "ot-shots-2", "shape": "mcq", "tags": [ "few-shot" ], "prompt": { "modality": "text", "value": "A developer includes three worked examples in the prompt before asking the model to handle a new case. Which technique is this?" }, "options": [ { "modality": "text", "value": "Few-shot prompting", "short": "Few-shot prompting" }, { "modality": "text", "value": "Fine-tuning", "short": "Fine-tuning" }, { "modality": "text", "value": "Zero-shot prompting", "short": "Zero-shot prompting" }, { "modality": "text", "value": "Chain-of-thought prompting", "short": "Chain-of-thought prompting" } ], "correctIndex": 0, "explanation": "Few-shot prompting means one to several worked examples in the prompt — it teaches the task format inside the context window with no weight updates. Chain-of-thought asks for step-by-step reasoning; fine-tuning changes weights.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1ut12td9tp15j" }, { "id": "ot-shots-3", "shape": "mcq", "tags": [ "chain-of-thought", "reasoning" ], "prompt": { "modality": "text", "value": "Which prompting technique does the guide credit with sharply improving performance on arithmetic and multi-step reasoning?" }, "options": [ { "modality": "text", "value": "Zero-shot prompting", "short": "Zero-shot prompting" }, { "modality": "text", "value": "Chain-of-thought prompting", "short": "Chain-of-thought prompting" }, { "modality": "text", "value": "Function calling", "short": "Function calling" }, { "modality": "text", "value": "Chunking", "short": "Chunking" } ], "correctIndex": 1, "explanation": "Chain-of-thought prompting makes the model reason step by step before its final answer, which sharply helps arithmetic and multi-step reasoning. Zero-shot is a bare instruction; function calling and chunking are not prompting styles.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "ylv17u1a7sh0y" }, { "id": "ot-shots-4", "shape": "mcq", "tags": [ "in-context-learning" ], "prompt": { "modality": "text", "value": "Which statement about in-context learning is NOT true?" }, "options": [ { "modality": "text", "value": "It blurs the line between 'prompting' and 'training' in people's minds", "short": "Blurs prompting vs training" }, { "modality": "text", "value": "It lets a model adapt its behavior from examples or instructions in the prompt", "short": "Adapts from prompt examples" }, { "modality": "text", "value": "It requires no weight updates", "short": "Requires no weight updates" }, { "modality": "text", "value": "The learning it produces persists after the context window ends", "short": "Persists after context ends" } ], "correctIndex": 3, "explanation": "No learning persists once the context window ends — that is the key contrast with real training. The other three match the guide's definition of in-context learning.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "ooc75y1lth0re" }, { "id": "ot-shots-5", "shape": "mcq", "tags": [ "zero-shot", "model-size" ], "prompt": { "modality": "text", "value": "Zero-shot prompting tests a model's general capability. Which models does the guide say most often need examples added instead?" }, "options": [ { "modality": "text", "value": "Smaller models", "short": "Smaller models" }, { "modality": "text", "value": "The largest models", "short": "Largest models" }, { "modality": "text", "value": "Models with the longest context windows", "short": "Longest-context models" }, { "modality": "text", "value": "Fine-tuned models", "short": "Fine-tuned models" } ], "correctIndex": 0, "explanation": "The guide says large models handle zero-shot well, while smaller ones often need examples. Context length and fine-tuning are separate levers and are not what the guide ties to needing examples.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1lrxrf1zjkz7r" }, { "id": "ot-rag-1", "shape": "mcq", "tags": [ "rag" ], "prompt": { "modality": "text", "value": "Which pattern retrieves relevant documents from an external store and inserts them into the prompt as context before the model answers?" }, "options": [ { "modality": "text", "value": "Function calling", "short": "Function calling" }, { "modality": "text", "value": "Fine-tuning", "short": "Fine-tuning" }, { "modality": "text", "value": "Chain-of-thought prompting", "short": "Chain-of-thought prompting" }, { "modality": "text", "value": "RAG (Retrieval-Augmented Generation)", "short": "RAG (retrieval-augmented)" } ], "correctIndex": 3, "explanation": "RAG, introduced by Lewis et al. (2020), retrieves documents and inserts them as context. Fine-tuning changes weights instead, and function calling emits a tool call rather than retrieving documents.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1nyvjy9bx0pr7" }, { "id": "ot-rag-2", "shape": "mcq", "tags": [ "vector-database", "embeddings" ], "prompt": { "modality": "text", "value": "What does a vector database store, and how does it find the material RAG needs?" }, "options": [ { "modality": "text", "value": "Model weights, found by gradient lookup", "short": "Model weights, gradient lookup" }, { "modality": "text", "value": "Chat transcripts, found by exact keyword match only", "short": "Chat transcripts, keyword match" }, { "modality": "text", "value": "Tokenizer vocabularies, found by prefix matching", "short": "Tokenizer vocab, prefix match" }, { "modality": "text", "value": "Document embeddings, found by fast similarity search", "short": "Embeddings, similarity search" } ], "correctIndex": 3, "explanation": "A vector database stores document embeddings and runs fast similarity search to return the most semantically relevant chunks. Semantic similarity — not exact keyword matching — is the point.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "eg8rzh1h6gz6n" }, { "id": "ot-rag-3", "shape": "mcq", "tags": [ "chunking", "rag" ], "prompt": { "modality": "text", "value": "Why are source documents split into smaller segments before being embedded for RAG?" }, "options": [ { "modality": "text", "value": "So retrieved passages fit the context window and stay semantically focused", "short": "Fit context, stay focused" }, { "modality": "text", "value": "So the system prompt stays hidden from end users", "short": "Hide the system prompt" }, { "modality": "text", "value": "So the vector database can skip similarity search", "short": "Skip similarity search" }, { "modality": "text", "value": "So the model's weights update faster during retrieval", "short": "Weights update faster" } ], "correctIndex": 0, "explanation": "Chunking keeps retrieved text small enough for the context window and semantically focused. Retrieval never updates weights, and the vector database still relies on similarity search.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "rlpqkv1fb1tq5" }, { "id": "ot-rag-4", "shape": "mcq", "tags": [ "vector-database", "examples" ], "prompt": { "modality": "text", "value": "Which of these is NOT one of the vector databases named in the guide?" }, "options": [ { "modality": "text", "value": "LangChain" }, { "modality": "text", "value": "Chroma" }, { "modality": "text", "value": "Weaviate" }, { "modality": "text", "value": "Pinecone" } ], "correctIndex": 0, "explanation": "The guide names Pinecone, Weaviate, and Chroma as vector databases. LangChain is not among them — it is not the retrieval store itself.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "mktnjc8a9yh4" }, { "id": "ot-rag-5", "shape": "mcq", "tags": [ "chunking", "tradeoffs" ], "prompt": { "modality": "text", "value": "Chunk size is a key RAG knob. According to the guide, what goes wrong when chunks are too large?" }, "options": [ { "modality": "text", "value": "Their embeddings can no longer be searched by similarity", "short": "Their embeddings unsearchable" }, { "modality": "text", "value": "They overwrite the model's weights", "short": "They overwrite model weights" }, { "modality": "text", "value": "They waste context", "short": "They waste context" }, { "modality": "text", "value": "They lose coherence", "short": "They lose coherence" } ], "correctIndex": 2, "explanation": "Too large wastes context; too small loses coherence — so 'lose coherence' is the too-small failure, not the too-large one. Chunk size never breaks similarity search or touches the weights.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "15y27twywvbdw" }, { "id": "ot-tools-1", "shape": "mcq", "tags": [ "function-calling", "tool-use" ], "prompt": { "modality": "text", "value": "In function calling, what does the model itself actually produce?" }, "options": [ { "modality": "text", "value": "The API's response data, computed directly by the model", "short": "The API's response data" }, { "modality": "text", "value": "A new set of fine-tuned weights for the task", "short": "New fine-tuned weights" }, { "modality": "text", "value": "An embedding to be stored in a vector database", "short": "An embedding to store" }, { "modality": "text", "value": "A structured, well-formed call that the host application runs", "short": "A structured tool call" } ], "correctIndex": 3, "explanation": "The model emits a structured call; the host application executes it against the real API or tool. The model does not run the tool or compute the result itself.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "11zto376k8n3d" }, { "id": "ot-tools-2", "shape": "mcq", "tags": [ "function-calling", "tool-use" ], "prompt": { "modality": "text", "value": "Once the host application has executed a tool the model requested, what happens next?" }, "options": [ { "modality": "text", "value": "The result is chunked and embedded permanently", "short": "Result is chunked and embedded" }, { "modality": "text", "value": "The model's weights are updated with the result", "short": "Weights updated with the result" }, { "modality": "text", "value": "The result is fed back to the model", "short": "Result is fed back to the model" }, { "modality": "text", "value": "The interaction ends with the raw tool output as the answer", "short": "Interaction ends with raw output" } ], "correctIndex": 2, "explanation": "The host runs the call and feeds the result back so the model can use it — that round trip is what grounds the model in real-time data. No weight update or embedding step is involved.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "9oncsp1fl1fp3" }, { "id": "ot-tools-3", "shape": "mcq", "tags": [ "function-calling", "tool-use" ], "prompt": { "modality": "text", "value": "An assistant must answer with today's live exchange rate from a currency API. Which mechanism gives it that ability?" }, "options": [ { "modality": "text", "value": "Zero-shot prompting" }, { "modality": "text", "value": "Fine-tuning" }, { "modality": "text", "value": "Function calling" }, { "modality": "text", "value": "Chunking" } ], "correctIndex": 2, "explanation": "Function calling lets the model emit a structured call to an external API, grounding it in real-time data. Fine-tuning bakes in static behavior, chunking is a retrieval step, and zero-shot is just a bare instruction.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "v7c2wu78yp82" }, { "id": "ot-tools-4", "shape": "mcq", "tags": [ "agent", "planning-loop" ], "prompt": { "modality": "text", "value": "Which of these is NOT part of the agent loop the guide describes?" }, "options": [ { "modality": "text", "value": "Observing the results of an action", "short": "Observing action results" }, { "modality": "text", "value": "Retraining the model's weights after each observation", "short": "Retraining model weights" }, { "modality": "text", "value": "Selecting which tool to use", "short": "Selecting which tool to use" }, { "modality": "text", "value": "Planning toward a goal", "short": "Planning toward a goal" } ], "correctIndex": 1, "explanation": "The loop is plan, select tools, act, observe — repeated until the goal is met. Nothing in it retrains the weights; that would be fine-tuning.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "xnwnjquyeltq" }, { "id": "ot-ftrag-1", "shape": "mcq", "tags": [ "fine-tuning", "rag" ], "prompt": { "modality": "text", "value": "Fine-tuning and RAG differ in where the new knowledge or behavior lands. Which pairing is correct?" }, "options": [ { "modality": "text", "value": "Fine-tuning → the model's weights; RAG → the prompt context", "short": "Fine-tune→weights; RAG→context" }, { "modality": "text", "value": "Fine-tuning → the prompt context; RAG → the model's weights", "short": "Fine-tune→context; RAG→weights" }, { "modality": "text", "value": "Both → the model's weights", "short": "Both→weights" }, { "modality": "text", "value": "Both → the prompt context", "short": "Both→context" } ], "correctIndex": 0, "explanation": "Fine-tuning bakes behavior into the weights; RAG inserts retrieved documents into the context before the model answers. The two operate at different layers, which is the whole contrast.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1v7atp919s46nz" }, { "id": "ot-ftrag-2", "shape": "mcq", "tags": [ "fine-tuning", "cost" ], "prompt": { "modality": "text", "value": "How does the guide characterize fine-tuning's cost and staying power?" }, "options": [ { "modality": "text", "value": "Free, and permanent", "short": "Free, and permanent" }, { "modality": "text", "value": "Cheap, and temporary within one conversation", "short": "Cheap, and temporary" }, { "modality": "text", "value": "Free, but bounded by context size", "short": "Free, but context-bound" }, { "modality": "text", "value": "Expensive, but durable across calls", "short": "Expensive, but durable" } ], "correctIndex": 3, "explanation": "Fine-tuning is expensive but persists across calls because it changes the weights. 'Free but bounded by context' describes prompting, not fine-tuning.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1q7omq4145fyi8" }, { "id": "ot-ftrag-3", "shape": "mcq", "tags": [ "rag", "use-case" ], "prompt": { "modality": "text", "value": "A team needs a bot to answer questions over internal documents that are rewritten every week. Which approach fits best?" }, "options": [ { "modality": "text", "value": "Zero-shot prompting", "short": "Zero-shot prompting" }, { "modality": "text", "value": "RAG", "short": "RAG" }, { "modality": "text", "value": "Fine-tuning", "short": "Fine-tuning" }, { "modality": "text", "value": "Chain-of-thought prompting", "short": "Chain-of-thought prompting" } ], "correctIndex": 1, "explanation": "RAG is ideal for fresh or proprietary knowledge: retrieve the current documents and insert them as context. Fine-tuning would have to be redone as the documents change, and prompting alone adds no new knowledge.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "hs08duc7l5n6" }, { "id": "ot-ftrag-4", "shape": "mcq", "tags": [ "fine-tuning", "use-case" ], "prompt": { "modality": "text", "value": "A company wants a house style and domain behavior the model keeps on every call, without adding instructions or examples to each prompt. Which approach fits best?" }, "options": [ { "modality": "text", "value": "Function calling" }, { "modality": "text", "value": "RAG" }, { "modality": "text", "value": "Few-shot prompting" }, { "modality": "text", "value": "Fine-tuning" } ], "correctIndex": 3, "explanation": "Fine-tuning bakes persistent style and behavior into the weights, so it holds across calls. Few-shot would work but must be re-supplied in every prompt, and RAG adds knowledge rather than behavior.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "8yl085llv3mz" }, { "id": "ot-ftrag-5", "shape": "mcq", "tags": [ "rag", "fine-tuning" ], "prompt": { "modality": "text", "value": "Which statement about RAG is NOT true?" }, "options": [ { "modality": "text", "value": "It suits fresh or proprietary knowledge", "short": "Suits fresh/proprietary data" }, { "modality": "text", "value": "It inserts retrieved text into the prompt as context before the model answers", "short": "Adds retrieved text to prompt" }, { "modality": "text", "value": "It permanently changes the model's weights", "short": "Permanently changes weights" }, { "modality": "text", "value": "It retrieves documents from an external store", "short": "Retrieves from external store" } ], "correctIndex": 2, "explanation": "RAG never touches the weights — that is fine-tuning's job. RAG works entirely by retrieving external documents and placing them into the context.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "jacu08pdlrdk" }, { "id": "ot-frontier-1", "shape": "mcq", "tags": [ "xai", "model-families" ], "prompt": { "modality": "text", "value": "Which model family is made by xAI, the lab Elon Musk founded in 2023?" }, "options": [ { "modality": "text", "value": "Gemini" }, { "modality": "text", "value": "Llama" }, { "modality": "text", "value": "Claude" }, { "modality": "text", "value": "Grok" } ], "correctIndex": 3, "explanation": "xAI makes Grok, which is integrated with X. Claude is Anthropic's, Llama is Meta AI's, and Gemini is Google DeepMind's.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "erjj5l1zqr0f" }, { "id": "ot-frontier-2", "shape": "mcq", "tags": [ "anthropic", "model-families" ], "prompt": { "modality": "text", "value": "Which flagship model family does Anthropic make?" }, "options": [ { "modality": "text", "value": "Claude" }, { "modality": "text", "value": "Command" }, { "modality": "text", "value": "GPT" }, { "modality": "text", "value": "Grok" } ], "correctIndex": 0, "explanation": "Anthropic, founded in 2021 by Dario and Daniela Amodei, makes Claude and pioneered Constitutional AI. GPT is OpenAI's, Command is Cohere's, and Grok is xAI's.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "r7yymk5qsjwk" }, { "id": "ot-frontier-3", "shape": "mcq", "tags": [ "openai", "model-families" ], "prompt": { "modality": "text", "value": "Which of these models is NOT part of OpenAI's flagship family?" }, "options": [ { "modality": "text", "value": "GPT-3.5" }, { "modality": "text", "value": "GPT-4" }, { "modality": "text", "value": "GPT-4o" }, { "modality": "text", "value": "Llama 3" } ], "correctIndex": 3, "explanation": "Llama 3 is Meta AI's open-weight model. OpenAI's GPT family includes GPT-3.5, GPT-4, and GPT-4o.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1c1ttkw1iumlk4" }, { "id": "ot-frontier-4", "shape": "mcq", "tags": [ "meta-ai", "model-families" ], "prompt": { "modality": "text", "value": "Which set of models makes up Meta AI's flagship family?" }, "options": [ { "modality": "text", "value": "LLaMA, Llama 2, and Llama 3", "short": "LLaMA, Llama 2, Llama 3" }, { "modality": "text", "value": "Command, Embed, and Rerank", "short": "Command, Embed, Rerank" }, { "modality": "text", "value": "GPT-3.5, GPT-4, and GPT-4o", "short": "GPT-3.5, GPT-4, GPT-4o" }, { "modality": "text", "value": "Mistral 7B, Mixtral 8x7B, and Mistral Large", "short": "Mistral 7B, Mixtral 8x7B, Large" } ], "correctIndex": 0, "explanation": "Meta AI's flagship family is Llama: LLaMA, Llama 2, and Llama 3. The Mistral models are Mistral AI's, the GPT models are OpenAI's, and Command/Embed/Rerank are Cohere's.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "gutsp11wgndvb" }, { "id": "ot-frontier-5", "shape": "mcq", "tags": [ "openai", "model-families" ], "prompt": { "modality": "text", "value": "Which lab-to-flagship-family pairing is correct?" }, "options": [ { "modality": "text", "value": "Anthropic — Grok" }, { "modality": "text", "value": "Meta AI — Claude" }, { "modality": "text", "value": "OpenAI — GPT" }, { "modality": "text", "value": "xAI — Llama" } ], "correctIndex": 2, "explanation": "OpenAI makes the GPT family and created ChatGPT. The other pairings each attach a family to the wrong lab: Claude is Anthropic's, Grok is xAI's, and Llama is Meta AI's.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "h09xp2x7hxxu" }, { "id": "ot-intl-1", "shape": "mcq", "tags": [ "google-deepmind", "gemini" ], "prompt": { "modality": "text", "value": "Google DeepMind's Gemini family succeeded which two earlier Google efforts?" }, "options": [ { "modality": "text", "value": "Command and Rerank" }, { "modality": "text", "value": "Bard and PaLM" }, { "modality": "text", "value": "AlphaGo and AlphaFold" }, { "modality": "text", "value": "Llama and Mixtral" } ], "correctIndex": 1, "explanation": "Gemini is the successor to Bard and PaLM. AlphaGo and AlphaFold are DeepMind projects but not language-model predecessors of Gemini.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "19sr2hb103i669" }, { "id": "ot-intl-2", "shape": "mcq", "tags": [ "mistral", "open-weights" ], "prompt": { "modality": "text", "value": "Which lab is Paris-based, founded in 2023, and known for Apache-licensed open-weight models and European AI sovereignty?" }, "options": [ { "modality": "text", "value": "xAI" }, { "modality": "text", "value": "Cohere" }, { "modality": "text", "value": "Mistral AI" }, { "modality": "text", "value": "Hugging Face" } ], "correctIndex": 2, "explanation": "Mistral AI is the Paris-based 2023 startup known for efficient Apache-2.0 open-weight models. Cohere is enterprise-focused, xAI makes Grok, and Hugging Face is a hub, not a frontier lab.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "cs1behwwsfcb" }, { "id": "ot-intl-3", "shape": "mcq", "tags": [ "cohere", "enterprise" ], "prompt": { "modality": "text", "value": "Command, Embed, and Rerank are the flagship products of which enterprise-focused lab known for RAG tooling?" }, "options": [ { "modality": "text", "value": "Anthropic" }, { "modality": "text", "value": "Cohere" }, { "modality": "text", "value": "Google DeepMind" }, { "modality": "text", "value": "Mistral AI" } ], "correctIndex": 1, "explanation": "Cohere's products are Command, Embed, and Rerank, with deep RAG and retrieval tooling. Mistral's models are Mistral 7B, Mixtral 8x7B, and Mistral Large.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "8mgihbg4n63p" }, { "id": "ot-intl-4", "shape": "mcq", "tags": [ "mistral", "model-families" ], "prompt": { "modality": "text", "value": "Which of these is NOT a Mistral AI model?" }, "options": [ { "modality": "text", "value": "Mistral 7B" }, { "modality": "text", "value": "Mixtral 8x7B" }, { "modality": "text", "value": "Command" }, { "modality": "text", "value": "Mistral Large" } ], "correctIndex": 2, "explanation": "Command is Cohere's flagship generation product. Mistral 7B, Mixtral 8x7B, and Mistral Large are all Mistral AI models.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "ia68oru6ooeh" }, { "id": "ot-god-1", "shape": "mcq", "tags": [ "lecun", "godfathers" ], "prompt": { "modality": "text", "value": "Which godfather of deep learning is Meta's Chief AI Scientist, invented convolutional networks, and is skeptical of current LLM reasoning?" }, "options": [ { "modality": "text", "value": "Yoshua Bengio" }, { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Demis Hassabis" }, { "modality": "text", "value": "Yann LeCun" } ], "correctIndex": 3, "explanation": "Yann LeCun is Meta's Chief AI Scientist and the inventor of convolutional networks. Hassabis leads Google DeepMind and is not one of the three godfathers.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "2fdnv71z0vsl" }, { "id": "ot-god-2", "shape": "mcq", "tags": [ "hinton", "godfathers" ], "prompt": { "modality": "text", "value": "Which godfather of deep learning pioneered backpropagation and left Google in 2023 to speak freely about AI risk?" }, "options": [ { "modality": "text", "value": "Ilya Sutskever" }, { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Yoshua Bengio" } ], "correctIndex": 1, "explanation": "Geoffrey Hinton pioneered backpropagation and left Google in 2023 to speak freely about AI risk. Sutskever was Hinton's student but is not one of the three godfathers.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "jn1f6kety48s" }, { "id": "ot-god-3", "shape": "mcq", "tags": [ "bengio", "godfathers" ], "prompt": { "modality": "text", "value": "Which godfather of deep learning is a professor at Universite de Montreal and heads Mila?" }, "options": [ { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Andrej Karpathy" }, { "modality": "text", "value": "Yoshua Bengio" }, { "modality": "text", "value": "Yann LeCun" } ], "correctIndex": 2, "explanation": "Yoshua Bengio is a professor at Universite de Montreal, head of Mila, and a leading safety advocate. Hinton was at Google until 2023; LeCun is at Meta.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "153ykrc1deuqps" }, { "id": "ot-god-4", "shape": "mcq", "tags": [ "godfathers", "turing-award" ], "prompt": { "modality": "text", "value": "Which researcher is NOT one of the three godfathers of deep learning?" }, "options": [ { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Yoshua Bengio" }, { "modality": "text", "value": "Ilya Sutskever" }, { "modality": "text", "value": "Geoffrey Hinton" } ], "correctIndex": 2, "explanation": "The three godfathers are Hinton, Bengio, and LeCun, joint 2018 Turing Award recipients. Sutskever was Hinton's PhD student and an OpenAI co-founder, but not part of that trio.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "c95t1y1gg6276" }, { "id": "ot-res-1", "shape": "mcq", "tags": [ "vaswani", "transformer" ], "prompt": { "modality": "text", "value": "Who was the lead author of 'Attention Is All You Need' (2017), the Transformer paper written at Google Brain?" }, "options": [ { "modality": "text", "value": "Ashish Vaswani" }, { "modality": "text", "value": "Ilya Sutskever" }, { "modality": "text", "value": "Andrej Karpathy" }, { "modality": "text", "value": "Aidan Gomez" } ], "correctIndex": 0, "explanation": "Ashish Vaswani led the 2017 Transformer paper at Google Brain. Aidan Gomez was a co-author (and later co-founded Cohere), but not the lead author.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1dscoubqpaitd" }, { "id": "ot-res-2", "shape": "mcq", "tags": [ "sutskever", "alexnet" ], "prompt": { "modality": "text", "value": "Which two landmark works did Ilya Sutskever co-author before co-founding Safe Superintelligence Inc.?" }, "options": [ { "modality": "text", "value": "AlexNet (2012) and seq2seq (2014)", "short": "AlexNet & seq2seq" }, { "modality": "text", "value": "AlphaFold and AlphaGo", "short": "AlphaFold & AlphaGo" }, { "modality": "text", "value": "The Transformer paper and AlphaGo", "short": "Transformer paper & AlphaGo" }, { "modality": "text", "value": "nanoGPT and Eureka Labs", "short": "nanoGPT & Eureka Labs" } ], "correctIndex": 0, "explanation": "Sutskever co-authored AlexNet (2012) and seq2seq (2014). AlphaGo and AlphaFold are Hassabis's DeepMind work; nanoGPT and Eureka Labs are Karpathy's.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "18jkao1mjjgs0" }, { "id": "ot-res-3", "shape": "mcq", "tags": [ "karpathy", "education" ], "prompt": { "modality": "text", "value": "Which researcher was a founding member of OpenAI and Director of AI at Tesla, and became a celebrated educator via nanoGPT and Eureka Labs?" }, "options": [ { "modality": "text", "value": "Ilya Sutskever" }, { "modality": "text", "value": "Ashish Vaswani" }, { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Andrej Karpathy" } ], "correctIndex": 3, "explanation": "Andrej Karpathy is the OpenAI founding member and ex-Tesla AI director behind nanoGPT and Eureka Labs. Sutskever also co-founded OpenAI but was its Chief Scientist, not at Tesla.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "395hyg1j07xic" }, { "id": "ot-res-4", "shape": "mcq", "tags": [ "vaswani", "adept" ], "prompt": { "modality": "text", "value": "After the Transformer paper, Ashish Vaswani went on to co-found which company?" }, "options": [ { "modality": "text", "value": "Safe Superintelligence Inc.", "short": "Safe Superintelligence Inc." }, { "modality": "text", "value": "Eureka Labs", "short": "Eureka Labs" }, { "modality": "text", "value": "Cohere", "short": "Cohere" }, { "modality": "text", "value": "Adept AI", "short": "Adept AI" } ], "correctIndex": 3, "explanation": "Vaswani co-founded Adept AI. Cohere was co-founded by fellow Transformer co-author Aidan Gomez, Safe Superintelligence by Sutskever, and Eureka Labs by Karpathy.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1q2xbst10g35mr" }, { "id": "ot-res-5", "shape": "mcq", "tags": [ "researchers", "companies" ], "prompt": { "modality": "text", "value": "Which organization was NOT founded or co-founded by Vaswani, Sutskever, or Karpathy?" }, "options": [ { "modality": "text", "value": "Eureka Labs", "short": "Eureka Labs" }, { "modality": "text", "value": "Mila", "short": "Mila" }, { "modality": "text", "value": "Safe Superintelligence Inc.", "short": "Safe Superintelligence Inc." }, { "modality": "text", "value": "Adept AI", "short": "Adept AI" } ], "correctIndex": 1, "explanation": "Mila is headed by Yoshua Bengio. Adept AI is Vaswani's, Safe Superintelligence Inc. (2024) is Sutskever's, and Eureka Labs is Karpathy's.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "f68o299xa4wf" }, { "id": "ot-lead-1", "shape": "mcq", "tags": [ "altman", "openai" ], "prompt": { "modality": "text", "value": "Who has been CEO of OpenAI since 2019 and was famously ousted and reinstated by the board in November 2023?" }, "options": [ { "modality": "text", "value": "Sam Altman" }, { "modality": "text", "value": "Elon Musk" }, { "modality": "text", "value": "Dario Amodei" }, { "modality": "text", "value": "Demis Hassabis" } ], "correctIndex": 0, "explanation": "Sam Altman has been OpenAI's CEO since 2019 and led ChatGPT's launch. Dario Amodei was OpenAI's VP of Research before leaving in 2021 to lead Anthropic.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "uhbev013xy5xc" }, { "id": "ot-lead-2", "shape": "mcq", "tags": [ "daniela-amodei", "anthropic" ], "prompt": { "modality": "text", "value": "Which role does Daniela Amodei hold at Anthropic?" }, "options": [ { "modality": "text", "value": "VP of Research" }, { "modality": "text", "value": "Chief Scientist" }, { "modality": "text", "value": "President" }, { "modality": "text", "value": "Chief AI Scientist" } ], "correctIndex": 2, "explanation": "Daniela Amodei is Anthropic's president and a former OpenAI safety leader; she co-founded the company with her brother Dario, who is CEO. VP of Research was Dario's old OpenAI title.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1lut1r7t35spt" }, { "id": "ot-lead-3", "shape": "mcq", "tags": [ "hassabis", "deepmind" ], "prompt": { "modality": "text", "value": "Demis Hassabis, co-founder and CEO of Google DeepMind, shared which prize for protein-structure prediction?" }, "options": [ { "modality": "text", "value": "The 2018 Turing Award", "short": "2018 Turing Award" }, { "modality": "text", "value": "The 2024 Nobel Prize in Physics", "short": "2024 Nobel Prize, Physics" }, { "modality": "text", "value": "The 2018 Nobel Prize in Chemistry", "short": "2018 Nobel Prize, Chemistry" }, { "modality": "text", "value": "The 2024 Nobel Prize in Chemistry", "short": "2024 Nobel Prize, Chemistry" } ], "correctIndex": 3, "explanation": "Hassabis shared the 2024 Nobel Prize in Chemistry for protein-structure prediction (AlphaFold). The 2018 Turing Award went to the three godfathers of deep learning.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "12k2eze1s4jm1q" }, { "id": "ot-lead-4", "shape": "mcq", "tags": [ "nvidia", "ecosystem" ], "prompt": { "modality": "text", "value": "Which company supplies the GPUs and CUDA software that train and run nearly every large model?" }, "options": [ { "modality": "text", "value": "Meta AI" }, { "modality": "text", "value": "Cohere" }, { "modality": "text", "value": "Nvidia" }, { "modality": "text", "value": "Hugging Face" } ], "correctIndex": 2, "explanation": "Nvidia is the hardware backbone of the AI boom via its GPUs and CUDA. Hugging Face is the other indispensable non-lab, but it hosts models and datasets rather than making chips.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "2u5k9h1pzogjr" }, { "id": "ot-gpt-sizes-1", "shape": "mcq", "tags": [ "gpt-1", "parameters" ], "prompt": { "modality": "text", "value": "How many parameters did GPT-1 (2018) have?" }, "options": [ { "modality": "text", "value": "117 million" }, { "modality": "text", "value": "175 billion" }, { "modality": "text", "value": "1.5 billion" }, { "modality": "text", "value": "340 million" } ], "correctIndex": 0, "explanation": "GPT-1 had 117 million parameters. 340 million is BERT-large; 1.5 billion is GPT-2; 175 billion is GPT-3.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "5w78botc4ipw" }, { "id": "ot-gpt-sizes-2", "shape": "mcq", "tags": [ "gpt-2", "parameters" ], "prompt": { "modality": "text", "value": "The largest version of GPT-2 (2019) had how many parameters?" }, "options": [ { "modality": "text", "value": "1.5 billion" }, { "modality": "text", "value": "117 million" }, { "modality": "text", "value": "15 billion" }, { "modality": "text", "value": "175 billion" } ], "correctIndex": 0, "explanation": "GPT-2 jumped to 1.5 billion parameters — about an order of magnitude over GPT-1's 117 million. 175 billion is GPT-3, a generation later.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1a0o0xaqo38cu" }, { "id": "ot-gpt-sizes-3", "shape": "mcq", "tags": [ "gpt-3", "parameters" ], "prompt": { "modality": "text", "value": "How many parameters did GPT-3 (2020) have?" }, "options": [ { "modality": "text", "value": "540 billion" }, { "modality": "text", "value": "175 billion" }, { "modality": "text", "value": "1.5 billion" }, { "modality": "text", "value": "70 billion" } ], "correctIndex": 1, "explanation": "GPT-3 had 175 billion parameters, the model that revealed in-context learning. 1.5 billion is GPT-2; 540 billion is PaLM.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "91k9yslvil6c" }, { "id": "ot-gpt-sizes-4", "shape": "mcq", "tags": [ "gpt-3", "training-data" ], "prompt": { "modality": "text", "value": "GPT-3 was trained on roughly how many tokens of filtered text?" }, "options": [ { "modality": "text", "value": "300 billion" }, { "modality": "text", "value": "40 billion" }, { "modality": "text", "value": "175 billion" }, { "modality": "text", "value": "4 trillion" } ], "correctIndex": 0, "explanation": "GPT-3 was trained on roughly 300 billion tokens. 175 billion is its parameter count, not its data size — a tempting mix-up.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "11rjefqqr2d7i" }, { "id": "ot-gpt-sizes-5", "shape": "mcq", "tags": [ "gpt-line", "parameters" ], "prompt": { "modality": "text", "value": "Which of these model-and-size pairings is NOT correct?" }, "options": [ { "modality": "text", "value": "GPT-2: 1.5 billion parameters", "short": "GPT-2: 1.5B params" }, { "modality": "text", "value": "GPT-3: 175 billion parameters", "short": "GPT-3: 175B params" }, { "modality": "text", "value": "GPT-2: 340 million parameters", "short": "GPT-2: 340M params" }, { "modality": "text", "value": "GPT-1: 117 million parameters", "short": "GPT-1: 117M params" } ], "correctIndex": 2, "explanation": "GPT-2 had 1.5 billion parameters, not 340 million — 340 million is BERT-large. The other three pairings are accurate.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "hgqlfz15fehjh" }, { "id": "ot-other-sizes-1", "shape": "mcq", "tags": [ "bert", "parameters" ], "prompt": { "modality": "text", "value": "How many parameters does BERT-large have?" }, "options": [ { "modality": "text", "value": "540 million" }, { "modality": "text", "value": "7 billion" }, { "modality": "text", "value": "117 million" }, { "modality": "text", "value": "340 million" } ], "correctIndex": 3, "explanation": "BERT-large has 340 million parameters. 117 million is GPT-1, released the same year — an easy confusion.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "v7pea9lj66w3" }, { "id": "ot-other-sizes-2", "shape": "mcq", "tags": [ "palm", "parameters" ], "prompt": { "modality": "text", "value": "How many parameters does PaLM have?" }, "options": [ { "modality": "text", "value": "540 billion" }, { "modality": "text", "value": "70 billion" }, { "modality": "text", "value": "340 billion" }, { "modality": "text", "value": "175 billion" } ], "correctIndex": 0, "explanation": "PaLM has 540 billion parameters, larger than GPT-3's 175 billion. 340 is BERT-large's count but in millions, not billions.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "po2buu1fh9fh2" }, { "id": "ot-other-sizes-3", "shape": "mcq", "tags": [ "mistral", "llama", "parameters" ], "prompt": { "modality": "text", "value": "Which pair of landmark models share the same parameter count?" }, "options": [ { "modality": "text", "value": "BERT-large and GPT-1", "short": "BERT-large & GPT-1" }, { "modality": "text", "value": "Mistral 7B and LLaMA-7B", "short": "Mistral 7B & LLaMA-7B" }, { "modality": "text", "value": "LLaMA-2 70B and PaLM", "short": "LLaMA-2 70B & PaLM" }, { "modality": "text", "value": "PaLM and GPT-3", "short": "PaLM & GPT-3" } ], "correctIndex": 1, "explanation": "Both are 7-billion-parameter models. BERT-large (340M) differs from GPT-1 (117M), PaLM (540B) from GPT-3 (175B), and LLaMA-2 70B (70B) from PaLM.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "z5xtjf19jtnm5" }, { "id": "ot-other-sizes-4", "shape": "mcq", "tags": [ "palm", "parameters" ], "prompt": { "modality": "text", "value": "Which of these model-and-size pairings is NOT correct?" }, "options": [ { "modality": "text", "value": "Mistral 7B: 7 billion parameters", "short": "Mistral 7B: 7B params" }, { "modality": "text", "value": "BERT-large: 340 million parameters", "short": "BERT-large: 340M params" }, { "modality": "text", "value": "PaLM: 175 billion parameters", "short": "PaLM: 175B params" }, { "modality": "text", "value": "LLaMA-2 70B: 70 billion parameters", "short": "LLaMA-2 70B: 70B params" } ], "correctIndex": 2, "explanation": "PaLM has 540 billion parameters; 175 billion is GPT-3's count. The other three pairings match the guide.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1drg926hakufi" }, { "id": "ot-other-sizes-5", "shape": "mcq", "tags": [ "parameters", "compare" ], "prompt": { "modality": "text", "value": "Which of these models has the largest parameter count?" }, "options": [ { "modality": "text", "value": "LLaMA-2 70B" }, { "modality": "text", "value": "GPT-3" }, { "modality": "text", "value": "BERT-large" }, { "modality": "text", "value": "PaLM" } ], "correctIndex": 3, "explanation": "PaLM's 540 billion parameters top GPT-3's 175 billion, LLaMA-2 70B's 70 billion, and BERT-large's 340 million.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "qw3v641v44j9k" }, { "id": "ot-dates-1", "shape": "mcq", "tags": [ "bert", "timeline" ], "prompt": { "modality": "text", "value": "In what year was BERT released?" }, "options": [ { "modality": "text", "value": "2019" }, { "modality": "text", "value": "2017" }, { "modality": "text", "value": "2020" }, { "modality": "text", "value": "2018" } ], "correctIndex": 3, "explanation": "BERT was released in 2018, the same year as GPT-1. GPT-2 followed in 2019 and GPT-3 in 2020.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1bj1vvvq6w6xt" }, { "id": "ot-dates-2", "shape": "mcq", "tags": [ "gpt-2", "timeline" ], "prompt": { "modality": "text", "value": "In what year was GPT-2 released?" }, "options": [ { "modality": "text", "value": "2018" }, { "modality": "text", "value": "2020" }, { "modality": "text", "value": "2021" }, { "modality": "text", "value": "2019" } ], "correctIndex": 3, "explanation": "GPT-2 arrived in 2019, between GPT-1 (2018) and GPT-3 (2020).", "source": { "label": "LLM 101 — Objective Test" }, "uid": "k6iqc3eya9lx" }, { "id": "ot-dates-3", "shape": "mcq", "tags": [ "llama", "timeline" ], "prompt": { "modality": "text", "value": "In what year did Meta release LLaMA?" }, "options": [ { "modality": "text", "value": "2023" }, { "modality": "text", "value": "2021" }, { "modality": "text", "value": "2020" }, { "modality": "text", "value": "2022" } ], "correctIndex": 0, "explanation": "LLaMA was released by Meta in 2023, the same year as GPT-4. ChatGPT had launched in late 2022.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "vuggyg18zbdvs" }, { "id": "ot-dates-4", "shape": "mcq", "tags": [ "timeline", "order" ], "prompt": { "modality": "text", "value": "Which of these models was released earliest?" }, "options": [ { "modality": "text", "value": "BERT" }, { "modality": "text", "value": "GPT-2" }, { "modality": "text", "value": "LLaMA" }, { "modality": "text", "value": "GPT-3" } ], "correctIndex": 0, "explanation": "BERT (2018) came first, then GPT-2 (2019), GPT-3 (2020), and LLaMA (2023).", "source": { "label": "LLM 101 — Objective Test" }, "uid": "au9rg184iglz" }, { "id": "ot-chatgpt-1", "shape": "mcq", "tags": [ "chatgpt", "timeline" ], "prompt": { "modality": "text", "value": "On what date did ChatGPT launch?" }, "options": [ { "modality": "text", "value": "January 30, 2023" }, { "modality": "text", "value": "November 30, 2021" }, { "modality": "text", "value": "November 30, 2022" }, { "modality": "text", "value": "March 14, 2023" } ], "correctIndex": 2, "explanation": "ChatGPT launched November 30, 2022. March 14, 2023 is GPT-4's release date, a few months later.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "vcsb9ovpm044" }, { "id": "ot-chatgpt-2", "shape": "mcq", "tags": [ "chatgpt", "adoption" ], "prompt": { "modality": "text", "value": "How long did ChatGPT take to reach about 100 million monthly users?" }, "options": [ { "modality": "text", "value": "About 2 months" }, { "modality": "text", "value": "About 2 years" }, { "modality": "text", "value": "About 6 months" }, { "modality": "text", "value": "About 12 months" } ], "correctIndex": 0, "explanation": "ChatGPT hit roughly 100 million monthly users within two months — the fastest-growing consumer app in history at the time.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1awzrec12oa0iw" }, { "id": "ot-chatgpt-3", "shape": "mcq", "tags": [ "gpt-4", "timeline" ], "prompt": { "modality": "text", "value": "GPT-4 was released on March 14 of which year?" }, "options": [ { "modality": "text", "value": "2024" }, { "modality": "text", "value": "2023" }, { "modality": "text", "value": "2022" }, { "modality": "text", "value": "2021" } ], "correctIndex": 1, "explanation": "GPT-4 launched March 14, 2023. 2022 is the year ChatGPT debuted, on November 30.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "17biq0jihaq01" }, { "id": "ot-chatgpt-4", "shape": "mcq", "tags": [ "gpt-4", "timeline" ], "prompt": { "modality": "text", "value": "Roughly how long after ChatGPT's debut did GPT-4 arrive?" }, "options": [ { "modality": "text", "value": "About two years", "short": "~2 years" }, { "modality": "text", "value": "About two months", "short": "~2 months" }, { "modality": "text", "value": "About three and a half months", "short": "~3.5 months" }, { "modality": "text", "value": "About nine months", "short": "~9 months" } ], "correctIndex": 2, "explanation": "ChatGPT launched November 30, 2022 and GPT-4 followed March 14, 2023 — about three and a half months. Two months is how long ChatGPT took to reach 100 million users.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1ylylgv16ouq3h" }, { "id": "ot-chatgpt-5", "shape": "mcq", "tags": [ "chatgpt", "adoption" ], "prompt": { "modality": "text", "value": "Which statement about ChatGPT's launch is NOT accurate?" }, "options": [ { "modality": "text", "value": "It launched on November 30, 2022", "short": "Launched Nov 30, 2022" }, { "modality": "text", "value": "It was the fastest-growing consumer app in history at the time", "short": "Fastest-growing app ever" }, { "modality": "text", "value": "It took about a year to reach 100 million monthly users", "short": "~1 year to 100M users" }, { "modality": "text", "value": "It reached about 100 million monthly users within two months", "short": "100M users in ~2 months" } ], "correctIndex": 2, "explanation": "The climb to 100 million monthly users took roughly two months, not a year — which is exactly why it set the growth record.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "4rtdm01ycrg4" }, { "id": "ot-lang-summarization-catch", "shape": "mcq", "tags": [ "summarization", "limitations" ], "prompt": { "modality": "text", "value": "Summarization is called a top LLM strength. What catch does the section attach to it?" }, "options": [ { "modality": "text", "value": "It cannot produce structured summaries, only prose", "short": "Can't do structured summaries" }, { "modality": "text", "value": "Nuance and exact figures can drop, so verify critical details", "short": "Nuance/figures can drop; verify" }, { "modality": "text", "value": "The model refuses documents longer than a few pages", "short": "Refuses long documents" }, { "modality": "text", "value": "Summaries come out longer than the source text", "short": "Summaries longer than source" } ], "correctIndex": 1, "explanation": "Summarization condenses long documents well, but nuance and precise figures can be lost, so critical details need verifying. Structured summaries are actually a stated strength, not a limit.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1asuj4t1u9yhpj" }, { "id": "ot-lang-not-a-strength", "shape": "mcq", "tags": [ "language-tasks", "fit" ], "prompt": { "modality": "text", "value": "Which of these is NOT listed among the language and writing tasks LLMs handle well?" }, "options": [ { "modality": "text", "value": "Plain-language simplification", "short": "Plain-language simplification" }, { "modality": "text", "value": "Rewriting and tone-shifting", "short": "Rewriting and tone-shifting" }, { "modality": "text", "value": "Tool-free exact arithmetic", "short": "Tool-free exact arithmetic" }, { "modality": "text", "value": "Classification and sentiment", "short": "Classification and sentiment" } ], "correctIndex": 2, "explanation": "Simplification, rewriting/tone-shifting, and classification are all listed language strengths. Needing exact arithmetic without tools is named as a marker of poor LLM fit.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "11b1nds1o06h48" }, { "id": "ot-lang-classification-catch", "shape": "mcq", "tags": [ "classification", "prompting" ], "prompt": { "modality": "text", "value": "Few-shot prompting can match dedicated classifiers on a given label set. What caveat does the section add?" }, "options": [ { "modality": "text", "value": "It cannot return its answer as a structured field", "short": "Can't return structured field" }, { "modality": "text", "value": "It only supports two labels at a time", "short": "Only two labels at a time" }, { "modality": "text", "value": "It needs a full fine-tune before it reaches classifier parity", "short": "Needs a full fine-tune first" }, { "modality": "text", "value": "It is non-deterministic, so use low temperature and test on held-out examples", "short": "Non-deterministic; test outputs" } ], "correctIndex": 3, "explanation": "The stated catch is non-determinism: lower the temperature and validate on held-out examples. Fine-tuning is not required for parity, which is the point of few-shot prompting.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "6858mrqqp8m5" }, { "id": "ot-lang-plain-language", "shape": "mcq", "tags": [ "accessibility", "simplification" ], "prompt": { "modality": "text", "value": "A hospital rewrites a dense consent form so low-literacy and non-native readers can follow it. Which listed strength is this?" }, "options": [ { "modality": "text", "value": "Structured data extraction", "short": "Structured data extraction" }, { "modality": "text", "value": "Plain-language simplification", "short": "Plain-language simplification" }, { "modality": "text", "value": "Semantic search", "short": "Semantic search" }, { "modality": "text", "value": "Classification and sentiment", "short": "Classification and sentiment" } ], "correctIndex": 1, "explanation": "Rewriting technical, legal, or medical text to a plain reading level is plain-language simplification, described as high value, low risk, and easy to verify. Extraction outputs fields, not rewrites.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "9wjv1m19u6rjm" }, { "id": "ot-kc-copilot-speedup", "shape": "mcq", "tags": [ "copilot", "evidence" ], "prompt": { "modality": "text", "value": "In the controlled study cited, how much faster did developers complete a task with GitHub Copilot than without it?" }, "options": [ { "modality": "text", "value": "72.6% faster" }, { "modality": "text", "value": "27.5% faster" }, { "modality": "text", "value": "55.8% faster" }, { "modality": "text", "value": "41.2% faster" } ], "correctIndex": 2, "explanation": "The controlled study found developers finished the task 55.8% faster with Copilot, roughly a 56% speedup. The other figures do not appear in the evidence cited.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "18deki81x4v0ss" }, { "id": "ot-kc-junior-gain", "shape": "mcq", "tags": [ "copilot", "coding" ], "prompt": { "modality": "text", "value": "In that Copilot study, which group of developers gained the most from the tool?" }, "options": [ { "modality": "text", "value": "Developers working in low-level languages", "short": "Low-level-language devs" }, { "modality": "text", "value": "Senior architects", "short": "Senior architects" }, { "modality": "text", "value": "Developers with prior Copilot experience", "short": "Prior-Copilot devs" }, { "modality": "text", "value": "Junior developers", "short": "Junior developers" } ], "correctIndex": 3, "explanation": "Junior developers gained most, consistent with the idea that the tool compresses the gap between knowing what to write and knowing how to phrase it. Seniors already have that phrasing fluency.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "13nxtmh1l67op3" }, { "id": "ot-kc-semantic-search", "shape": "mcq", "tags": [ "semantic-search", "embeddings" ], "prompt": { "modality": "text", "value": "What makes semantic search different from keyword search, and where does it get weaker?" }, "options": [ { "modality": "text", "value": "Embeddings retrieve by meaning; it weakens on niche jargon", "short": "Meaning-based; weak on jargon" }, { "modality": "text", "value": "It ranks results by recency; it weakens on older archives", "short": "Ranks by recency; weak on old" }, { "modality": "text", "value": "It matches exact keyword strings faster; it weakens on long documents", "short": "Exact strings; weak on long docs" }, { "modality": "text", "value": "It requires fine-tuning per corpus; it weakens on short queries", "short": "Per-corpus tune; weak on short" } ], "correctIndex": 0, "explanation": "Embeddings retrieve passages by meaning, so 'contract renewals' can find docs that never use the phrase. The catch is niche jargon unless the embedding model covers it. No per-corpus fine-tune is needed.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "n0xnmo1ky95h0" }, { "id": "ot-kc-test-writing-catch", "shape": "mcq", "tags": [ "testing", "coding" ], "prompt": { "modality": "text", "value": "LLMs generate unit tests covering happy paths and common edge cases. What does the guide say still needs a human?" }, "options": [ { "modality": "text", "value": "Docstrings for the function under test", "short": "Docstrings for the function" }, { "modality": "text", "value": "Happy-path coverage, which the model cannot produce", "short": "Happy-path coverage" }, { "modality": "text", "value": "Basic assertion syntax for the target test framework", "short": "Basic assertion syntax" }, { "modality": "text", "value": "Your domain's invariants and business rules, so unique edge cases get missed", "short": "Domain invariants & biz rules" } ], "correctIndex": 3, "explanation": "Test generation covers happy paths and common edge cases, but the model does not know your domain's invariants or business rules, so unique edge cases still need a human. Docstrings are a stated strength.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "13fks2m1rolci6" }, { "id": "ot-work-meeting-notes", "shape": "mcq", "tags": [ "meeting-notes", "limitations" ], "prompt": { "modality": "text", "value": "LLMs summarize meeting transcripts into decisions and to-dos. What does the section say tends to go wrong?" }, "options": [ { "modality": "text", "value": "Speaker attribution and acronyms get mangled", "short": "Speaker attribution/acronyms off" }, { "modality": "text", "value": "Action items are produced only in the speaker's language", "short": "Action items in speaker's lang" }, { "modality": "text", "value": "Transcripts over ten minutes cannot be processed", "short": "Can't do >10-min transcripts" }, { "modality": "text", "value": "Decisions are always omitted from the summary", "short": "Decisions always omitted" } ], "correctIndex": 0, "explanation": "The catch is mangled speaker attribution and acronyms, which is why attendees should skim notes before they are distributed. Extracting decisions and to-dos is the strength, not the failure.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "9ijum5we5zzn" }, { "id": "ot-work-support-catch", "shape": "mcq", "tags": [ "support", "guardrails" ], "prompt": { "modality": "text", "value": "RAG-backed support chatbots answer tier-1 questions around the clock. What is the risk the section names?" }, "options": [ { "modality": "text", "value": "They must be retrained for each new product manual", "short": "Retrain for each new manual" }, { "modality": "text", "value": "They only work during business hours in one time zone", "short": "Only work business hours" }, { "modality": "text", "value": "They cannot escalate anything to a human agent", "short": "Can't escalate to a human" }, { "modality": "text", "value": "Out-of-scope questions get confident wrong answers without good guardrails", "short": "Confident wrong out-of-scope" } ], "correctIndex": 3, "explanation": "Good bots escalate ambiguous questions; the danger is out-of-scope questions answered confidently and wrongly when guardrails are weak. RAG also means new manuals are indexed, not retrained.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1m4tcf89b884k" }, { "id": "ot-work-tutoring-catch", "shape": "mcq", "tags": [ "tutoring", "limitations" ], "prompt": { "modality": "text", "value": "LLMs make patient, always-available tutors. Why does the section still recommend pairing them with a verified curriculum?" }, "options": [ { "modality": "text", "value": "They are unavailable outside scheduled sessions", "short": "Unavailable off-schedule" }, { "modality": "text", "value": "They refuse to answer beginner-level questions", "short": "Refuse beginner questions" }, { "modality": "text", "value": "Factual errors are hard for learners to spot", "short": "Learners miss factual errors" }, { "modality": "text", "value": "They can only explain a concept one way", "short": "Only explain one way" } ], "correctIndex": 2, "explanation": "A learner lacks the background to catch a confident factual error, so a verified curriculum supplies the ground truth. Explaining a concept ten ways and constant availability are the stated strengths.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1sgx29prm9muf" }, { "id": "ot-work-not-listed", "shape": "mcq", "tags": [ "work-tasks", "scope" ], "prompt": { "modality": "text", "value": "Which of these is NOT one of the everyday work or personal uses the section lists?" }, "options": [ { "modality": "text", "value": "Email and report drafting", "short": "Email and report drafting" }, { "modality": "text", "value": "Data cleaning and normalization", "short": "Data cleaning & normalization" }, { "modality": "text", "value": "Executing a trade from a portfolio brief", "short": "Executing a trade from a brief" }, { "modality": "text", "value": "Brainstorming and ideation", "short": "Brainstorming and ideation" } ], "correctIndex": 2, "explanation": "Data cleaning, drafting, and brainstorming are all listed everyday uses. An executed trade is used as an example of an action too consequential to automate without human approval.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "15wnovo1kox12o" }, { "id": "ot-cop-definition", "shape": "mcq", "tags": [ "copilot-pattern", "definition" ], "prompt": { "modality": "text", "value": "How does the guide define the copilot pattern?" }, "options": [ { "modality": "text", "value": "The LLM acts first and reports its actions to a human afterward", "short": "LLM acts; reports after" }, { "modality": "text", "value": "The LLM suggests, drafts, or assists and a human approves before anything takes effect", "short": "LLM assists; human OKs first" }, { "modality": "text", "value": "A human writes the draft and the LLM grades it for quality", "short": "Human drafts; LLM grades it" }, { "modality": "text", "value": "Two models cross-check each other and ship the agreed output", "short": "Two models cross-check & ship" } ], "correctIndex": 1, "explanation": "The copilot pattern accelerates work without finalizing it: the model suggests or drafts, and a human approves before anything takes effect. Acting first and reporting later is the autonomous design it contrasts with.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1oaqhds181nejc" }, { "id": "ot-cop-contrast", "shape": "mcq", "tags": [ "copilot-pattern", "autonomy" ], "prompt": { "modality": "text", "value": "The copilot pattern is defined in contrast to what other design?" }, "options": [ { "modality": "text", "value": "Retrieval pipelines that ground answers in documents", "short": "Retrieval pipelines (RAG)" }, { "modality": "text", "value": "Autonomous agents that act without per-step review", "short": "Autonomous agents, no review" }, { "modality": "text", "value": "Fine-tuned models trained on proprietary data", "short": "Fine-tuned models" }, { "modality": "text", "value": "Few-shot classifiers tuned with a fixed label set", "short": "Few-shot classifiers" } ], "correctIndex": 1, "explanation": "The contrast case is autonomous agents that act without per-step human review. RAG, few-shot classification, and fine-tuning are techniques, not alternatives to the human-approval checkpoint.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1rsbpyrue0l5x" }, { "id": "ot-cop-apply-automation", "shape": "mcq", "tags": [ "verification", "automation" ], "prompt": { "modality": "text", "value": "Using the cost-of-verification framework, which task best justifies more automation with a lighter human checkpoint?" }, "options": [ { "modality": "text", "value": "Approving a step in a surgical procedure", "short": "Approving a surgery step" }, { "modality": "text", "value": "Generating a unit test that a CI run immediately checks", "short": "Unit test checked by CI" }, { "modality": "text", "value": "Executing a trade on a live account", "short": "Trade on a live account" }, { "modality": "text", "value": "Filing a brief in federal court", "short": "Filing a federal court brief" } ], "correctIndex": 1, "explanation": "A unit test is cheap and fast to verify, so more automation is appropriate. A filing, a trade, and a surgical step are expensive or impossible to verify after the fact, so a human must approve first.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1nlc5nazs0qky" }, { "id": "ot-cop-failure-mode", "shape": "mcq", "tags": [ "autonomy", "risk" ], "prompt": { "modality": "text", "value": "According to the guide, which of these is the failure mode rather than the goal?" }, "options": [ { "modality": "text", "value": "A human reviewing a generated draft before it ships", "short": "Human reviews draft first" }, { "modality": "text", "value": "Running RAG over verified data sources", "short": "RAG over verified sources" }, { "modality": "text", "value": "Running unit tests against generated code", "short": "Unit tests on generated code" }, { "modality": "text", "value": "Fully autonomous LLM deployment on high-stakes decisions", "short": "Autonomous LLM, high-stakes" } ], "correctIndex": 3, "explanation": "The guide states plainly that fully autonomous deployment on high-stakes decisions is the failure mode, not the goal. Human review, verified-source RAG, and testing are the recommended guardrails.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "19gb4kbaq9quh" }, { "id": "ot-fit-good-fit", "shape": "mcq", "tags": [ "good-fit", "task-selection" ], "prompt": { "modality": "text", "value": "Which set of conditions marks a good LLM fit for a task?" }, "options": [ { "modality": "text", "value": "Output is easy to judge, errors are cheap to catch, it is language-in / language-out, and volume justifies it", "short": "Easy judge, cheap errors, volume" }, { "modality": "text", "value": "The task needs exact arithmetic and no calculator tool is available", "short": "Exact arithmetic, no calculator" }, { "modality": "text", "value": "Accountability requires a fully auditable decision trail", "short": "Needs auditable decision trail" }, { "modality": "text", "value": "Correctness can only be confirmed by an expensive external audit", "short": "Only expensive audit verifies" } ], "correctIndex": 0, "explanation": "Good fit means judgeable output, cheap-to-catch errors, a language-in / language-out shape, and enough volume to justify integration. The other three are listed as markers of poor fit.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1j4muzde6upsb" }, { "id": "ot-fit-not-poor-fit", "shape": "mcq", "tags": [ "poor-fit", "task-selection" ], "prompt": { "modality": "text", "value": "Which of these is NOT a signal of poor LLM fit?" }, "options": [ { "modality": "text", "value": "Drafting a summary a reviewer can skim in a minute", "short": "Summary a reviewer can skim" }, { "modality": "text", "value": "Real-time data is needed but there is no retrieval", "short": "Real-time data, no retrieval" }, { "modality": "text", "value": "Correctness cannot be cheaply verified", "short": "Can't cheaply verify correctness" }, { "modality": "text", "value": "Exact arithmetic is required and no tools are available", "short": "Exact arithmetic, no tools" } ], "correctIndex": 0, "explanation": "Tool-less arithmetic, real-time data without retrieval, and un-verifiable correctness are all poor-fit signals. A summary a reviewer skims in a minute is cheap to check, which is the definition of good fit.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "7f5dk21ozgtdu" }, { "id": "ot-dom-legal-sanction", "shape": "mcq", "tags": [ "legal", "hallucination" ], "prompt": { "modality": "text", "value": "What is the canonical 2023 cautionary case the guide cites for legal use of LLMs?" }, "options": [ { "modality": "text", "value": "A judge banned all AI-assisted filings nationwide", "short": "Judge banned AI filings" }, { "modality": "text", "value": "A firm was fined for uploading client contracts to a public model", "short": "Firm fined, uploaded contracts" }, { "modality": "text", "value": "Two paralegals were disbarred for mistranslating a statute", "short": "Paralegals disbarred, statute" }, { "modality": "text", "value": "Two attorneys were sanctioned after filing a brief citing cases ChatGPT had fabricated", "short": "Attorneys sanctioned, fake cases" } ], "correctIndex": 3, "explanation": "Two attorneys were sanctioned for a federal filing built on fabricated citations that looked real but named cases that did not exist. The lesson: LLM legal research starts human search, never replaces it.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "yqes3w1r9fea4" }, { "id": "ot-dom-healthcare-review", "shape": "mcq", "tags": [ "healthcare", "guardrails" ], "prompt": { "modality": "text", "value": "What does the guide require before an LLM output is used in a clinical setting?" }, "options": [ { "modality": "text", "value": "A self-correction prompt asking the model to check its own dosages", "short": "Model self-checks its dosages" }, { "modality": "text", "value": "Clinician review of every output, plus domain-specific evaluation suites", "short": "Clinician review + eval suites" }, { "modality": "text", "value": "Fine-tuning on patient records in place of human review", "short": "Fine-tune, no human review" }, { "modality": "text", "value": "A larger context window holding the full patient chart", "short": "Bigger window for full chart" } ], "correctIndex": 1, "explanation": "Because hallucination may be intrinsic to LLMs, deployment cannot rely on the model's self-correction: every output needs clinician review, and domain-specific evals are a prerequisite, not an option.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "y3t378nwbsjo" }, { "id": "ot-dom-finance-practice", "shape": "mcq", "tags": [ "finance", "guardrails" ], "prompt": { "modality": "text", "value": "What does the guide name as best practice for LLM use in regulated finance?" }, "options": [ { "modality": "text", "value": "RAG over verified data sources plus mandatory human review before customer-facing output", "short": "RAG + mandatory human review" }, { "modality": "text", "value": "Relying on the model to flag its own hallucinated figures", "short": "Model flags own hallucinations" }, { "modality": "text", "value": "Fine-tuning on public filings and shipping outputs directly to customers", "short": "Fine-tune, ship to customers" }, { "modality": "text", "value": "Allowing the model to execute low-value trades autonomously", "short": "Model auto-trades low value" } ], "correctIndex": 0, "explanation": "Outputs driving investment, lending, or compliance carry regulatory exposure, so best practice is RAG over verified sources plus mandatory human review before anything reaches a customer or decision system.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "ysnemj17utawt" }, { "id": "ot-halluc-cutoff-1", "shape": "mcq", "tags": [ "knowledge-cutoff", "staleness" ], "prompt": { "modality": "text", "value": "What does a model's knowledge cutoff refer to?" }, "options": [ { "modality": "text", "value": "The date after which the model's training data ends", "short": "Date training data ends" }, { "modality": "text", "value": "The confidence threshold below which the model refuses to answer", "short": "Confidence refusal threshold" }, { "modality": "text", "value": "The point at which the model stops sampling and returns an answer", "short": "When it stops sampling" }, { "modality": "text", "value": "The maximum number of tokens the model can hold in one call", "short": "Max tokens per call" } ], "correctIndex": 0, "explanation": "The knowledge cutoff is the date after which training data ends — the model has no intrinsic knowledge of later events. The token cap is the context window, a separate limit.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1zola41d8kd98" }, { "id": "ot-halluc-confident-2", "shape": "mcq", "tags": [ "confidence", "reliability" ], "prompt": { "modality": "text", "value": "The guide says an LLM's confident tone is not a signal of correctness. Why?" }, "options": [ { "modality": "text", "value": "The training that makes prose fluent makes it uniformly confident, so a guess and a fact read the same", "short": "Guess and fact read the same" }, { "modality": "text", "value": "Confidence is calibrated only for facts after the cutoff date", "short": "Calibrated only post-cutoff" }, { "modality": "text", "value": "Certainty scores exist but are stripped out before display", "short": "Scores stripped before display" }, { "modality": "text", "value": "The model deliberately overstates certainty to seem helpful", "short": "Overstates to seem helpful" } ], "correctIndex": 0, "explanation": "The surface text carries no certainty signal: a low-probability guess and a well-established fact are written in exactly the same register. Treating fluency as a proxy for accuracy reliably misleads.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "k98m8s1eexp1g" }, { "id": "ot-halluc-toolkit-3", "shape": "mcq", "tags": [ "hallucination", "mitigation" ], "prompt": { "modality": "text", "value": "Which of these is NOT one of the guide's defenses against hallucination?" }, "options": [ { "modality": "text", "value": "Raise the sampling temperature so the model explores more answers", "short": "Raise sampling temperature" }, { "modality": "text", "value": "Require the model to quote and link citations", "short": "Quote and link citations" }, { "modality": "text", "value": "Ground answers in retrieved source documents (RAG)", "short": "Ground answers in RAG" }, { "modality": "text", "value": "Have a domain expert verify high-stakes output before it is acted on", "short": "Expert verifies high-stakes" } ], "correctIndex": 0, "explanation": "The four listed defenses are RAG, quoted/linked citations, training the model to say 'I don't know', and expert verification for high-stakes output. Raising temperature adds variance, not grounding.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "milub01q2z6ek" }, { "id": "ot-halluc-staleness-4", "shape": "mcq", "tags": [ "staleness", "rag" ], "prompt": { "modality": "text", "value": "A model deployed two years after its training cutoff is asked about current software versions. What does the guide predict, and what is the fix?" }, "options": [ { "modality": "text", "value": "It states outdated versions as current; the fix is live-data RAG or surfacing the cutoff to users", "short": "Stale as current; fix = RAG" }, { "modality": "text", "value": "It will refuse to answer; the fix is to lower the temperature", "short": "Refuses; fix = lower temp" }, { "modality": "text", "value": "It will flag its own uncertainty; the fix is to trust the flag", "short": "Flags doubt; fix = trust flag" }, { "modality": "text", "value": "It will guess randomly; the fix is a longer context window", "short": "Guesses; fix = longer window" } ], "correctIndex": 0, "explanation": "Models discuss stale prices, laws, office-holders, and software versions as if current, with no built-in warning. The remedy is live-data RAG for time-sensitive tasks, or at minimum telling users the cutoff.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1nze7pc1e11s1w" }, { "id": "ot-bias-stages-1", "shape": "mcq", "tags": [ "bias", "training-stages" ], "prompt": { "modality": "text", "value": "According to the guide, at which three stages does bias enter an LLM?" }, "options": [ { "modality": "text", "value": "Prompting, retrieval, and output filtering", "short": "Prompt, retrieve, filter" }, { "modality": "text", "value": "Data collection, pretraining, and fine-tuning", "short": "Collect data, pretrain, tune" }, { "modality": "text", "value": "Pretraining, inference, and caching", "short": "Pretrain, inference, cache" }, { "modality": "text", "value": "Tokenization, embedding, and decoding", "short": "Tokenize, embed, decode" } ], "correctIndex": 1, "explanation": "Bias enters at data collection, pretraining, and fine-tuning — three distinct stages, each partly mitigated but rarely eliminated. Tokenization and decoding are mechanics, not bias-entry stages.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1bzh72l2prj6" }, { "id": "ot-bias-aave-2", "shape": "mcq", "tags": [ "bias", "aave" ], "prompt": { "modality": "text", "value": "Which documented accuracy gap does the guide cite as evidence that LLM bias is systematic?" }, "options": [ { "modality": "text", "value": "Worse on questions about the future than about the past", "short": "Worse on future than past" }, { "modality": "text", "value": "Worse on written text than on transcribed speech", "short": "Worse on writing than speech" }, { "modality": "text", "value": "Worse on short prompts than on long prompts", "short": "Worse on short than long prompts" }, { "modality": "text", "value": "Worse on African-American Vernacular English than on Standard American English", "short": "Worse on AAVE than standard" } ], "correctIndex": 3, "explanation": "The AAVE-versus-Standard-American-English gap is a measured, repeatable disadvantage for the same groups — which is why 'random noise' is the wrong mental model for bias.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1w4ri7v3fn5i1" }, { "id": "ot-bias-forms-3", "shape": "mcq", "tags": [ "bias", "forms" ], "prompt": { "modality": "text", "value": "Which is NOT listed in the guide as a documented form of LLM bias?" }, "options": [ { "modality": "text", "value": "Weaker output in low-resource languages", "short": "Weaker in low-resource langs" }, { "modality": "text", "value": "Randomly varying accuracy that affects every demographic group equally", "short": "Random, all groups equally" }, { "modality": "text", "value": "Cultural skew, such as assuming Western norms in advice", "short": "Western-norm cultural skew" }, { "modality": "text", "value": "Stereotypical associations, such as job titles correlated with genders", "short": "Job titles tied to genders" } ], "correctIndex": 1, "explanation": "The guide stresses the problems are systematic, not random — the same groups are disadvantaged repeatedly. Stereotypical associations, cultural skew, and low-resource-language weakness are all documented forms.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1u08n791m36fhz" }, { "id": "ot-bias-source-4", "shape": "mcq", "tags": [ "bias", "training-data" ], "prompt": { "modality": "text", "value": "Where does the guide say an LLM's bias originates?" }, "options": [ { "modality": "text", "value": "Comes from users repeating biased prompts at inference time", "short": "From biased user prompts" }, { "modality": "text", "value": "Injected deliberately by fine-tuning teams to match a target market", "short": "Injected by fine-tuning teams" }, { "modality": "text", "value": "Emerges from arithmetic rounding errors during pretraining", "short": "Rounding errors in pretraining" }, { "modality": "text", "value": "Absorbed from a corpus over-representing English, Western demographics, and majority viewpoints", "short": "English/Western-skewed corpus" } ], "correctIndex": 3, "explanation": "Bias is not added accidentally — it is absorbed from the training corpus. The dominant sources of web text over-represent English, Western demographics, and majority viewpoints, so the model reflects them by default.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1cgvm47187e42h" }, { "id": "ot-bias-hard-5", "shape": "mcq", "tags": [ "bias", "mitigation" ], "prompt": { "modality": "text", "value": "Why do alignment techniques and output filters fail to eliminate bias?" }, "options": [ { "modality": "text", "value": "They work perfectly but are too expensive to run at scale", "short": "Too expensive at scale" }, { "modality": "text", "value": "They operate on the tokenizer, which is not where bias resides", "short": "Operate on the tokenizer" }, { "modality": "text", "value": "They only run above temperature 0, so they miss deterministic outputs", "short": "Only run above temperature 0" }, { "modality": "text", "value": "They leave the training distribution intact, and debiasing one dimension creates disparities elsewhere", "short": "Leave the training data intact" } ], "correctIndex": 3, "explanation": "Filters suppress the most visible bias but leave the underlying training distribution intact, and fixing one dimension can shift the problem elsewhere. That is why NIST's AI RMF treats data, evaluation, and human factors as ongoing challenges.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1mib3d1eh0t3f" }, { "id": "ot-privacy-paste-1", "shape": "mcq", "tags": [ "privacy", "consumer-llm" ], "prompt": { "modality": "text", "value": "On a consumer LLM product, what does the guide say may happen to the inputs you paste?" }, "options": [ { "modality": "text", "value": "Stored, but legally no human may ever read them", "short": "Stored, but no human reads them" }, { "modality": "text", "value": "Logged for abuse monitoring, used to train future models, or reviewed by support staff", "short": "Logged, trained on, or reviewed" }, { "modality": "text", "value": "Encrypted and discarded the moment the response is generated", "short": "Encrypted, then discarded" }, { "modality": "text", "value": "Shared only with other users in the same session", "short": "Shared with users in session" } ], "correctIndex": 1, "explanation": "Consumer tiers may log inputs for abuse monitoring, train on them, or expose them to support staff. The model will not keep your input confidential.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "17s8rtq1ut2zj2" }, { "id": "ot-privacy-enterprise-2", "shape": "mcq", "tags": [ "privacy", "enterprise" ], "prompt": { "modality": "text", "value": "How do enterprise API tiers with data-processing agreements typically differ from consumer chat products?" }, "options": [ { "modality": "text", "value": "They eliminate hallucination through mandatory RAG", "short": "Eliminate hallucination via RAG" }, { "modality": "text", "value": "They guarantee the model has no knowledge cutoff", "short": "Guarantee no knowledge cutoff" }, { "modality": "text", "value": "They typically prohibit training on customer data", "short": "Prohibit training on your data" }, { "modality": "text", "value": "They remove all context-window limits", "short": "Remove context-window limits" } ], "correctIndex": 2, "explanation": "Enterprise tiers with data-processing agreements typically forbid training on customer data — the key contractual difference from consumer chat. They do nothing about cutoffs or hallucination.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "o436dcg7mn3o" }, { "id": "ot-privacy-memorize-3", "shape": "mcq", "tags": [ "privacy", "memorization", "pii" ], "prompt": { "modality": "text", "value": "The memorization risk is described as separate from the risk to your pasted inputs. What is it?" }, "options": [ { "modality": "text", "value": "It caches your prompt so other users can retrieve it later", "short": "Caches prompts for other users" }, { "modality": "text", "value": "It stores every conversation in its weights during chat", "short": "Stores every chat in weights" }, { "modality": "text", "value": "It remembers only the current context window and nothing else", "short": "Remembers only context window" }, { "modality": "text", "value": "It can regurgitate fragments of training data — emails, phone numbers, passages — when prompted right", "short": "Regurgitates training-data bits" } ], "correctIndex": 3, "explanation": "Memorization is about training data, not your inputs: models can reproduce memorized fragments — emails, phone numbers, whole passages — under the right prompting.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "16gjco8wiq2m4" }, { "id": "ot-privacy-rule-4", "shape": "mcq", "tags": [ "privacy", "high-stakes" ], "prompt": { "modality": "text", "value": "You want to summarize a document containing patient records using a free consumer chatbot. What does the guide advise?" }, "options": [ { "modality": "text", "value": "Go ahead — support staff are legally barred from reading inputs", "short": "Go ahead; staff can't read it" }, { "modality": "text", "value": "Go ahead, but delete the chat afterwards to remove it from training", "short": "Go ahead, then delete the chat" }, { "modality": "text", "value": "Don't paste it without reading the provider's data-use terms; the model won't keep it confidential", "short": "Don't paste; read data-use terms" }, { "modality": "text", "value": "Paste it only if you set temperature to 0", "short": "Paste only at temperature 0" } ], "correctIndex": 2, "explanation": "The rule is simple: never paste credentials, patient records, or confidential IP into a consumer chat without reading the provider's data-use terms first. Deleting a chat and temperature settings do not address data use.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "fivwc6h87jsm" }, { "id": "ot-sec-injection-1", "shape": "mcq", "tags": [ "prompt-injection", "owasp" ], "prompt": { "modality": "text", "value": "What is a prompt injection, per the guide?" }, "options": [ { "modality": "text", "value": "A technique for compressing long prompts to fit the context window", "short": "Compressing prompts to fit" }, { "modality": "text", "value": "Malicious text in user input or a retrieved document that overrides the system prompt", "short": "Malicious text overrides prompt" }, { "modality": "text", "value": "A method of inserting retrieved documents into a RAG pipeline", "short": "Inserting docs into RAG" }, { "modality": "text", "value": "A safety layer injected before the model's output reaches the user", "short": "Safety layer on the output" } ], "correctIndex": 1, "explanation": "Prompt injection is malicious text that overrides the system prompt and makes the model act against intent (OWASP LLM01). Indirect injection hides the instructions in retrieved content; RAG insertion is the legitimate mechanism it abuses.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1of1vn21fodhpe" }, { "id": "ot-sec-jailbreak-2", "shape": "mcq", "tags": [ "jailbreak", "guardrails" ], "prompt": { "modality": "text", "value": "How does a jailbreak relate to prompt injection?" }, "options": [ { "modality": "text", "value": "It is a prompt injection aimed at bypassing safety guardrails to get refused content", "short": "Injection to bypass guardrails" }, { "modality": "text", "value": "It is a hosting setup that runs the model without any system prompt", "short": "Hosting with no system prompt" }, { "modality": "text", "value": "It is the opposite of injection — a defense that restores guardrails", "short": "Opposite: restores guardrails" }, { "modality": "text", "value": "It is unrelated; it exploits the tokenizer rather than the prompt", "short": "Unrelated; exploits tokenizer" } ], "correctIndex": 0, "explanation": "A jailbreak is a specific kind of prompt injection targeting safety guardrails, exploiting the tension between instruction-following and safety fine-tuning.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "njcyry19d2bqa" }, { "id": "ot-sec-automation-3", "shape": "mcq", "tags": [ "automation-bias", "over-reliance" ], "prompt": { "modality": "text", "value": "What is automation bias?" }, "options": [ { "modality": "text", "value": "A skew introduced when training data is scraped automatically", "short": "Skew from auto-scraped data" }, { "modality": "text", "value": "The human tendency to over-trust and under-scrutinize confident-sounding automated output", "short": "Humans over-trust automation" }, { "modality": "text", "value": "The tendency of agents to take actions without being asked", "short": "Agents act without being asked" }, { "modality": "text", "value": "The model's tendency to favor automated tools over reasoning", "short": "Model favors automated tools" } ], "correctIndex": 1, "explanation": "Automation bias is a human failure mode: accepting an LLM's confident answer without applying one's own judgment. Agents acting unasked is 'excessive agency', a different OWASP risk.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "q6vcxr1oocv1p" }, { "id": "ot-sec-agent-fix-4", "shape": "mcq", "tags": [ "agents", "mitigation" ], "prompt": { "modality": "text", "value": "What structural mitigation does the guide recommend for tool-enabled agents?" }, "options": [ { "modality": "text", "value": "Raise the temperature so injected instructions are followed less reliably", "short": "Raise sampling temperature" }, { "modality": "text", "value": "Rely on the system prompt to forbid harmful actions", "short": "Rely on the system prompt" }, { "modality": "text", "value": "Disable RAG so no external documents are ever retrieved", "short": "Disable RAG retrieval" }, { "modality": "text", "value": "Grant minimum necessary permissions and require human confirmation before consequential actions", "short": "Least-privilege + confirm" } ], "correctIndex": 3, "explanation": "The mitigation is structural: minimum necessary permissions plus human confirmation, treating any tool-enabled model as a security boundary. Static system-prompt guardrails alone are exactly what the guide says is insufficient.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1rrxgex608b8v" }, { "id": "ot-gov-redteam-1", "shape": "mcq", "tags": [ "red-teaming", "nist" ], "prompt": { "modality": "text", "value": "What is red-teaming in the context of LLM governance?" }, "options": [ { "modality": "text", "value": "Filtering harmful content out of the training corpus before pretraining", "short": "Filtering the training corpus" }, { "modality": "text", "value": "A standardized document disclosing a model's intended uses and limitations", "short": "A doc of uses and limits" }, { "modality": "text", "value": "Running the model at temperature 0 to check reproducibility", "short": "Running at temp 0 for repro" }, { "modality": "text", "value": "Structured adversarial testing to find jailbreaks, bias failures, and safety gaps before deployment", "short": "Adversarial testing pre-deploy" } ], "correctIndex": 3, "explanation": "Red-teaming is adversarial pre-deployment testing by internal or external specialists; NIST's generative-AI profile recommends it as a core practice. The disclosure document is a model card.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1mpf96ycczajy" }, { "id": "ot-gov-modelcard-2", "shape": "mcq", "tags": [ "model-card", "transparency" ], "prompt": { "modality": "text", "value": "Which of the following does a model card NOT do?" }, "options": [ { "modality": "text", "value": "Report evaluation results across demographic groups", "short": "Report results by demographic" }, { "modality": "text", "value": "Document training data and ethical considerations", "short": "Document data and ethics" }, { "modality": "text", "value": "Disclose intended uses and known limitations", "short": "Disclose uses and limitations" }, { "modality": "text", "value": "Guarantee the model will refuse all unsafe requests", "short": "Guarantee it refuses unsafe" } ], "correctIndex": 3, "explanation": "A model card is a disclosure document — a kind of nutrition label for a model — proposed by Mitchell and colleagues in 2019. It documents behavior; it guarantees nothing about refusals.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "t0gkx045jqnc" }, { "id": "ot-gov-nondet-3", "shape": "mcq", "tags": [ "non-determinism", "temperature" ], "prompt": { "modality": "text", "value": "Your compliance workflow needs the same prompt to return the same answer every time. What does the guide advise?" }, "options": [ { "modality": "text", "value": "Send the prompt several times and take the majority answer as guaranteed", "short": "Send several, take majority" }, { "modality": "text", "value": "Use a fine-tuned model, which is always deterministic", "short": "Fine-tuned = deterministic" }, { "modality": "text", "value": "Use temperature 0 for reproducibility, and regression-test outputs when upgrading model versions", "short": "Use temp 0, regression-test" }, { "modality": "text", "value": "Increase the context window so less sampling is needed", "short": "Increase the context window" } ], "correctIndex": 2, "explanation": "Above temperature 0, outputs are sampled from a probability distribution, so answers vary. Temperature 0 plus regression tests on version upgrades — the distribution shifts — is the guide's fix.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "mmwj131t8rek9" }, { "id": "ot-gov-oversight-4", "shape": "mcq", "tags": [ "governance", "oversight", "high-stakes" ], "prompt": { "modality": "text", "value": "A vendor says its model was red-teamed and has a model card, so it can auto-deny loans with no human review. What does the guide say?" }, "options": [ { "modality": "text", "value": "That is sufficient — red-teaming plus a published model card clears the bar for autonomous high-stakes decisions", "short": "Sufficient for auto decisions" }, { "modality": "text", "value": "Those are testing and disclosure practices, not guarantees; regulatory accountability demands auditable logic", "short": "Not guarantees; audit the logic" }, { "modality": "text", "value": "It is fine as long as the model runs at temperature 0, since outputs are then reproducible", "short": "Fine if run at temperature 0" }, { "modality": "text", "value": "It is fine if the red team was external rather than internal", "short": "Fine if red team was external" } ], "correctIndex": 1, "explanation": "Red-teaming finds gaps and a model card discloses limits; neither removes them — red-teamers keep finding bypasses of guardrails. Where regulatory accountability demands auditable logic, an LLM shouldn't be sole decision-maker.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "6xd1rk1hbu2m4" }, { "id": "ot-arch-token-size", "shape": "mcq", "tags": [ "token", "architecture" ], "prompt": { "modality": "text", "value": "A token is the unit an LLM actually processes. Roughly how much text does one token represent?" }, "options": [ { "modality": "text", "value": "About three-quarters of a word", "short": "About 3/4 of a word" }, { "modality": "text", "value": "About three words", "short": "About three words" }, { "modality": "text", "value": "About one character", "short": "About one character" }, { "modality": "text", "value": "Exactly one word", "short": "Exactly one word" } ], "correctIndex": 0, "explanation": "A token averages about ¾ of a word — it is the unit models process and the currency of context limits and pricing. Tokens are usually sub-word pieces, so \"exactly one word\" is wrong.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "wdk0uv75xiq" }, { "id": "ot-arch-context-window", "shape": "mcq", "tags": [ "context-window", "architecture" ], "prompt": { "modality": "text", "value": "Which term names the model's token-limited working memory — the bound on everything it can \"see\" at once?" }, "options": [ { "modality": "text", "value": "Pretraining" }, { "modality": "text", "value": "Self-attention" }, { "modality": "text", "value": "Context window" }, { "modality": "text", "value": "Few-shot" } ], "correctIndex": 2, "explanation": "The context window is the token-limited working memory. Self-attention is how tokens weigh one another inside a layer, not the limit on how many tokens fit.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1m088eo1un2ngk" }, { "id": "ot-arch-self-attention", "shape": "mcq", "tags": [ "self-attention", "transformer" ], "prompt": { "modality": "text", "value": "Inside a Transformer layer, what does self-attention actually do?" }, "options": [ { "modality": "text", "value": "Retrieves outside documents into the prompt", "short": "Retrieves outside documents" }, { "modality": "text", "value": "Lets tokens weigh one another to decide what matters", "short": "Tokens weigh one another" }, { "modality": "text", "value": "Processes the sequence one step at a time, left to right", "short": "Processes left to right" }, { "modality": "text", "value": "Aligns the model with human preference feedback", "short": "Aligns with human feedback" } ], "correctIndex": 1, "explanation": "Self-attention is how tokens weigh one another inside a Transformer layer. Step-by-step left-to-right processing describes recurrence — exactly what the Transformer replaced.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "tso5zg6bmiis" }, { "id": "ot-arch-odd-one-out", "shape": "mcq", "tags": [ "architecture", "vocabulary" ], "prompt": { "modality": "text", "value": "Three of these terms describe the architecture or its units of text. Which one does NOT?" }, "options": [ { "modality": "text", "value": "Token" }, { "modality": "text", "value": "RLHF" }, { "modality": "text", "value": "Self-attention" }, { "modality": "text", "value": "Context window" } ], "correctIndex": 1, "explanation": "RLHF is a training/alignment step — it uses human preference feedback to turn a raw predictor into an assistant. Token, self-attention, and context window all describe the architecture and what it processes.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "16ienp3tymq8p" }, { "id": "ot-train-pretraining-def", "shape": "mcq", "tags": [ "pretraining", "training" ], "prompt": { "modality": "text", "value": "Which stage of building a model is self-supervised next-token prediction over a huge corpus?" }, "options": [ { "modality": "text", "value": "Inference", "short": "Inference" }, { "modality": "text", "value": "Retrieval-augmented generation", "short": "Retrieval-augmented generation" }, { "modality": "text", "value": "RLHF", "short": "RLHF" }, { "modality": "text", "value": "Pretraining", "short": "Pretraining" } ], "correctIndex": 3, "explanation": "Pretraining is the first and most expensive stage — self-supervised next-token training on huge corpora, where the model absorbs its broad knowledge. RLHF comes after, using human preferences.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "14w80hv1j5ksol" }, { "id": "ot-train-rlhf-role", "shape": "mcq", "tags": [ "rlhf", "alignment" ], "prompt": { "modality": "text", "value": "In the training pipeline, what is RLHF's job?" }, "options": [ { "modality": "text", "value": "Absorb broad knowledge from a huge corpus", "short": "Absorb knowledge from corpus" }, { "modality": "text", "value": "Turn a raw predictor into a helpful assistant using human preferences", "short": "Helpful assistant from prefs" }, { "modality": "text", "value": "Ground answers in retrieved documents", "short": "Ground answers in retrieved docs" }, { "modality": "text", "value": "Extend the model's context window", "short": "Extend the context window" } ], "correctIndex": 1, "explanation": "RLHF — Reinforcement Learning from Human Feedback — is the alignment step: it trains on human preferences to turn a raw next-token predictor into a helpful assistant. Absorbing broad knowledge from a corpus is pretraining's job.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1neptnboi78ex" }, { "id": "ot-train-chinchilla-ratio", "shape": "mcq", "tags": [ "chinchilla", "scaling" ], "prompt": { "modality": "text", "value": "Chinchilla is shorthand for a compute-optimal rule of thumb. Roughly how many training tokens per parameter does it call for?" }, "options": [ { "modality": "text", "value": "About 80" }, { "modality": "text", "value": "About 5" }, { "modality": "text", "value": "About 300" }, { "modality": "text", "value": "About 20" } ], "correctIndex": 3, "explanation": "Chinchilla's rule of thumb is about 20 training tokens per parameter — the 20-to-1 ratio. The other ratios are off by an order of magnitude or more.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "ekfkimt41iqu" }, { "id": "ot-train-chinchilla-apply", "shape": "mcq", "tags": [ "chinchilla", "scaling" ], "prompt": { "modality": "text", "value": "A team plans a 7-billion-parameter model. Applying Chinchilla's compute-optimal rule, roughly how many training tokens should they budget?" }, "options": [ { "modality": "text", "value": "About 7 billion" }, { "modality": "text", "value": "About 140 billion" }, { "modality": "text", "value": "About 1.4 trillion" }, { "modality": "text", "value": "About 35 billion" } ], "correctIndex": 1, "explanation": "Chinchilla calls for ~20 training tokens per parameter, so 7B parameters × 20 ≈ 140B tokens. One token per parameter (7B) or 200-to-1 (1.4T) both miss the 20-to-1 ratio.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1tvw5e51llta4z" }, { "id": "ot-train-pretraining-not", "shape": "mcq", "tags": [ "pretraining", "training" ], "prompt": { "modality": "text", "value": "Which statement about pretraining is NOT true?" }, "options": [ { "modality": "text", "value": "It is the first stage of building a model", "short": "First stage of building a model" }, { "modality": "text", "value": "It uses self-supervised next-token prediction", "short": "Self-supervised next-token" }, { "modality": "text", "value": "It trains the model on rankings of human preferences", "short": "Trains on preference rankings" }, { "modality": "text", "value": "It is the most expensive stage", "short": "The most expensive stage" } ], "correctIndex": 2, "explanation": "Training on human preferences is RLHF, not pretraining. Pretraining is the first and most expensive stage: self-supervised next-token prediction over a huge corpus.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "sc53uyryonem" }, { "id": "ot-usage-rag-apply", "shape": "mcq", "tags": [ "rag", "usage" ], "prompt": { "modality": "text", "value": "A support bot keeps confidently inventing product details that do not exist. Which technique most directly targets that failure?" }, "options": [ { "modality": "text", "value": "RAG" }, { "modality": "text", "value": "Few-shot prompting" }, { "modality": "text", "value": "Inference" }, { "modality": "text", "value": "Emergence" } ], "correctIndex": 0, "explanation": "RAG pulls relevant documents into the prompt so the model answers from grounded sources rather than memory — the main defense against hallucination. Few-shot adds task examples, which teaches format, not factual grounding.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "93hbp118l6nfb" }, { "id": "ot-usage-inference-def", "shape": "mcq", "tags": [ "inference", "usage" ], "prompt": { "modality": "text", "value": "Which term describes what happens every time you send a message — running the model with no weights changing?" }, "options": [ { "modality": "text", "value": "RLHF" }, { "modality": "text", "value": "Pretraining" }, { "modality": "text", "value": "Inference" }, { "modality": "text", "value": "Emergence" } ], "correctIndex": 2, "explanation": "Inference is running the frozen model; nothing about the weights changes. Pretraining and RLHF are training stages, where weights do change.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "19hwjgok61wg8" }, { "id": "ot-usage-hallucination-def", "shape": "mcq", "tags": [ "hallucination", "usage" ], "prompt": { "modality": "text", "value": "Which best describes a hallucination from an LLM?" }, "options": [ { "modality": "text", "value": "Fluent output that is confidently fabricated", "short": "Fluent, confidently fabricated" }, { "modality": "text", "value": "Output that repeats the prompt back verbatim", "short": "Repeats the prompt verbatim" }, { "modality": "text", "value": "Output the model declines to produce", "short": "Model declines to produce" }, { "modality": "text", "value": "Output cut off when the token limit is hit", "short": "Cut off at the token limit" } ], "correctIndex": 0, "explanation": "A hallucination is fluent, confident output that is simply fabricated — which is why RAG's grounding in retrieved sources is the main defense. Hitting a token limit is a context-window issue, not a hallucination.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "xojack15szs20" }, { "id": "ot-usage-fewshot-def", "shape": "mcq", "tags": [ "few-shot", "prompting" ], "prompt": { "modality": "text", "value": "You put a handful of worked examples in the prompt to teach the model a task on the fly. What is that called?" }, "options": [ { "modality": "text", "value": "RAG" }, { "modality": "text", "value": "Emergence" }, { "modality": "text", "value": "Few-shot prompting" }, { "modality": "text", "value": "Pretraining" } ], "correctIndex": 2, "explanation": "Few-shot prompting means giving examples in the prompt to teach a task on the fly. RAG also adds text to the prompt, but it adds retrieved source documents for grounding, not task examples.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "cn6a7vahzlgh" }, { "id": "ot-usage-emergence-def", "shape": "mcq", "tags": [ "emergence", "scaling" ], "prompt": { "modality": "text", "value": "Which term names capabilities that appear only past a certain model scale?" }, "options": [ { "modality": "text", "value": "Inference" }, { "modality": "text", "value": "Emergence" }, { "modality": "text", "value": "Hallucination" }, { "modality": "text", "value": "Few-shot" } ], "correctIndex": 1, "explanation": "Emergence is the term for capabilities that show up only once a model passes a certain scale. Inference is just running the frozen model, regardless of its size.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "ivozsonaxmis" }, { "id": "ot-godfathers-hinton", "shape": "mcq", "tags": [ "turing-award", "people" ], "prompt": { "modality": "text", "value": "Which of the three 2018 Turing Award winners is a British-Canadian cognitive psychologist and computer scientist?" }, "options": [ { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Yoshua Bengio" }, { "modality": "text", "value": "Ilya Sutskever" }, { "modality": "text", "value": "Yann LeCun" } ], "correctIndex": 0, "explanation": "Hinton is British-Canadian and trained as a cognitive psychologist; his neural-network research underpins modern LLMs. Bengio is Canadian and LeCun is French-American, and Sutskever was not a Turing Award recipient.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "50dedo1x5gqww" }, { "id": "ot-godfathers-lecun", "shape": "mcq", "tags": [ "turing-award", "cnn" ], "prompt": { "modality": "text", "value": "Which 2018 Turing Award recipient is a founding figure of convolutional neural networks, crucial for image processing?" }, "options": [ { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Andrew Ng" }, { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Yoshua Bengio" } ], "correctIndex": 0, "explanation": "LeCun, a French-American computer scientist, is one of the founding figures of convolutional neural networks. Hinton and Bengio shared the same award but for other deep-learning contributions; Ng did not receive it.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1mtq1r5fdjq2b" }, { "id": "ot-godfathers-bengio", "shape": "mcq", "tags": [ "turing-award", "attention" ], "prompt": { "modality": "text", "value": "Which Turing Award 'godfather' led the group that introduced influential neural language models and attention mechanisms?" }, "options": [ { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Yoshua Bengio" }, { "modality": "text", "value": "Alec Radford" }, { "modality": "text", "value": "Yann LeCun" } ], "correctIndex": 1, "explanation": "Bengio, a Canadian computer scientist, led the group that introduced influential neural language models and attention mechanisms. LeCun is associated with convolutional networks, and Radford was never a Turing laureate.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "adseg2qqa9zm" }, { "id": "ot-godfathers-not", "shape": "mcq", "tags": [ "turing-award", "people" ], "prompt": { "modality": "text", "value": "Which researcher is NOT one of the three who shared the 2018 Turing Award for deep learning?" }, "options": [ { "modality": "text", "value": "Andrew Ng" }, { "modality": "text", "value": "Yoshua Bengio" }, { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Geoffrey Hinton" } ], "correctIndex": 0, "explanation": "The 2018 Turing Award went jointly to Bengio, Hinton, and LeCun. Andrew Ng is influential in AI as a Google Brain co-founder and educator, but he was not among the three recipients.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "jit34ny1v8mh" }, { "id": "ot-openai-sutskever-ssi", "shape": "mcq", "tags": [ "openai", "people" ], "prompt": { "modality": "text", "value": "Who co-founded OpenAI and served as its chief scientist before leaving to found Safe Superintelligence?" }, "options": [ { "modality": "text", "value": "Alec Radford" }, { "modality": "text", "value": "Andrew Ng" }, { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Ilya Sutskever" } ], "correctIndex": 3, "explanation": "Sutskever co-founded OpenAI, was its chief scientist, and later left to found Safe Superintelligence. Radford also worked at OpenAI but is known for the original GPT model rather than co-founding the lab.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1mhq51y5qfi12" }, { "id": "ot-openai-radford-gpt", "shape": "mcq", "tags": [ "openai", "gpt" ], "prompt": { "modality": "text", "value": "Alec Radford is best known for which contribution?" }, "options": [ { "modality": "text", "value": "Founding Safe Superintelligence", "short": "Founding Safe Superintelligence" }, { "modality": "text", "value": "Co-founding Google Brain", "short": "Co-founding Google Brain" }, { "modality": "text", "value": "The original GPT model and its successors", "short": "Original GPT and successors" }, { "modality": "text", "value": "The word2vec embedding technique", "short": "word2vec embedding technique" } ], "correctIndex": 2, "explanation": "Radford, an AI researcher at OpenAI focused on natural language processing, was a key figure behind the original GPT model and its successors. Google Brain traces to Ng, word2vec to Mikolov's team, and Safe Superintelligence to Sutskever.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "f1j2hbl3vj4h" }, { "id": "ot-openai-ng-brain", "shape": "mcq", "tags": [ "people", "google-brain" ], "prompt": { "modality": "text", "value": "Andrew Ng co-founded which AI research organization?" }, "options": [ { "modality": "text", "value": "OpenAI" }, { "modality": "text", "value": "Google Brain" }, { "modality": "text", "value": "Safe Superintelligence" }, { "modality": "text", "value": "DeepMind" } ], "correctIndex": 1, "explanation": "Ng co-founded Google Brain. OpenAI was co-founded by Sutskever among others, and Safe Superintelligence is the lab Sutskever founded after leaving OpenAI.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1j2u7qa1fng4pi" }, { "id": "ot-openai-ng-coursera", "shape": "mcq", "tags": [ "people", "education" ], "prompt": { "modality": "text", "value": "Andrew Ng significantly impacted online education through which platform?" }, "options": [ { "modality": "text", "value": "Udacity" }, { "modality": "text", "value": "Coursera" }, { "modality": "text", "value": "edX" }, { "modality": "text", "value": "Khan Academy" } ], "correctIndex": 1, "explanation": "Ng co-founded Coursera and taught many foundational AI courses there. The other platforms are real online-education services but are not the one credited to him in the guide.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "k8cls81m92og4" }, { "id": "ot-openai-sutskever-seq2seq", "shape": "mcq", "tags": [ "openai", "seq2seq" ], "prompt": { "modality": "text", "value": "Ilya Sutskever played a significant role in developing which of the following?" }, "options": [ { "modality": "text", "value": "The sequence-to-sequence model and attention mechanisms", "short": "Seq2seq model and attention" }, { "modality": "text", "value": "The word2vec embedding technique", "short": "word2vec embedding technique" }, { "modality": "text", "value": "The LSTM recurrent architecture", "short": "LSTM recurrent architecture" }, { "modality": "text", "value": "Convolutional neural networks for image processing", "short": "CNNs for image processing" } ], "correctIndex": 0, "explanation": "Sutskever helped develop the sequence-to-sequence model and attention mechanisms, plus generative models like the GPT series. Convolutional networks trace to LeCun and word2vec to Mikolov's team.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "2wbh301kxxkt0" }, { "id": "ot-found-word2vec-def", "shape": "mcq", "tags": [ "word2vec", "embeddings" ], "prompt": { "modality": "text", "value": "Which technique represents words as vectors in a high-dimensional space, capturing semantic relationships through vector arithmetic?" }, "options": [ { "modality": "text", "value": "BERT" }, { "modality": "text", "value": "word2vec" }, { "modality": "text", "value": "Transformer" }, { "modality": "text", "value": "LSTM" } ], "correctIndex": 1, "explanation": "word2vec represents words as high-dimensional vectors whose arithmetic captures semantic relationships. LSTM is a recurrent architecture and the Transformer is a parallel sequence architecture, not embedding techniques.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1y6v0gqh4ux1i" }, { "id": "ot-found-word2vec-origin", "shape": "mcq", "tags": [ "word2vec", "attribution" ], "prompt": { "modality": "text", "value": "word2vec was introduced by a team led by which researcher, and in what year?" }, "options": [ { "modality": "text", "value": "Geoffrey Hinton, in 1990", "short": "Geoffrey Hinton, in 1990" }, { "modality": "text", "value": "Vaswani et al., in 2017", "short": "Vaswani et al., in 2017" }, { "modality": "text", "value": "Alec Radford, in 2018", "short": "Alec Radford, in 2018" }, { "modality": "text", "value": "Tomas Mikolov, in 2013", "short": "Tomas Mikolov, in 2013" } ], "correctIndex": 3, "explanation": "word2vec was introduced in 2013 by a team led by Tomas Mikolov. Vaswani et al. in 2017 introduced the Transformer instead, a different milestone.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1ndoa2it3g26u" }, { "id": "ot-found-transformer-attn", "shape": "mcq", "tags": [ "transformer", "attention" ], "prompt": { "modality": "text", "value": "The Transformer architecture replaces recurrent layers with what?" }, "options": [ { "modality": "text", "value": "Word-vector arithmetic", "short": "Word-vector arithmetic" }, { "modality": "text", "value": "Convolutional layers", "short": "Convolutional layers" }, { "modality": "text", "value": "Self-attention mechanisms", "short": "Self-attention mechanisms" }, { "modality": "text", "value": "LSTM memory gates", "short": "LSTM memory gates" } ], "correctIndex": 2, "explanation": "Introduced in the 2017 paper 'Attention Is All You Need', the Transformer swaps recurrent layers for self-attention. LSTM gates are exactly the recurrent machinery it replaces, not what replaced it.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "wg2o2918et763" }, { "id": "ot-found-lstm-gradient", "shape": "mcq", "tags": [ "lstm", "rnn" ], "prompt": { "modality": "text", "value": "LSTM was designed to address which weakness of standard RNNs?" }, "options": [ { "modality": "text", "value": "The curse of dimensionality", "short": "The curse of dimensionality" }, { "modality": "text", "value": "The cost of self-attention", "short": "The cost of self-attention" }, { "modality": "text", "value": "Overfitting on small datasets", "short": "Overfitting on small datasets" }, { "modality": "text", "value": "The vanishing gradient problem", "short": "The vanishing gradient problem" } ], "correctIndex": 3, "explanation": "LSTMs remember information over long sequences by addressing the vanishing gradient problem of standard RNNs. Self-attention cost belongs to Transformers, an architecture that came much later.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "167kt9dm8lhb7" }, { "id": "ot-found-lstm-era", "shape": "mcq", "tags": [ "lstm", "timeline" ], "prompt": { "modality": "text", "value": "LSTM was developed in which decade, long before it became popular for NLP tasks around 2014?" }, "options": [ { "modality": "text", "value": "The 1970s" }, { "modality": "text", "value": "The 2000s" }, { "modality": "text", "value": "The 1990s" }, { "modality": "text", "value": "The 2010s" } ], "correctIndex": 2, "explanation": "LSTM was developed in the 1990s and only became popular for NLP tasks around 2014. The 2010s is tempting because of that popularity date, but the architecture itself is far older.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "3puthj2nh62l" }, { "id": "ot-models-bert-training", "shape": "mcq", "tags": [ "bert", "training" ], "prompt": { "modality": "text", "value": "How does BERT learn to understand the context of words in a sentence?" }, "options": [ { "modality": "text", "value": "Left-to-right next-word prediction", "short": "Left-to-right next-word pred" }, { "modality": "text", "value": "Vector arithmetic over word embeddings", "short": "Vector arithmetic on embeddings" }, { "modality": "text", "value": "Reinforcement learning from human feedback", "short": "RL from human feedback" }, { "modality": "text", "value": "Bidirectional training to predict masked words", "short": "Bidirectional masked-word pred" } ], "correctIndex": 3, "explanation": "BERT, released by Google in October 2018, uses bidirectional training to predict masked words, which is how it captures sentence context. Reinforcement learning from human feedback is ChatGPT's technique, not BERT's.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "xwks3b17llwl5" }, { "id": "ot-models-gpt3-fewshot", "shape": "mcq", "tags": [ "gpt-3", "few-shot" ], "prompt": { "modality": "text", "value": "Which capability is GPT-3 best known for?" }, "options": [ { "modality": "text", "value": "Capturing semantics through vector arithmetic", "short": "Semantics via vector arithmetic" }, { "modality": "text", "value": "Bidirectional prediction of masked words", "short": "Bidirectional masked-word pred" }, { "modality": "text", "value": "Learning from human feedback on its responses", "short": "Learning from human feedback" }, { "modality": "text", "value": "Few-shot learning — performing tasks without explicit training", "short": "Few-shot learning (no training)" } ], "correctIndex": 3, "explanation": "GPT-3, released by OpenAI in 2020, is known for few-shot learning: it can perform tasks without explicit training. Masked-word prediction is BERT's approach and human-feedback learning is ChatGPT's.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "f7x9qtl3guo3" }, { "id": "ot-models-chatgpt-rlhf", "shape": "mcq", "tags": [ "chatgpt", "rlhf" ], "prompt": { "modality": "text", "value": "Which technique does ChatGPT use to improve its responses?" }, "options": [ { "modality": "text", "value": "Convolutional filters over text", "short": "Conv filters over text" }, { "modality": "text", "value": "Word-vector arithmetic", "short": "Word-vector arithmetic" }, { "modality": "text", "value": "Reinforcement learning from human feedback", "short": "RL from human feedback" }, { "modality": "text", "value": "Bidirectional masked-word pretraining", "short": "Bidir. masked pretraining" } ], "correctIndex": 2, "explanation": "ChatGPT, launched by OpenAI in November 2022, uses reinforcement learning from human feedback to improve responses. Bidirectional masked-word training is BERT's approach instead.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "kvb93n1sxfj1l" }, { "id": "ot-models-recency", "shape": "mcq", "tags": [ "timeline", "models" ], "prompt": { "modality": "text", "value": "Which of these was released most recently?" }, "options": [ { "modality": "text", "value": "word2vec" }, { "modality": "text", "value": "ChatGPT" }, { "modality": "text", "value": "GPT-3" }, { "modality": "text", "value": "BERT" } ], "correctIndex": 1, "explanation": "ChatGPT launched in November 2022, after GPT-3 (2020), BERT (October 2018), and word2vec (2013). GPT-3 is the closest tempting answer but predates ChatGPT by two years.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1sq8crg1kwvzb4" }, { "id": "fi-ot-pre-transformer-3", "tags": [ "transformer", "attention" ], "title": "Attention replaces recurrence", "body": "The 2017 Transformer paper by Vaswani et al. proved that recurrence could be discarded entirely — a model built from attention alone was enough.", "shape": "fact", "illustration": { "imageSearchTerm": "attention mechanism replaces recurrence", "imagePrompt": "An abstract web of glowing lines connecting many points directly to one another at once, replacing a single looping chain, representing direct attention instead of sequential steps.", "alt": "Attention replaces recurrence", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-pre-transformer-3.webp", "credit": "AI-generated (gpt-image-1.5)" }, "uid": "1d95muc1d4n09g" }, { "id": "fi-ot-scaling-1", "tags": [ "scaling-laws", "kaplan" ], "title": "Kaplan's scaling laws", "body": "Kaplan et al.'s 2020 scaling laws showed that loss falls predictably as a power function of compute, data, and parameters — a quantitative map that justified ever-larger models.", "shape": "fact", "illustration": { "imageSearchTerm": "compute data parameter scaling", "imagePrompt": "A smooth curved line descending steadily and predictably across an open plane, representing loss falling in a consistent, foreseeable pattern.", "alt": "Kaplan's scaling laws", "depictable": false, "credit": "AI-generated (gpt-image-1.5)", "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-scaling-1.webp" }, "uid": "j9l6x91h8yfmv" }, { "id": "fi-ot-generation-loop-1", "tags": [ "next-token", "training" ], "title": "Next-token prediction", "body": "The core skill an LLM learns in training is next-token prediction: given all previous tokens, predict the single most likely next one. Its broader language ability follows from that one skill.", "shape": "fact", "illustration": { "imageSearchTerm": "next token prediction sequence", "imagePrompt": "A single glowing puzzle piece being chosen from several floating candidates to connect onto the end of a growing chain.", "alt": "Next-token prediction", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-generation-loop-1.webp", "credit": "AI-generated (gpt-image-1.5)" }, "uid": "15e8qxu6m0muu" }, { "id": "fi-ot-generation-loop-3", "tags": [ "autoregressive", "error-compounding" ], "title": "Why early errors compound", "body": "A model can't go back and edit its own output — it commits to each token in sequence, so a small factual slip early in an answer compounds as later tokens build on the mistake.", "shape": "fact", "illustration": { "imageSearchTerm": "sequential error compounding", "imagePrompt": "A row of dominoes falling in sequence, where one slightly misaligned domino near the start causes the rest of the chain to tip increasingly off course.", "alt": "Why early errors compound", "depictable": false, "credit": "AI-generated (gpt-image-1.5)", "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-generation-loop-3.webp" }, "uid": "1ue8o571pt71lx" }, { "id": "fi-ot-sampling-1", "tags": [ "logits", "sampling" ], "title": "What logits are", "body": "Logits are the raw, unnormalized scores an LLM assigns to every possible next token, before those scores are converted into probabilities.", "shape": "fact", "illustration": { "imageSearchTerm": "raw token probability scores", "imagePrompt": "A cluster of glowing bars of uneven heights floating side by side, not yet settled into a smooth, balanced arrangement.", "alt": "What logits are", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-sampling-1.webp", "credit": "AI-generated (gpt-image-1.5)" }, "uid": "1cj202sjlgwok" }, { "id": "fi-ot-sampling-3", "tags": [ "temperature", "sampling" ], "title": "Low temperature, focused output", "body": "Setting an LLM's temperature low makes it favor the safe, high-probability token, producing output that is focused and as deterministic as possible.", "shape": "fact", "illustration": { "imageSearchTerm": "low temperature focused sampling", "imagePrompt": "A single tight, focused beam of light shining directly on one point, in contrast to a wide, scattered spray of light nearby.", "alt": "Low temperature, focused output", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-sampling-3.webp", "credit": "AI-generated (gpt-image-1.5)" }, "uid": "1sbp3fyeesamu" }, { "id": "fi-ot-rlhf-4", "tags": [ "constitutional-ai", "alignment" ], "title": "Constitutional AI's self-critique", "body": "In Anthropic's Constitutional AI, the model critiques and revises its own outputs against a written set of principles, then trains on those AI-generated preferences — reducing reliance on human harm-labeling.", "shape": "fact", "illustration": { "imageSearchTerm": "model self critique revision", "imagePrompt": "A hand holding a red pen, marking corrections on a printed page beside an earlier draft of the same document, representing self-review and revision.", "alt": "Constitutional AI's self-critique", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-rlhf-4.webp", "credit": "AI-generated (gpt-image-1.5)" }, "uid": "10pnwq6ym61ky" }, { "id": "fi-ot-rlhf-5", "tags": [ "constitutional-ai", "auditability" ], "title": "Constitutional AI's auditability payoff", "body": "Constitutional AI's written constitution gives explicit, auditable guiding principles instead of preferences buried in opaque data — on top of reducing reliance on human harm-labeling.", "shape": "fact", "illustration": { "imageSearchTerm": "written constitution auditable principles", "imagePrompt": "An open book lying beside a locked, opaque black box, representing transparent guiding principles instead of hidden internal preferences.", "alt": "Constitutional AI's auditability payoff", "depictable": false, "credit": "AI-generated (FlashFeed, model unrecorded)", "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-rlhf-5.webp" }, "uid": "rwf1781vvd904" }, { "id": "fi-ot-instruct-1", "tags": [ "instruct-model", "pipeline" ], "title": "Base model plus alignment", "body": "An instruct or chat model is produced by taking a pretrained base model and adding supervised fine-tuning plus RLHF, or an equivalent alignment step.", "shape": "fact", "illustration": { "imageSearchTerm": "base model plus alignment", "imagePrompt": "A plain block of raw stone being carved and polished in progressive stages into a refined sculpture, representing a base form reshaped through additional steps.", "alt": "Base model plus alignment", "depictable": false }, "uid": "1hoomo81jboh0s" }, { "id": "fi-ot-infer-2", "tags": [ "cost", "training-vs-inference" ], "title": "Why fine-tuning costs millions", "body": "Fine-tuning is expensive because training runs on massive GPU clusters for weeks or months at a cost of millions, while querying an API afterward is cheap per call.", "shape": "fact", "illustration": { "imageSearchTerm": "data center server racks", "imagePrompt": "A data center hallway lined with thousands of glowing server racks stretching into the distance, representing the infrastructure needed for large-scale training.", "alt": "Why fine-tuning costs millions", "depictable": true, "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-infer-2.webp", "credit": "Pexels · panumas nikhomkhai", "creditUrl": "https://www.pexels.com/photo/data-center-server-racks-with-active-equipment-37730212/" }, "uid": "qv7px0zv09ns" }, { "id": "fi-ot-rag-2", "tags": [ "vector-database", "embeddings" ], "title": "Vector databases, explained", "body": "A vector database stores document embeddings and finds relevant material through fast similarity search — matching by meaning, not exact keyword matching.", "shape": "fact", "illustration": { "imageSearchTerm": "embedding similarity vector search", "imagePrompt": "A three-dimensional cloud of glowing points in space, with similar points clustered closely together and dissimilar points scattered farther apart.", "alt": "Vector databases, explained", "depictable": false, "credit": "Unsplash · Logan Voss · Unsplash License", "creditUrl": "https://unsplash.com/photos/a-group-of-colorful-pieces-of-paper-on-a-white-background-a4YX91-Dmbs", "subject": "Three separate clusters of overlapping jagged, gradient-coloured paper-like shapes in blue, red, yellow and white on a plain white ground; no text", "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-rag-2.webp" }, "uid": "18h81emx6hyka" }, { "id": "fi-ot-rag-3", "tags": [ "chunking", "rag" ], "title": "Why RAG chunks documents", "body": "Source documents are split into smaller segments before being embedded for RAG so the retrieved passages fit the context window and stay semantically focused.", "shape": "fact", "illustration": { "imageSearchTerm": "document chunking context window", "imagePrompt": "A long scroll of paper being carefully cut into several smaller, evenly sized segments, representing a large document divided into focused, manageable pieces.", "alt": "Why RAG chunks documents", "depictable": false }, "uid": "138zq3j1sweok9" }, { "id": "fi-mcq-rw-l4-p2-rag", "tags": [ "agents", "tool-use" ], "title": "The agent planning loop", "body": "An LLM agent differs from a single-prompt interaction by running a planning loop: it acts as a controller that selects tools, takes action, and observes results until the goal is met.", "shape": "fact", "illustration": { "imageSearchTerm": "agent controller tool loop", "imagePrompt": "A robotic arm repeatedly selecting a different tool from a rack, using it, then pausing to check its work before reaching for the next one, in a continuous loop.", "alt": "The agent planning loop", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/def-agent.jpg", "credit": "Pexels · Kindel Media", "creditUrl": "https://www.pexels.com/photo/white-robot-action-figure-on-blue-string-lights-8566470/" }, "uid": "1tjzfe460c3vg" }, { "id": "fi-ot-tools-1", "tags": [ "function-calling", "tool-use" ], "title": "What function calling produces", "body": "In function calling, the model itself produces a structured, well-formed call — it's the host application that actually runs that call against the real API or tool.", "shape": "fact", "illustration": { "imageSearchTerm": "structured function call execution", "imagePrompt": "A hand filling out a detailed request form and passing it to a separate machine operator, who is the one who actually pulls the lever to run the machine.", "alt": "What function calling produces", "depictable": false, "credit": "Pexels · Daniil Komov · Pexels License", "creditUrl": "https://www.pexels.com/photo/close-up-of-programming-code-on-computer-screen-34803988/", "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-tools-1.webp" }, "uid": "1qt3s65txg2oj" }, { "id": "fi-mcq-rw-l8-p2-knowledge-code", "tags": [ "rag", "enterprise" ], "title": "RAG Beats Fine-Tuning", "body": "RAG grounds an LLM in private company knowledge more cheaply than fine-tuning, lets the knowledge base update without retraining, and keeps proprietary data out of the model's weights.", "shape": "fact", "illustration": { "imageSearchTerm": "private knowledge base retrieval", "imagePrompt": "A librarian retrieving one specific book from open shelves to answer a question, rather than a scholar who had to memorize an entire library in advance.", "alt": "RAG Beats Fine-Tuning", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/def-rag.jpg", "credit": "Pexels · panumas nikhomkhai", "creditUrl": "https://www.pexels.com/photo/close-up-of-a-blue-server-rack-in-datacenter-37730211/" }, "uid": "sqgeen1x3omx9" }, { "id": "fi-ot-kc-semantic-search", "tags": [ "semantic-search", "embeddings" ], "title": "Embeddings Search By Meaning", "body": "Semantic search retrieves passages by meaning using embeddings, so a query like \"contract renewals\" can surface documents that never use that phrase — though it weakens on niche jargon the embedding model doesn't cover.", "shape": "fact", "illustration": { "imageSearchTerm": "semantic meaning based retrieval", "imagePrompt": "A search beam sweeping across a shelf of unlabeled folders and highlighting several related ones scattered among the rest, representing retrieval by meaning rather than exact wording.", "alt": "Embeddings Search By Meaning", "depictable": false }, "uid": "1l19j1wx45drg" }, { "id": "fi-ot-kc-test-writing-catch", "tags": [ "testing", "coding" ], "title": "Where AI Testing Falls Short", "body": "LLMs write unit tests that cover happy paths and common edge cases well, but they don't know your domain's invariants or business rules, so the edge cases unique to your system still need a human to catch them.", "shape": "fact", "illustration": { "imageSearchTerm": "unit test edge cases", "imagePrompt": "A robotic inspector checking most items on a conveyor belt, while a human inspector leans in to catch one unusual item further down the line that the machine passed over.", "alt": "Where AI Testing Falls Short", "depictable": false }, "uid": "ws8xij1685okh" }, { "id": "fi-ot-cop-definition", "tags": [ "copilot-pattern", "definition" ], "title": "The Copilot Pattern", "body": "In the copilot pattern, the LLM suggests, drafts, or assists, but a human must approve before anything takes effect — it accelerates work without ever finalizing it on its own.", "shape": "fact", "illustration": { "imageSearchTerm": "human approval before action", "imagePrompt": "A co-pilot in an aircraft cockpit adjusting controls while the pilot in the next seat keeps a hand ready near the final switch before any change takes effect.", "alt": "The Copilot Pattern", "depictable": false, "credit": "Pexels · Donald Tong · Pexels License", "creditUrl": "https://www.pexels.com/photo/person-doing-thumbs-up-193821/", "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-ot-cop-definition.webp" }, "uid": "1p2oqum10pg6du" }, { "id": "fi-mcq-rw-l9-p1-reliability", "tags": [ "hallucination", "reliability" ], "title": "Hallucination Isn't A Glitch", "body": "LLMs hallucinate because they're trained to predict plausible-sounding text, not to verify it — there's no built-in mechanism checking whether an output is actually factually grounded before it's produced.", "shape": "fact", "illustration": { "imageSearchTerm": "plausible text no verification", "imagePrompt": "A storyteller speaking fluently and confidently to a captivated audience, with no one present to check any of the claims being made.", "alt": "Hallucination Isn't A Glitch", "depictable": false, "credit": "Pexels · Nataliya Vaitkevich · Pexels License", "creditUrl": "https://www.pexels.com/photo/vacation-smartphone-travelling-technology-8830663/", "url": "https://cdn.recurxive.com/packs/llm-101/images/fi-mcq-rw-l9-p1-reliability.webp" }, "uid": "1ju2mxp38a3mr" }, { "id": "fi-ot-sec-injection-1", "tags": [ "prompt-injection", "owasp" ], "title": "When Input Overrides Instructions", "body": "Prompt injection is malicious text hidden in user input or a retrieved document that overrides the system prompt, steering the model to act against its original instructions.", "shape": "fact", "illustration": { "imageSearchTerm": "hidden malicious instruction override", "imagePrompt": "A hidden note slipped secretly inside the pages of an official instruction manual, a hand inserting it among the legitimate pages.", "alt": "When Input Overrides Instructions", "depictable": false, "url": "https://cdn.recurxive.com/packs/llm-101/x-lim-injection.jpg", "credit": "Openverse · Visual Content · CC BY 2.0", "creditUrl": "https://www.flickr.com/photos/143601516@N03/29723649810" }, "uid": "fl7pp168g0tj" }, { "id": "ot-cov-openai-figures-1", "shape": "mcq", "tags": [ "openai", "researchers" ], "prompt": { "modality": "text", "value": "Which OpenAI co-founder and former chief scientist left to found Safe Superintelligence?" }, "options": [ { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Ilya Sutskever" }, { "modality": "text", "value": "Andrew Ng" }, { "modality": "text", "value": "Alec Radford" } ], "correctIndex": 1, "explanation": "Ilya Sutskever co-founded and served as chief scientist of OpenAI before leaving to found Safe Superintelligence.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1y3pif1t6pm8d" }, { "id": "ot-cov-openai-figures-2", "shape": "mcq", "tags": [ "openai", "researchers", "llm-history" ], "prompt": { "modality": "text", "value": "According to the guide, who helped develop sequence-to-sequence and attention methods and contributed to the GPT series?" }, "options": [ { "modality": "text", "value": "Yoshua Bengio" }, { "modality": "text", "value": "Alec Radford" }, { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Ilya Sutskever" } ], "correctIndex": 3, "explanation": "The guide credits Ilya Sutskever with the sequence-to-sequence model, attention mechanisms, and contributions to the GPT series.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "p3egk41jzbrdk" }, { "id": "ot-cov-openai-figures-3", "shape": "mcq", "tags": [ "openai", "researchers" ], "prompt": { "modality": "text", "value": "Which AI researcher was a key figure in developing the original GPT model at OpenAI?" }, "options": [ { "modality": "text", "value": "Alec Radford" }, { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Ilya Sutskever" }, { "modality": "text", "value": "Andrew Ng" } ], "correctIndex": 0, "explanation": "The guide identifies Alec Radford as a key figure behind the original GPT model and its successors at OpenAI.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "ngstfcpbitpc" }, { "id": "ot-cov-openai-figures-4", "shape": "mcq", "tags": [ "openai", "researchers", "nlp" ], "prompt": { "modality": "text", "value": "According to the guide, whose OpenAI research centered on NLP and generative models?" }, "options": [ { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Alec Radford" }, { "modality": "text", "value": "Andrew Ng" }, { "modality": "text", "value": "Yoshua Bengio" } ], "correctIndex": 1, "explanation": "The guide describes Alec Radford's work at OpenAI as focused on natural language processing and generative models.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "uopajv1mj5ifp" }, { "id": "ot-cov-openai-figures-5", "shape": "mcq", "tags": [ "education", "researchers" ], "prompt": { "modality": "text", "value": "Which computer scientist co-founded Google Brain and had a major impact on AI education via Coursera?" }, "options": [ { "modality": "text", "value": "Alec Radford" }, { "modality": "text", "value": "Ilya Sutskever" }, { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Andrew Ng" } ], "correctIndex": 3, "explanation": "The guide credits Andrew Ng with co-founding Google Brain and shaping online AI education through Coursera.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "4g1bp2fs01lu" }, { "id": "ot-cov-openai-figures-6", "shape": "mcq", "tags": [ "education", "researchers" ], "prompt": { "modality": "text", "value": "According to the guide, which AI figure taught many foundational courses while also working on deep learning?" }, "options": [ { "modality": "text", "value": "Ilya Sutskever" }, { "modality": "text", "value": "Alec Radford" }, { "modality": "text", "value": "Andrew Ng" }, { "modality": "text", "value": "Geoffrey Hinton" } ], "correctIndex": 2, "explanation": "The guide describes Andrew Ng as an entrepreneur and educator who taught foundational AI courses and worked on deep learning.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "lr9sw010dg36o" }, { "id": "ot-cov-foundational-things-1", "shape": "mcq", "tags": [ "nlp", "foundations" ], "prompt": { "modality": "text", "value": "Which technique represents words as vectors, capturing semantic relationships through vector arithmetic?" }, "options": [ { "modality": "text", "value": "BERT" }, { "modality": "text", "value": "LSTM" }, { "modality": "text", "value": "Transformer" }, { "modality": "text", "value": "word2vec" } ], "correctIndex": 3, "explanation": "word2vec represents words as vectors in a high-dimensional space, capturing semantic relationships via vector arithmetic.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1apcnoqvvni42" }, { "id": "ot-cov-foundational-things-2", "shape": "mcq", "tags": [ "nlp", "foundations", "llm-history" ], "prompt": { "modality": "text", "value": "According to the guide, word2vec was introduced in 2013 by a team led by whom?" }, "options": [ { "modality": "text", "value": "Alec Radford" }, { "modality": "text", "value": "Andrew Ng" }, { "modality": "text", "value": "Tomas Mikolov" }, { "modality": "text", "value": "Ashish Vaswani" } ], "correctIndex": 2, "explanation": "The guide states word2vec was introduced by a team led by Tomas Mikolov in 2013.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "136inny1x4r4ri" }, { "id": "ot-cov-foundational-things-3", "shape": "mcq", "tags": [ "foundations", "architectures" ], "prompt": { "modality": "text", "value": "Which architecture is a recurrent neural network designed to remember information over long sequences?" }, "options": [ { "modality": "text", "value": "BERT" }, { "modality": "text", "value": "word2vec" }, { "modality": "text", "value": "LSTM" }, { "modality": "text", "value": "Transformer" } ], "correctIndex": 2, "explanation": "LSTM is a type of recurrent neural network built to remember information across long sequences.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "i10srd1ct88qf" }, { "id": "ot-cov-foundational-things-4", "shape": "mcq", "tags": [ "foundations", "architectures" ], "prompt": { "modality": "text", "value": "According to the guide, LSTM was developed to address which problem in standard RNNs?" }, "options": [ { "modality": "text", "value": "The vanishing gradient problem" }, { "modality": "text", "value": "Insufficient vector dimensionality" }, { "modality": "text", "value": "Lack of parallel processing" }, { "modality": "text", "value": "Overfitting on small datasets" } ], "correctIndex": 0, "explanation": "The guide states LSTM addresses the vanishing gradient problem found in standard RNNs.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1fxoujf1uq0mh5" }, { "id": "ot-cov-foundational-things-5", "shape": "mcq", "tags": [ "architectures", "foundations" ], "prompt": { "modality": "text", "value": "Which architecture replaces recurrent layers with self-attention to process sequences in parallel?" }, "options": [ { "modality": "text", "value": "LSTM" }, { "modality": "text", "value": "BERT" }, { "modality": "text", "value": "word2vec" }, { "modality": "text", "value": "Transformer" } ], "correctIndex": 3, "explanation": "The Transformer architecture processes sequences in parallel using self-attention instead of recurrent layers.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "uh1mr1a4lgzh" }, { "id": "ot-cov-foundational-things-6", "shape": "mcq", "tags": [ "architectures", "llm-history" ], "prompt": { "modality": "text", "value": "According to the guide, the Transformer was introduced in which 2017 paper?" }, "options": [ { "modality": "text", "value": "Neural Machine Translation by Alignment" }, { "modality": "text", "value": "BERT: Pre-training of Deep Transformers" }, { "modality": "text", "value": "Attention Is All You Need" }, { "modality": "text", "value": "Sequence to Sequence Learning" } ], "correctIndex": 2, "explanation": "The guide states the Transformer was introduced in the 2017 paper 'Attention Is All You Need' by Vaswani et al.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1j7r35k1jbh6eg" }, { "id": "ot-cov-landmark-models-1", "shape": "mcq", "tags": [ "nlp", "landmark-models" ], "prompt": { "modality": "text", "value": "Which model uses bidirectional training to predict masked words in a sentence?" }, "options": [ { "modality": "text", "value": "GPT-3" }, { "modality": "text", "value": "LSTM" }, { "modality": "text", "value": "ChatGPT" }, { "modality": "text", "value": "BERT" } ], "correctIndex": 3, "explanation": "BERT uses bidirectional training to predict masked words and understand word context.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "j8v5z2gbgj1y" }, { "id": "ot-cov-landmark-models-2", "shape": "mcq", "tags": [ "nlp", "llm-history" ], "prompt": { "modality": "text", "value": "According to the guide, which company released BERT in October 2018?" }, "options": [ { "modality": "text", "value": "Google" }, { "modality": "text", "value": "Meta" }, { "modality": "text", "value": "Microsoft" }, { "modality": "text", "value": "OpenAI" } ], "correctIndex": 0, "explanation": "The guide states BERT was released by Google in October 2018 and set new NLP benchmarks.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "16f8maq113h8pi" }, { "id": "ot-cov-landmark-models-3", "shape": "mcq", "tags": [ "landmark-models", "openai" ], "prompt": { "modality": "text", "value": "Which AI model uses reinforcement learning from human feedback to improve conversational responses?" }, "options": [ { "modality": "text", "value": "GPT-3" }, { "modality": "text", "value": "word2vec" }, { "modality": "text", "value": "ChatGPT" }, { "modality": "text", "value": "BERT" } ], "correctIndex": 2, "explanation": "ChatGPT uses reinforcement learning from human feedback (RLHF) to improve its conversational responses.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "onltw317f0x7h" }, { "id": "ot-cov-landmark-models-4", "shape": "mcq", "tags": [ "openai", "llm-history" ], "prompt": { "modality": "text", "value": "According to the guide, in what month and year did OpenAI launch ChatGPT?" }, "options": [ { "modality": "text", "value": "January 2023" }, { "modality": "text", "value": "October 2018" }, { "modality": "text", "value": "November 2022" }, { "modality": "text", "value": "July 2020" } ], "correctIndex": 2, "explanation": "The guide states OpenAI launched ChatGPT in November 2022, and it quickly gained widespread use.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "ku507o1csjyn0" }, { "id": "ot-cov-frontier-labs-1", "shape": "mcq", "tags": [ "companies", "frontier-labs" ], "prompt": { "modality": "text", "value": "Which US frontier lab, founded by Elon Musk in 2023, makes the Grok model family?" }, "options": [ { "modality": "text", "value": "Anthropic" }, { "modality": "text", "value": "OpenAI" }, { "modality": "text", "value": "xAI" }, { "modality": "text", "value": "Meta AI" } ], "correctIndex": 2, "explanation": "xAI, founded by Elon Musk in 2023, makes the Grok model family and is integrated with X.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "9wl1h0zv6r08" }, { "id": "ot-cov-frontier-labs-2", "shape": "mcq", "tags": [ "companies", "frontier-labs" ], "prompt": { "modality": "text", "value": "According to the guide, xAI's Grok models are integrated with which platform?" }, "options": [ { "modality": "text", "value": "Discord" }, { "modality": "text", "value": "LinkedIn" }, { "modality": "text", "value": "X (formerly Twitter)" }, { "modality": "text", "value": "Telegram" } ], "correctIndex": 2, "explanation": "The guide states xAI's Grok is integrated with X (formerly Twitter).", "source": { "label": "LLM 101 — Objective Test" }, "uid": "14al1xt1302zzj" }, { "id": "ot-cov-second-tier-1", "shape": "mcq", "tags": [ "companies", "second-tier" ], "prompt": { "modality": "text", "value": "Which company focuses on the enterprise with Command, Embed, and Rerank products?" }, "options": [ { "modality": "text", "value": "Mistral AI" }, { "modality": "text", "value": "xAI" }, { "modality": "text", "value": "Hugging Face" }, { "modality": "text", "value": "Cohere" } ], "correctIndex": 3, "explanation": "The guide states Cohere focuses on the enterprise with its Command, Embed, and Rerank products and RAG tooling.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1bqhsekilur8w" }, { "id": "ot-cov-second-tier-2", "shape": "mcq", "tags": [ "companies", "second-tier" ], "prompt": { "modality": "text", "value": "According to the guide, Cohere was co-founded by Aidan Gomez, a co-author of which paper?" }, "options": [ { "modality": "text", "value": "BERT: Pre-training of Deep Transformers" }, { "modality": "text", "value": "Attention Is All You Need" }, { "modality": "text", "value": "Language Models are Few-Shot Learners" }, { "modality": "text", "value": "Constitutional AI" } ], "correctIndex": 1, "explanation": "The guide notes Cohere co-founder Aidan Gomez co-authored the original Transformer paper, 'Attention Is All You Need'.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1gtb578l1ggf8" }, { "id": "ot-cov-second-tier-3", "shape": "mcq", "tags": [ "companies", "open-source" ], "prompt": { "modality": "text", "value": "Which lab became the driving force of open-source LLMs by releasing its Llama model weights freely?" }, "options": [ { "modality": "text", "value": "OpenAI" }, { "modality": "text", "value": "Meta AI" }, { "modality": "text", "value": "Anthropic" }, { "modality": "text", "value": "Google DeepMind" } ], "correctIndex": 1, "explanation": "The guide states Meta AI, by releasing its Llama weights freely, became the driving force of open-source LLMs.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "iei1sv1o5anth" }, { "id": "ot-cov-second-tier-4", "shape": "mcq", "tags": [ "companies", "open-source" ], "prompt": { "modality": "text", "value": "According to the guide, which flagship model family exemplifies Meta AI's open-weight approach?" }, "options": [ { "modality": "text", "value": "Grok" }, { "modality": "text", "value": "GPT" }, { "modality": "text", "value": "Llama" }, { "modality": "text", "value": "Gemini" } ], "correctIndex": 2, "explanation": "The guide identifies Llama as Meta AI's flagship, freely-released open-weight model family.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "wlfvsh1235twb" }, { "id": "ot-cov-leaders-ecosystem-1", "shape": "mcq", "tags": [ "llm", "people", "leaders" ], "prompt": { "modality": "text", "value": "Who has been CEO of OpenAI since 2019 and led ChatGPT's launch?" }, "options": [ { "modality": "text", "value": "Andrej Karpathy" }, { "modality": "text", "value": "Demis Hassabis" }, { "modality": "text", "value": "Dario Amodei" }, { "modality": "text", "value": "Sam Altman" } ], "correctIndex": 3, "explanation": "Sam Altman has been OpenAI's CEO since 2019 and led ChatGPT's launch and OpenAI's commercial pivot.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1p4jvbrqdhu0h" }, { "id": "ot-cov-leaders-ecosystem-2", "shape": "mcq", "tags": [ "llm", "people", "leaders" ], "prompt": { "modality": "text", "value": "Which OpenAI leader was briefly ousted and then reinstated by the board in November 2023?" }, "options": [ { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Sam Altman" }, { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Ilya Sutskever" } ], "correctIndex": 1, "explanation": "Sam Altman was famously ousted and reinstated by OpenAI's board in November 2023.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "4e6hdb9rdv71" }, { "id": "ot-cov-leaders-ecosystem-3", "shape": "mcq", "tags": [ "llm", "people", "anthropic" ], "prompt": { "modality": "text", "value": "Who is Anthropic's president and a former VP of Safety and Policy at OpenAI?" }, "options": [ { "modality": "text", "value": "Sam Altman" }, { "modality": "text", "value": "Demis Hassabis" }, { "modality": "text", "value": "Daniela Amodei" }, { "modality": "text", "value": "Dario Amodei" } ], "correctIndex": 2, "explanation": "Daniela Amodei is Anthropic's president and previously led Safety and Policy at OpenAI.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "pxxcg7qrf8ix" }, { "id": "ot-cov-leaders-ecosystem-4", "shape": "mcq", "tags": [ "llm", "people", "anthropic" ], "prompt": { "modality": "text", "value": "Which sibling co-founded Anthropic with Dario Amodei in 2021?" }, "options": [ { "modality": "text", "value": "Ilya Sutskever" }, { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Daniela Amodei" }, { "modality": "text", "value": "Yoshua Bengio" } ], "correctIndex": 2, "explanation": "Daniela Amodei co-founded Anthropic in 2021 with her brother Dario.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1wzk0hdrqqkcb" }, { "id": "ot-cov-leaders-ecosystem-5", "shape": "mcq", "tags": [ "llm", "people", "deepmind" ], "prompt": { "modality": "text", "value": "Who co-founded and leads Google DeepMind, having built AlphaGo and AlphaFold?" }, "options": [ { "modality": "text", "value": "Yann LeCun" }, { "modality": "text", "value": "Demis Hassabis" }, { "modality": "text", "value": "Aidan Gomez" }, { "modality": "text", "value": "Sam Altman" } ], "correctIndex": 1, "explanation": "Demis Hassabis co-founded and is CEO of Google DeepMind, and built AlphaGo and AlphaFold.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1156mm319xab2t" }, { "id": "ot-cov-leaders-ecosystem-6", "shape": "mcq", "tags": [ "llm", "people", "deepmind" ], "prompt": { "modality": "text", "value": "Which DeepMind leader shared the 2024 Nobel Prize in Chemistry for protein-structure prediction?" }, "options": [ { "modality": "text", "value": "Demis Hassabis" }, { "modality": "text", "value": "Ashish Vaswani" }, { "modality": "text", "value": "Geoffrey Hinton" }, { "modality": "text", "value": "Yoshua Bengio" } ], "correctIndex": 0, "explanation": "Demis Hassabis shared the 2024 Nobel Prize in Chemistry for protein-structure prediction work at DeepMind.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "18oyb1f1blh06t" }, { "id": "ot-cov-other-model-sizes-1", "shape": "mcq", "tags": [ "llm", "numbers", "model-sizes" ], "prompt": { "modality": "text", "value": "What is the parameter count of Mistral 7B?" }, "options": [ { "modality": "text", "value": "13 billion" }, { "modality": "text", "value": "70 billion" }, { "modality": "text", "value": "540 billion" }, { "modality": "text", "value": "7 billion" } ], "correctIndex": 3, "explanation": "Mistral 7B, one of Mistral AI's flagship open-weight models, has 7 billion parameters.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "kffokjcrhvmx" }, { "id": "ot-cov-other-model-sizes-2", "shape": "mcq", "tags": [ "llm", "numbers", "model-sizes" ], "prompt": { "modality": "text", "value": "Mistral AI's efficient open-weight model, Mistral 7B, has roughly how many parameters?" }, "options": [ { "modality": "text", "value": "175 billion" }, { "modality": "text", "value": "340 million" }, { "modality": "text", "value": "7 billion" }, { "modality": "text", "value": "1.5 billion" } ], "correctIndex": 2, "explanation": "Mistral 7B carries about 7 billion parameters, matching LLaMA-7B in scale.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "qsphkpixp5y3" }, { "id": "ot-cov-milestone-dates-1", "shape": "mcq", "tags": [ "llm", "numbers", "milestones" ], "prompt": { "modality": "text", "value": "In what year was BERT released?" }, "options": [ { "modality": "text", "value": "2017" }, { "modality": "text", "value": "2018" }, { "modality": "text", "value": "2016" }, { "modality": "text", "value": "2019" } ], "correctIndex": 1, "explanation": "BERT was released in 2018, ahead of GPT-2 (2019) and GPT-3 (2020) in the field's timeline.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1x68xuaaxu2ky" }, { "id": "ot-cov-milestone-dates-2", "shape": "mcq", "tags": [ "llm", "numbers", "milestones" ], "prompt": { "modality": "text", "value": "Which year marks BERT's release, the earliest milestone in this lesson's timeline?" }, "options": [ { "modality": "text", "value": "2018" }, { "modality": "text", "value": "2019" }, { "modality": "text", "value": "2020" }, { "modality": "text", "value": "2021" } ], "correctIndex": 0, "explanation": "BERT came out in 2018, the earliest of the milestones drilled in this lesson.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "jbln5c10weitw" }, { "id": "ot-cov-milestone-dates-3", "shape": "mcq", "tags": [ "llm", "numbers", "milestones" ], "prompt": { "modality": "text", "value": "What year was GPT-2 released?" }, "options": [ { "modality": "text", "value": "2019" }, { "modality": "text", "value": "2020" }, { "modality": "text", "value": "2018" }, { "modality": "text", "value": "2021" } ], "correctIndex": 0, "explanation": "GPT-2 was released in 2019, jumping to 1.5 billion parameters.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1t395wx1c5b2cr" }, { "id": "ot-cov-milestone-dates-4", "shape": "mcq", "tags": [ "llm", "numbers", "milestones" ], "prompt": { "modality": "text", "value": "GPT-2, with 1.5 billion parameters, was released in which year?" }, "options": [ { "modality": "text", "value": "2017" }, { "modality": "text", "value": "2018" }, { "modality": "text", "value": "2019" }, { "modality": "text", "value": "2021" } ], "correctIndex": 2, "explanation": "GPT-2 launched in 2019, between GPT-1 (2018) and GPT-3 (2020).", "source": { "label": "LLM 101 — Objective Test" }, "uid": "dyzgqeqvqr2e" }, { "id": "ot-cov-milestone-dates-5", "shape": "mcq", "tags": [ "llm", "numbers", "milestones" ], "prompt": { "modality": "text", "value": "What year was GPT-4 released?" }, "options": [ { "modality": "text", "value": "2024" }, { "modality": "text", "value": "2022" }, { "modality": "text", "value": "2023" }, { "modality": "text", "value": "2021" } ], "correctIndex": 2, "explanation": "GPT-4 was released on March 14, 2023, following ChatGPT's late-2022 launch.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "19d45zt1l20mxf" }, { "id": "ot-cov-milestone-dates-6", "shape": "mcq", "tags": [ "llm", "numbers", "milestones" ], "prompt": { "modality": "text", "value": "GPT-4 launched on March 14 of which year?" }, "options": [ { "modality": "text", "value": "2021" }, { "modality": "text", "value": "2022" }, { "modality": "text", "value": "2024" }, { "modality": "text", "value": "2023" } ], "correctIndex": 3, "explanation": "GPT-4 launched on March 14, 2023, closing out the field's fastest 18-month stretch.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "3tapa11qovqz3" }, { "id": "ot-cov-scaling-context-1", "shape": "mcq", "tags": [ "llm", "numbers", "scaling" ], "prompt": { "modality": "text", "value": "In what year was the Kaplan scaling-laws paper published?" }, "options": [ { "modality": "text", "value": "2022" }, { "modality": "text", "value": "2019" }, { "modality": "text", "value": "2018" }, { "modality": "text", "value": "2020" } ], "correctIndex": 3, "explanation": "The Kaplan scaling-laws paper, showing loss falls predictably with scale, was published in 2020.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "y6pktm1fr1xiu" }, { "id": "ot-cov-scaling-context-2", "shape": "mcq", "tags": [ "llm", "numbers", "scaling" ], "prompt": { "modality": "text", "value": "The scaling-laws paper establishing that model loss falls predictably with scale appeared in which year?" }, "options": [ { "modality": "text", "value": "2023" }, { "modality": "text", "value": "2017" }, { "modality": "text", "value": "2020" }, { "modality": "text", "value": "2021" } ], "correctIndex": 2, "explanation": "Kaplan's scaling-laws paper was published in 2020, two years before Chinchilla refined the picture.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1mqyye21gw2zsu" }, { "id": "ot-cov-scaling-context-3", "shape": "mcq", "tags": [ "llm", "numbers", "scaling" ], "prompt": { "modality": "text", "value": "According to the Chinchilla result, how many training tokens per parameter is compute-optimal?" }, "options": [ { "modality": "text", "value": "10" }, { "modality": "text", "value": "50" }, { "modality": "text", "value": "5" }, { "modality": "text", "value": "20" } ], "correctIndex": 3, "explanation": "Chinchilla's rule is about 20 training tokens per parameter for compute-optimal training.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1j9h205nq0jgn" }, { "id": "ot-cov-scaling-context-4", "shape": "mcq", "tags": [ "llm", "numbers", "scaling" ], "prompt": { "modality": "text", "value": "Chinchilla showed many earlier giant models were undertrained relative to what token-per-parameter ratio?" }, "options": [ { "modality": "text", "value": "100 to 1" }, { "modality": "text", "value": "20 to 1" }, { "modality": "text", "value": "1 to 1" }, { "modality": "text", "value": "400 to 1" } ], "correctIndex": 1, "explanation": "Chinchilla's compute-optimal ratio is about 20 tokens per parameter, well above what earlier giant models used.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1l8ucmt1lg6sbv" }, { "id": "ot-cov-architecture-terms-1", "shape": "mcq", "tags": [ "llm", "architecture", "recall" ], "prompt": { "modality": "text", "value": "Which term describes how tokens weigh one another inside a Transformer layer?" }, "options": [ { "modality": "text", "value": "Self-attention" }, { "modality": "text", "value": "Inference" }, { "modality": "text", "value": "Few-shot" }, { "modality": "text", "value": "Pretraining" } ], "correctIndex": 0, "explanation": "Self-attention is the mechanism by which tokens weigh one another inside a Transformer layer.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "3pwi9f1x41e95" }, { "id": "ot-cov-architecture-terms-2", "shape": "mcq", "tags": [ "llm", "architecture", "recall" ], "prompt": { "modality": "text", "value": "What is the mechanism inside a Transformer layer that lets tokens attend to one another?" }, "options": [ { "modality": "text", "value": "RLHF" }, { "modality": "text", "value": "RAG" }, { "modality": "text", "value": "Chinchilla" }, { "modality": "text", "value": "Self-attention" } ], "correctIndex": 3, "explanation": "Self-attention is the Transformer mechanism that lets tokens attend to one another.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "133fnwp1t51kwr" }, { "id": "ot-cov-obj-training-terms-1", "shape": "mcq", "tags": [ "llm", "training", "chinchilla" ], "prompt": { "modality": "text", "value": "What does the Chinchilla scaling rule recommend as the compute-optimal amount of training data?" }, "options": [ { "modality": "text", "value": "A fixed 1 trillion tokens regardless of size" }, { "modality": "text", "value": "About 20 training tokens per parameter" }, { "modality": "text", "value": "About 100 training tokens per parameter" }, { "modality": "text", "value": "One training token per parameter" } ], "correctIndex": 1, "explanation": "Chinchilla is shorthand for the compute-optimal rule of thumb: roughly twenty training tokens per model parameter.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1rqxd2saseevg" }, { "id": "ot-cov-obj-training-terms-2", "shape": "mcq", "tags": [ "llm", "training", "chinchilla" ], "prompt": { "modality": "text", "value": "The Chinchilla rule of thumb is a guideline about the ratio of which two things?" }, "options": [ { "modality": "text", "value": "Context window size to token count" }, { "modality": "text", "value": "Attention heads to Transformer layers" }, { "modality": "text", "value": "Training tokens to model parameters" }, { "modality": "text", "value": "Human feedback examples to RLHF steps" } ], "correctIndex": 2, "explanation": "Chinchilla names the compute-optimal ratio of training tokens to model parameters, about 20 to 1.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1te11tz5rw4ph" }, { "id": "ot-cov-obj-usage-terms-1", "shape": "mcq", "tags": [ "llm", "prompting", "few-shot" ], "prompt": { "modality": "text", "value": "Teaching a model a new task by including worked examples directly in the prompt, with no retraining, is called what?" }, "options": [ { "modality": "text", "value": "RLHF" }, { "modality": "text", "value": "Few-shot prompting" }, { "modality": "text", "value": "Fine-tuning" }, { "modality": "text", "value": "Pretraining" } ], "correctIndex": 1, "explanation": "Few-shot prompting means giving the model examples in the prompt itself to teach it a task on the fly, with no weight updates.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "g0tbiy271mga" }, { "id": "ot-cov-obj-usage-terms-2", "shape": "mcq", "tags": [ "llm", "prompting", "few-shot" ], "prompt": { "modality": "text", "value": "In few-shot prompting, where do the task-teaching examples live?" }, "options": [ { "modality": "text", "value": "In the retraining dataset" }, { "modality": "text", "value": "In a separate RAG database" }, { "modality": "text", "value": "In the prompt alongside the request" }, { "modality": "text", "value": "In the model's fine-tuned weights" } ], "correctIndex": 2, "explanation": "Few-shot prompting puts example input/output pairs right in the prompt, so the model picks up the pattern without any training step.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "ukhdpz1eo68pt" }, { "id": "ot-cov-obj-usage-terms-3", "shape": "mcq", "tags": [ "llm", "scaling", "emergence" ], "prompt": { "modality": "text", "value": "What term names capabilities that show up only once a model crosses a scale threshold?" }, "options": [ { "modality": "text", "value": "Chinchilla ratio" }, { "modality": "text", "value": "Context window" }, { "modality": "text", "value": "Emergence" }, { "modality": "text", "value": "Hallucination" } ], "correctIndex": 2, "explanation": "Emergence describes capabilities that appear only past a certain model scale, absent in smaller models.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "glb4dbb8k521" }, { "id": "ot-cov-obj-usage-terms-4", "shape": "mcq", "tags": [ "llm", "scaling", "emergence" ], "prompt": { "modality": "text", "value": "Emergence in LLMs describes which pattern?" }, "options": [ { "modality": "text", "value": "Retrieval improving factual grounding" }, { "modality": "text", "value": "Human feedback correcting bad outputs" }, { "modality": "text", "value": "A new skill appearing only above a scale threshold" }, { "modality": "text", "value": "A model forgetting older training data" } ], "correctIndex": 2, "explanation": "Emergence is the term for capabilities absent in smaller models that appear once a model is scaled past a certain size.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1h4tuz716v4cc9" }, { "id": "ot-cov-obj-usage-terms-5", "shape": "mcq", "tags": [ "llm", "rag", "grounding" ], "prompt": { "modality": "text", "value": "Retrieving documents and injecting them into the prompt so a model answers from grounded sources is called what?" }, "options": [ { "modality": "text", "value": "Few-shot prompting" }, { "modality": "text", "value": "RAG" }, { "modality": "text", "value": "RLHF" }, { "modality": "text", "value": "Pretraining" } ], "correctIndex": 1, "explanation": "RAG (Retrieval-Augmented Generation) retrieves relevant documents into the prompt so the answer is grounded in that content.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "188xbux1ttdu5v" }, { "id": "ot-cov-obj-usage-terms-6", "shape": "mcq", "tags": [ "llm", "rag", "hallucination" ], "prompt": { "modality": "text", "value": "What technique is described as the main defense against hallucination?" }, "options": [ { "modality": "text", "value": "Few-shot prompting" }, { "modality": "text", "value": "RAG" }, { "modality": "text", "value": "A larger context window" }, { "modality": "text", "value": "RLHF" } ], "correctIndex": 1, "explanation": "The guide names RAG as the main defense against hallucination, since it grounds answers in retrieved documents.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1cxgg3ieps2au" }, { "id": "ot-cov-obj-language-tasks-1", "shape": "mcq", "tags": [ "llm", "language-tasks", "summarization" ], "prompt": { "modality": "text", "value": "What LLM strength involves condensing long documents into structured summaries?" }, "options": [ { "modality": "text", "value": "Extraction" }, { "modality": "text", "value": "Summarization" }, { "modality": "text", "value": "Classification" }, { "modality": "text", "value": "Translation" } ], "correctIndex": 1, "explanation": "The guide names summarization — condensing long documents into structured summaries — as a top LLM strength.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "2pfctfsd2bgp" }, { "id": "ot-cov-obj-language-tasks-2", "shape": "mcq", "tags": [ "llm", "language-tasks", "summarization" ], "prompt": { "modality": "text", "value": "What is the main catch when an LLM summarizes a document, per the guide?" }, "options": [ { "modality": "text", "value": "It always mistranslates idioms" }, { "modality": "text", "value": "It refuses documents over one page" }, { "modality": "text", "value": "Nuance and exact figures can drop" }, { "modality": "text", "value": "It cannot process structured text" } ], "correctIndex": 2, "explanation": "The guide's catch for summarization is that nuance and exact figures can drop, so critical details need verifying.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "13pkxhc1ieac8" }, { "id": "ot-cov-obj-language-tasks-3", "shape": "mcq", "tags": [ "llm", "language-tasks", "simplification" ], "prompt": { "modality": "text", "value": "Rewriting technical, legal, or medical text to a plain reading level is called what?" }, "options": [ { "modality": "text", "value": "Plain-language simplification" }, { "modality": "text", "value": "Rewriting and tone-shifting" }, { "modality": "text", "value": "Classification" }, { "modality": "text", "value": "Structured data extraction" } ], "correctIndex": 0, "explanation": "Plain-language simplification rewrites technical, legal, or medical text for low-literacy or non-native readers.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "tjlw5gglaodw" }, { "id": "ot-cov-obj-language-tasks-4", "shape": "mcq", "tags": [ "llm", "language-tasks", "simplification" ], "prompt": { "modality": "text", "value": "Why does the guide favor plain-language simplification as an LLM use case?" }, "options": [ { "modality": "text", "value": "It only works on short marketing copy" }, { "modality": "text", "value": "It requires no human review of any kind" }, { "modality": "text", "value": "It replaces the need for translation entirely" }, { "modality": "text", "value": "It's high value, low risk, and easy to verify" } ], "correctIndex": 3, "explanation": "The guide calls plain-language simplification high value, low risk, and easy to verify.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "d01im01ph53a4" }, { "id": "ot-cov-obj-language-tasks-5", "shape": "mcq", "tags": [ "llm", "language-tasks", "classification" ], "prompt": { "modality": "text", "value": "For classification and sentiment tasks, what can few-shot prompting match?" }, "options": [ { "modality": "text", "value": "RAG retrieval accuracy" }, { "modality": "text", "value": "Human expert reviewers" }, { "modality": "text", "value": "Dedicated classifiers" }, { "modality": "text", "value": "Fine-tuned translation models" } ], "correctIndex": 2, "explanation": "Given a label set, few-shot prompting matches dedicated classifiers, per the guide.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "bd99t02lf7o4" }, { "id": "ot-cov-obj-language-tasks-6", "shape": "mcq", "tags": [ "llm", "language-tasks", "classification" ], "prompt": { "modality": "text", "value": "What catch does the guide note about LLM classification?" }, "options": [ { "modality": "text", "value": "It's non-deterministic; test on held-out data" }, { "modality": "text", "value": "It cannot handle more than two labels" }, { "modality": "text", "value": "It always outperforms dedicated classifiers" }, { "modality": "text", "value": "It requires retraining for every label set" } ], "correctIndex": 0, "explanation": "Classification is non-deterministic, so the guide recommends low temperature and testing on held-out examples.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1g98sbunt6zqi" }, { "id": "ot-cov-obj-knowledge-code-1", "shape": "mcq", "tags": [ "llm", "knowledge", "research" ], "prompt": { "modality": "text", "value": "What catch does the guide flag about LLM research assistance and citations?" }, "options": [ { "modality": "text", "value": "It needs a fine-tuned model per topic" }, { "modality": "text", "value": "It automatically fact-checks every claim" }, { "modality": "text", "value": "It can't browse live without a search tool" }, { "modality": "text", "value": "It always cites peer-reviewed journals only" } ], "correctIndex": 2, "explanation": "Research assistance can't browse live unless given a search tool, so citations and sources must always be verified.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "2vkmekf33n6k" }, { "id": "ot-cov-obj-knowledge-code-2", "shape": "mcq", "tags": [ "llm", "knowledge", "research" ], "prompt": { "modality": "text", "value": "What does the guide say LLMs do well for research assistance?" }, "options": [ { "modality": "text", "value": "Independently browse the live web by default" }, { "modality": "text", "value": "Guarantee citation accuracy without review" }, { "modality": "text", "value": "Replace peer review entirely" }, { "modality": "text", "value": "Synthesize sources and explain dense papers" } ], "correctIndex": 3, "explanation": "The guide credits research assistance with synthesizing sources, drafting summaries, and explaining dense papers.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "17v5m9n3quvz1" }, { "id": "ot-cov-obj-knowledge-code-3", "shape": "mcq", "tags": [ "llm", "knowledge", "extraction" ], "prompt": { "modality": "text", "value": "Structured data extraction turns messy text like invoices into what, per the guide?" }, "options": [ { "modality": "text", "value": "Clean JSON or CSV" }, { "modality": "text", "value": "A retrieval index" }, { "modality": "text", "value": "A fine-tuned classifier" }, { "modality": "text", "value": "A vector database" } ], "correctIndex": 0, "explanation": "Structured data extraction parses messy text into clean JSON or CSV fields, the guide says.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1be1lyi1ntvzsy" }, { "id": "ot-cov-obj-knowledge-code-4", "shape": "mcq", "tags": [ "llm", "knowledge", "extraction" ], "prompt": { "modality": "text", "value": "What catch does the guide note for structured data extraction?" }, "options": [ { "modality": "text", "value": "It requires manual retyping every time" }, { "modality": "text", "value": "It cannot output JSON, only CSV" }, { "modality": "text", "value": "Rare or ambiguous fields err more often" }, { "modality": "text", "value": "It only works on plain English text" } ], "correctIndex": 2, "explanation": "Rare or ambiguous fields err more often in extraction, so the guide says to validate outputs programmatically.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "6qis331l87zsh" }, { "id": "ot-cov-obj-knowledge-code-5", "shape": "mcq", "tags": [ "llm", "coding", "debugging" ], "prompt": { "modality": "text", "value": "Given an error and the code, what can an LLM often spot, per the guide?" }, "options": [ { "modality": "text", "value": "A reason to skip testing" }, { "modality": "text", "value": "A way to auto-deploy a patch" }, { "modality": "text", "value": "A full codebase rewrite" }, { "modality": "text", "value": "Root causes and fixes" } ], "correctIndex": 3, "explanation": "The guide says pasting an error and the code often lets an LLM spot root causes and fixes.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1pf88ou1u5kv0u" }, { "id": "ot-cov-obj-knowledge-code-6", "shape": "mcq", "tags": [ "llm", "coding", "debugging" ], "prompt": { "modality": "text", "value": "What catch applies to LLM debugging assistance, per the guide?" }, "options": [ { "modality": "text", "value": "It cannot read error messages" }, { "modality": "text", "value": "It always finds the root cause instantly" }, { "modality": "text", "value": "Novel bugs needing profiler data are harder" }, { "modality": "text", "value": "It only works on Python code" } ], "correctIndex": 2, "explanation": "The guide notes novel bugs or performance issues needing profiler data are harder for LLM debugging help.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1mksxxwqfsxj0" }, { "id": "ot-cov-obj-work-tasks-1", "shape": "mcq", "tags": [ "llm", "work-tasks", "support" ], "prompt": { "modality": "text", "value": "What do RAG-backed customer support chatbots do well, per the guide?" }, "options": [ { "modality": "text", "value": "Handle only billing disputes" }, { "modality": "text", "value": "Replace all human support staff permanently" }, { "modality": "text", "value": "Answer tier-1 questions 24/7 and escalate the rest" }, { "modality": "text", "value": "Guarantee zero wrong answers" } ], "correctIndex": 2, "explanation": "RAG-backed support bots answer tier-1 questions 24/7 and escalate ambiguous ones, the guide says.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "gdqddx1udgspj" }, { "id": "ot-cov-obj-work-tasks-2", "shape": "mcq", "tags": [ "llm", "work-tasks", "support" ], "prompt": { "modality": "text", "value": "What catch does the guide flag for customer support chatbots?" }, "options": [ { "modality": "text", "value": "Out-of-scope questions get confident wrong answers" }, { "modality": "text", "value": "They need a fine-tuned model per customer" }, { "modality": "text", "value": "They cannot operate outside business hours" }, { "modality": "text", "value": "They can answer only in one language" } ], "correctIndex": 0, "explanation": "Without good guardrails, out-of-scope questions can yield confident wrong answers, the guide warns.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "szgjm91p2skwv" }, { "id": "ot-cov-obj-work-tasks-3", "shape": "mcq", "tags": [ "llm", "work-tasks", "meeting-notes" ], "prompt": { "modality": "text", "value": "LLM meeting-notes tools summarize transcripts into what, per the guide?" }, "options": [ { "modality": "text", "value": "Decisions and to-dos" }, { "modality": "text", "value": "Financial forecasts" }, { "modality": "text", "value": "Verbatim word-for-word transcripts" }, { "modality": "text", "value": "Calendar invites for follow-ups" } ], "correctIndex": 0, "explanation": "The guide says meeting-notes tools summarize transcripts into decisions and to-dos.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1hfz2x5w8lurz" }, { "id": "ot-cov-obj-work-tasks-4", "shape": "mcq", "tags": [ "llm", "work-tasks", "meeting-notes" ], "prompt": { "modality": "text", "value": "What catch does the guide note about LLM meeting notes?" }, "options": [ { "modality": "text", "value": "They need video, not audio, input" }, { "modality": "text", "value": "They always miscount attendees" }, { "modality": "text", "value": "They can't summarize transcripts over 5 minutes" }, { "modality": "text", "value": "Speaker attribution and acronyms get mangled" } ], "correctIndex": 3, "explanation": "Speaker attribution and acronyms get mangled, so the guide says attendees should skim before distributing.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "76x4bypaxmze" }, { "id": "ot-cov-obj-work-tasks-5", "shape": "mcq", "tags": [ "llm", "work-tasks", "data-cleaning" ], "prompt": { "modality": "text", "value": "What does LLM data cleaning do with a field like 'NY', per the guide's example?" }, "options": [ { "modality": "text", "value": "Normalizes it to 'New York, NY'" }, { "modality": "text", "value": "Converts it into an image file" }, { "modality": "text", "value": "Deletes it as a duplicate" }, { "modality": "text", "value": "Encrypts it for privacy" } ], "correctIndex": 0, "explanation": "The guide's example shows data cleaning normalizing 'NY' / 'new york' into 'New York, NY'.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "370xgj15jxpzx" }, { "id": "ot-cov-obj-work-tasks-6", "shape": "mcq", "tags": [ "llm", "work-tasks", "data-cleaning" ], "prompt": { "modality": "text", "value": "What catch does the guide note for LLM data cleaning at scale?" }, "options": [ { "modality": "text", "value": "It's always perfectly consistent at any volume" }, { "modality": "text", "value": "High-volume pipelines need structured-output modes" }, { "modality": "text", "value": "It only works on numeric data, never text" }, { "modality": "text", "value": "It can't process more than 10 rows at once" } ], "correctIndex": 1, "explanation": "High-volume pipelines need structured-output modes or fine-tuning for consistency, the guide notes.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1y5dw0nv918u9" }, { "id": "ot-cov-obj-fit-domains-1", "shape": "mcq", "tags": [ "healthcare", "guardrails" ], "prompt": { "modality": "text", "value": "In healthcare, what does the guide say every LLM output requires before use?" }, "options": [ { "modality": "text", "value": "A blockchain audit log" }, { "modality": "text", "value": "A larger context window" }, { "modality": "text", "value": "Clinician review" }, { "modality": "text", "value": "Real-time internet access" } ], "correctIndex": 2, "explanation": "Because hallucination may be intrinsic to LLMs, healthcare deployment requires clinician review of every output plus domain-specific evaluation.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1ny66e815s8kf4" }, { "id": "ot-cov-obj-fit-domains-2", "shape": "mcq", "tags": [ "healthcare", "hallucination" ], "prompt": { "modality": "text", "value": "What specific risk does the guide flag as central to LLM use in healthcare?" }, "options": [ { "modality": "text", "value": "Medical hallucination such as a wrong dosage" }, { "modality": "text", "value": "Slow response latency" }, { "modality": "text", "value": "High compute cost" }, { "modality": "text", "value": "Poor translation quality" } ], "correctIndex": 0, "explanation": "The guide names medical hallucination — a wrong dosage or invented drug interaction — as the core healthcare risk requiring review.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "4gb6cx1759b37" }, { "id": "ot-cov-obj-fit-domains-3", "shape": "mcq", "tags": [ "legal", "guardrails" ], "prompt": { "modality": "text", "value": "What must happen to all LLM-generated legal content before it carries any weight?" }, "options": [ { "modality": "text", "value": "Automatic filing with the court" }, { "modality": "text", "value": "Cite-verification by a qualified attorney" }, { "modality": "text", "value": "Translation into plain language" }, { "modality": "text", "value": "Peer review by another LLM" } ], "correctIndex": 1, "explanation": "The guide requires all LLM legal content be cite-verified by a qualified attorney, since fabricated citations can look real.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1b5hv0t1ws9axb" }, { "id": "ot-cov-obj-fit-domains-4", "shape": "mcq", "tags": [ "legal", "case-law" ], "prompt": { "modality": "text", "value": "What 2023 incident does the guide cite as a warning about unsupervised LLM use in law?" }, "options": [ { "modality": "text", "value": "A law firm's court filing was leaked in a data breach" }, { "modality": "text", "value": "An LLM won a mock trial competition" }, { "modality": "text", "value": "A judge overturned a verdict predicted by an LLM" }, { "modality": "text", "value": "Attorneys sanctioned for fabricated ChatGPT citations" } ], "correctIndex": 3, "explanation": "Two attorneys were sanctioned after filing a brief with case citations ChatGPT had fabricated — the citations looked real but the cases did not exist.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1wz5ehb1ctkq4d" }, { "id": "ot-cov-obj-fit-domains-5", "shape": "mcq", "tags": [ "finance", "guardrails" ], "prompt": { "modality": "text", "value": "What does the guide recommend as best practice for finance LLM deployments?" }, "options": [ { "modality": "text", "value": "Fine-tuning on customer chat logs only" }, { "modality": "text", "value": "Fully autonomous trading without oversight" }, { "modality": "text", "value": "RAG over verified data plus mandatory human review" }, { "modality": "text", "value": "Skipping review for internal-only reports" } ], "correctIndex": 2, "explanation": "Best practice in finance is RAG over verified data sources combined with mandatory human review before output reaches a customer or decision system.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1xjacoierfajy" }, { "id": "ot-cov-obj-fit-domains-6", "shape": "mcq", "tags": [ "finance", "regulation" ], "prompt": { "modality": "text", "value": "Why does a hallucinated figure in a customer-facing finance document matter, per the guide?" }, "options": [ { "modality": "text", "value": "It slightly increases API costs" }, { "modality": "text", "value": "It only affects internal training data" }, { "modality": "text", "value": "It creates legal and financial harm under regulation" }, { "modality": "text", "value": "It is automatically corrected by the model" } ], "correctIndex": 2, "explanation": "Finance outputs fall under existing regulation, so a hallucinated figure in a customer-facing document creates both legal and financial harm.", "source": { "label": "LLM 101 — Objective Test" }, "uid": "1np074g3ccb0s" } ], "lessons": [ { "id": "l1-history", "title": "A Brief History of LLMs", "studyGuidePath": "/packs/llm-101/guides/l1-p1-before.md", "parts": [ { "id": "l1-p1-before", "title": "Before the Transformer", "studyGuideAnchor": "before-the-transformer", "itemIds": [ "f-ngrams", "f-rnn-lstm", "f-word2vec", "f-seq2seq", "num-transformer-year", "pair-attention-paper", "mcq-rw-l1-p1-before", "cz-llm-101-f-rnn-lstm", "ot-pre-transformer-1", "ot-pre-transformer-2", "ot-pre-transformer-3", "ot-pre-transformer-4", "fi-ot-pre-transformer-3" ], "order": 1, "studyGuidePath": "/packs/llm-101/guides/l1-p1-before.md" }, { "id": "l1-p2-gpt", "title": "The GPT era", "studyGuideAnchor": "the-gpt-era", "itemIds": [ "f-bert", "f-gpt1", "f-gpt2", "f-gpt3", "pair-kaplan", "pair-chinchilla", "mcq-rw-l1-p2-gpt", "cz-llm-101-f-bert", "cz-llm-101-f-gpt3", "concept-rw-llm-101-gpt-1", "concept-rw-llm-101-gpt-2", "ot-scaling-1", "ot-scaling-2", "ot-scaling-3", "ot-scaling-4", "ot-scaling-5", "ot-bert-gpt-1", "ot-bert-gpt-2", "ot-bert-gpt-3", "ot-bert-gpt-4", "fi-ot-scaling-1" ], "order": 2, "studyGuidePath": "/packs/llm-101/guides/l1-p2-gpt.md" }, { "id": "l1-p3-modern", "title": "ChatGPT and the modern wave", "studyGuideAnchor": "chatgpt-and-the-modern-wave", "itemIds": [ "f-instructgpt", "f-chatgpt-launch", "f-gpt4", "f-llama", "f-mistral", "f-claude-gemini", "mcq-rw-l1-p3-modern", "cz-llm-101-f-instructgpt", "cz-llm-101-f-chatgpt-launch", "cz-llm-101-f-gpt4", "cz-llm-101-f-mistral", "cz-llm-101-f-claude-gemini", "concept-rw-llm-101-gpt-4", "concept-rw-llm-101-llama", "concept-rw-llm-101-mistral-7b", "ot-align-1", "ot-align-2", "ot-align-3", "ot-align-4", "ot-align-5", "ot-open-gemini", "ot-open-2", "ot-open-3", "ot-open-4" ], "order": 3, "studyGuidePath": "/packs/llm-101/guides/l1-p3-modern.md" } ], "order": 1, "objectives": [ { "id": "obj-pre-transformer", "statement": "Trace the pre-Transformer progression from RNNs/LSTMs through the attention mechanism to the 2017 Transformer.", "demonstrationIds": [ "d-pre-transformer" ] }, { "id": "obj-scaling-laws", "statement": "Contrast the Kaplan (2020) and Chinchilla (2022) scaling laws and what each revised about optimal model training.", "demonstrationIds": [ "d-scaling-laws" ] }, { "id": "obj-bert-gpt", "statement": "Explain how BERT's bidirectional (masked) pretraining differed from GPT and identify the GPT-1 to GPT-3 progression.", "demonstrationIds": [ "d-bert", "d-gpt-progression" ] }, { "id": "obj-alignment-chatgpt", "statement": "Describe how RLHF/InstructGPT and the ChatGPT and GPT-4 launches turned capable models into useful products.", "demonstrationIds": [ "d-alignment-launch" ] }, { "id": "obj-open-weights", "statement": "Recognize the modern open-weight wave (LLaMA, Mistral) and the frontier assistants (Claude, Gemini, GPT-4).", "demonstrationIds": [ "d-open-weights", "d-frontier-things" ] } ], "demonstrations": [ { "id": "d-pre-transformer", "label": "RNN/LSTM to attention to Transformer", "itemIds": [ "cz-llm-101-f-rnn-lstm", "mcq-rw-l1-p1-before", "num-transformer-year", "pair-attention-paper" ] }, { "id": "d-scaling-laws", "label": "Kaplan vs Chinchilla", "itemIds": [ "pair-kaplan", "pair-chinchilla" ] }, { "id": "d-bert", "label": "BERT bidirectional pretraining", "itemIds": [ "mcq-rw-l1-p2-gpt", "cz-llm-101-f-bert" ] }, { "id": "d-gpt-progression", "label": "GPT-1 to GPT-3", "itemIds": [ "concept-rw-llm-101-gpt-1", "concept-rw-llm-101-gpt-2", "cz-llm-101-f-gpt3" ] }, { "id": "d-alignment-launch", "label": "RLHF, ChatGPT and GPT-4", "itemIds": [ "cz-llm-101-f-instructgpt", "cz-llm-101-f-chatgpt-launch", "cz-llm-101-f-gpt4" ] }, { "id": "d-open-weights", "label": "Open-weight models", "itemIds": [ "mcq-rw-l1-p3-modern", "cz-llm-101-f-mistral", "concept-rw-llm-101-llama", "concept-rw-llm-101-mistral-7b" ] }, { "id": "d-frontier-things", "label": "Frontier assistants", "itemIds": [ "cz-llm-101-f-claude-gemini", "concept-rw-llm-101-gpt-4" ] } ], "objectiveTests": [ { "id": "test-pre-transformer", "objectiveId": "obj-pre-transformer", "title": "Before the Transformer", "mcqIds": [ "mcq-rw-l1-p1-before", "ot-pre-transformer-1", "ot-pre-transformer-2", "ot-pre-transformer-3", "ot-pre-transformer-4" ] }, { "id": "test-scaling-laws", "objectiveId": "obj-scaling-laws", "title": "Scaling Laws: Kaplan vs Chinchilla", "mcqIds": [ "ot-scaling-1", "ot-scaling-2", "ot-scaling-3", "ot-scaling-4", "ot-scaling-5" ] }, { "id": "test-bert-gpt", "objectiveId": "obj-bert-gpt", "title": "BERT and the GPT progression", "mcqIds": [ "mcq-rw-l1-p2-gpt", "ot-bert-gpt-1", "ot-bert-gpt-2", "ot-bert-gpt-3", "ot-bert-gpt-4" ] }, { "id": "test-alignment-chatgpt", "objectiveId": "obj-alignment-chatgpt", "title": "RLHF, ChatGPT and GPT-4", "mcqIds": [ "ot-align-1", "ot-align-2", "ot-align-3", "ot-align-4", "ot-align-5" ] }, { "id": "test-open-weights", "objectiveId": "obj-open-weights", "title": "Open weights and the frontier", "mcqIds": [ "mcq-rw-l1-p3-modern", "ot-open-gemini", "ot-open-2", "ot-open-3", "ot-open-4" ] } ] }, { "id": "l2-mechanics", "title": "How LLMs Work", "studyGuidePath": "/packs/llm-101/guides/l2-mechanics.md", "parts": [ { "id": "l2-p1-tokens", "title": "Tokens, embeddings and attention", "studyGuideAnchor": "tokens-embeddings-and-attention", "itemIds": [ "def-token", "def-tokenization", "def-embedding-llm", "def-Transformer", "def-self-attention", "def-context-window", "def-parameters", "mcq-rw-l2-p1-tokens", "cz-llm-101-def-token", "cz-llm-101-def-embedding-llm", "cz-llm-101-def-Transformer", "tf-d-llm-101-def-token", "tf-d-llm-101-def-tokenization", "tf-d-llm-101-def-embedding-llm", "tf-d-llm-101-def-Transformer", "tf-d-llm-101-def-self-attention", "concept-rw-llm-101-embedding", "concept-rw-llm-101-self-attention", "concept-rw-llm-101-context-window", "ot-text-to-numbers-1", "ot-text-to-numbers-2", "ot-text-to-numbers-3", "ot-text-to-numbers-4", "ot-model-internals-1", "ot-model-internals-2", "ot-model-internals-3", "ot-model-internals-4", "ot-model-internals-5" ], "order": 1 }, { "id": "l2-p2-generate", "title": "How text is generated", "studyGuideAnchor": "how-text-is-generated", "itemIds": [ "def-next-token", "def-autoregressive", "def-logits", "def-temperature", "def-hallucination", "f-why-hallucinate", "def-emergence", "mcq-rw-l2-p2-generate", "cz-llm-101-def-logits", "cz-llm-101-def-temperature", "cz-llm-101-def-hallucination", "tf-d-llm-101-def-next-token", "tf-df-llm-101-def-autoregressive", "tf-df-llm-101-def-logits", "tf-df-llm-101-def-temperature", "tf-d-llm-101-def-hallucination", "concept-rw-llm-101-temperature", "concept-rw-llm-101-hallucination", "ot-generation-loop-1", "ot-generation-loop-2", "ot-generation-loop-3", "ot-generation-loop-4", "ot-generation-loop-5", "ot-sampling-1", "ot-sampling-2", "ot-sampling-3", "ot-sampling-4", "ot-sampling-5", "ot-halluc-emerge-1", "ot-halluc-emerge-2", "ot-halluc-emerge-3", "ot-halluc-emerge-4", "fi-ot-generation-loop-1", "fi-ot-generation-loop-3", "fi-ot-sampling-1", "fi-ot-sampling-3" ], "order": 2 } ], "order": 2, "objectives": [ { "id": "obj-text-to-numbers", "statement": "Explain how text becomes tokens and then embedding vectors a model can compute over.", "demonstrationIds": [ "d-tokens", "d-embeddings" ] }, { "id": "obj-model-internals", "statement": "Describe the Transformer, self-attention, context window, and parameters that make up a model's internals.", "demonstrationIds": [ "d-transformer-attention", "d-context-params" ] }, { "id": "obj-generation-loop", "statement": "Describe next-token prediction and autoregressive generation as the core loop that produces text.", "demonstrationIds": [ "d-next-token", "d-autoregressive" ] }, { "id": "obj-sampling", "statement": "Explain how logits and the temperature knob shape which next token is sampled.", "demonstrationIds": [ "d-logits", "d-temperature" ] }, { "id": "obj-hallucination-emergence", "statement": "Explain why LLMs hallucinate and recognize emergent abilities that appear only at scale.", "demonstrationIds": [ "d-hallucination", "d-emergence" ] } ], "demonstrations": [ { "id": "d-tokens", "label": "Tokens and tokenization", "itemIds": [ "def-token", "def-tokenization", "mcq-rw-l2-p1-tokens", "cz-llm-101-def-token" ] }, { "id": "d-embeddings", "label": "Embeddings", "itemIds": [ "def-embedding-llm", "cz-llm-101-def-embedding-llm", "concept-rw-llm-101-embedding", "tf-d-llm-101-def-embedding-llm" ] }, { "id": "d-transformer-attention", "label": "Transformer and self-attention", "itemIds": [ "def-Transformer", "def-self-attention", "cz-llm-101-def-Transformer", "concept-rw-llm-101-self-attention" ], "requiredCorrect": 3 }, { "id": "d-context-params", "label": "Context window and parameters", "itemIds": [ "def-context-window", "def-parameters", "concept-rw-llm-101-context-window" ], "requiredCorrect": 3 }, { "id": "d-next-token", "label": "Next-token prediction", "itemIds": [ "def-next-token", "tf-d-llm-101-def-next-token" ] }, { "id": "d-autoregressive", "label": "Autoregressive generation", "itemIds": [ "def-autoregressive", "tf-df-llm-101-def-autoregressive" ] }, { "id": "d-logits", "label": "Logits", "itemIds": [ "def-logits", "cz-llm-101-def-logits", "tf-df-llm-101-def-logits" ] }, { "id": "d-temperature", "label": "Temperature", "itemIds": [ "def-temperature", "cz-llm-101-def-temperature", "tf-df-llm-101-def-temperature", "concept-rw-llm-101-temperature" ], "requiredCorrect": 3 }, { "id": "d-hallucination", "label": "Hallucination", "itemIds": [ "def-hallucination", "mcq-rw-l2-p2-generate", "cz-llm-101-def-hallucination", "concept-rw-llm-101-hallucination" ], "requiredCorrect": 3 }, { "id": "d-emergence", "label": "Emergent abilities", "itemIds": [ "def-emergence", "tf-d-llm-101-def-hallucination" ] } ], "objectiveTests": [ { "id": "test-text-to-numbers", "objectiveId": "obj-text-to-numbers", "title": "Tokens & Embeddings", "mcqIds": [ "mcq-rw-l2-p1-tokens", "ot-text-to-numbers-1", "ot-text-to-numbers-2", "ot-text-to-numbers-3", "ot-text-to-numbers-4" ] }, { "id": "test-model-internals", "objectiveId": "obj-model-internals", "title": "Inside the Model", "mcqIds": [ "ot-model-internals-1", "ot-model-internals-2", "ot-model-internals-3", "ot-model-internals-4", "ot-model-internals-5" ] }, { "id": "test-generation-loop", "objectiveId": "obj-generation-loop", "title": "The Generation Loop", "mcqIds": [ "ot-generation-loop-1", "ot-generation-loop-2", "ot-generation-loop-3", "ot-generation-loop-4", "ot-generation-loop-5" ] }, { "id": "test-sampling", "objectiveId": "obj-sampling", "title": "Logits & Temperature", "mcqIds": [ "ot-sampling-1", "ot-sampling-2", "ot-sampling-3", "ot-sampling-4", "ot-sampling-5" ] }, { "id": "test-hallucination-emergence", "objectiveId": "obj-hallucination-emergence", "title": "Hallucination & Emergence", "mcqIds": [ "mcq-rw-l2-p2-generate", "ot-halluc-emerge-1", "ot-halluc-emerge-2", "ot-halluc-emerge-3", "ot-halluc-emerge-4" ] } ] }, { "id": "l3-training", "title": "The Training Pipeline", "studyGuidePath": "/packs/llm-101/guides/l3-training.md", "parts": [ { "id": "l3-p1-pretrain", "title": "Pretraining to RLHF", "studyGuideAnchor": "pretraining-to-rlhf", "itemIds": [ "def-pretraining", "def-base-model", "def-sft", "def-rlhf", "def-reward-model", "def-rlaif", "mcq-rw-l3-p1-pretrain", "proc-fine-tune", "tf-d-llm-101-def-pretraining", "tf-d-llm-101-def-base-model", "tf-d-llm-101-def-sft", "tf-d-llm-101-def-reward-model", "concept-rw-llm-101-pretraining", "concept-rw-llm-101-supervised-fine-tuning", "concept-rw-llm-101-rlhf", "ot-pretrain-1", "ot-pretrain-2", "ot-pretrain-3", "ot-pretrain-4", "ot-sft-1", "ot-sft-2", "ot-sft-3", "ot-sft-4", "ot-sft-5", "ot-rlhf-1", "ot-rlhf-2", "ot-rlhf-3" ], "order": 1 }, { "id": "l3-p2-align", "title": "Alignment, inference and compute", "studyGuideAnchor": "alignment-inference-and-compute", "itemIds": [ "def-cai", "def-instruct-model", "f-train-vs-infer", "def-inference-llm", "def-flops", "mcq-rw-l3-p2-align", "cz-llm-101-def-inference-llm", "tf-df-llm-101-def-instruct-model", "tf-d-llm-101-def-inference-llm", "tf-df-llm-101-def-flops", "concept-rw-llm-101-constitutional-ai", "concept-rw-llm-101-instruct-or-chat-model", "ot-rlhf-4", "ot-rlhf-5", "ot-instruct-1", "ot-instruct-2", "ot-instruct-3", "ot-instruct-4", "ot-instruct-5", "ot-infer-1", "ot-infer-2", "ot-infer-3", "ot-infer-4", "fi-ot-rlhf-4", "fi-ot-rlhf-5", "fi-ot-instruct-1", "fi-ot-infer-2" ], "order": 2 } ], "order": 3, "objectives": [ { "id": "obj-pretraining-base", "statement": "Explain self-supervised pretraining and why the resulting base model is not yet a helpful assistant.", "demonstrationIds": [ "d-pretraining", "d-pretraining-tf" ] }, { "id": "obj-sft", "statement": "Describe supervised fine-tuning as training on prompt/response pairs to make a model follow instructions.", "demonstrationIds": [ "d-sft" ] }, { "id": "obj-rlhf", "statement": "Explain how RLHF and its reward model align a model, and how RLAIF and Constitutional AI scale that alignment.", "demonstrationIds": [ "d-rlhf", "d-rlaif" ] }, { "id": "obj-instruct-model", "statement": "Identify the instruct/chat model as the aligned product of the full base-plus-SFT-plus-RLHF pipeline.", "demonstrationIds": [ "d-instruct" ] }, { "id": "obj-training-vs-inference", "statement": "Distinguish training from inference and relate the FLOPs compute cost to why fine-tuning is costly but querying is cheap.", "demonstrationIds": [ "d-inference", "d-flops" ] } ], "demonstrations": [ { "id": "d-pretraining", "label": "Pretraining and base models", "itemIds": [ "def-pretraining", "def-base-model", "mcq-rw-l3-p1-pretrain", "concept-rw-llm-101-pretraining" ] }, { "id": "d-pretraining-tf", "label": "Pretraining/base-model checks", "itemIds": [ "tf-d-llm-101-def-pretraining", "tf-d-llm-101-def-base-model" ] }, { "id": "d-sft", "label": "Supervised fine-tuning", "itemIds": [ "def-sft", "tf-d-llm-101-def-sft", "concept-rw-llm-101-supervised-fine-tuning" ] }, { "id": "d-rlhf", "label": "RLHF and reward model", "itemIds": [ "def-rlhf", "def-reward-model", "concept-rw-llm-101-rlhf", "tf-d-llm-101-def-reward-model" ], "requiredCorrect": 3 }, { "id": "d-rlaif", "label": "RLAIF and Constitutional AI", "itemIds": [ "def-rlaif", "def-cai", "concept-rw-llm-101-constitutional-ai" ] }, { "id": "d-instruct", "label": "Instruct/chat model", "itemIds": [ "def-instruct-model", "tf-df-llm-101-def-instruct-model", "concept-rw-llm-101-instruct-or-chat-model" ] }, { "id": "d-inference", "label": "Training vs inference", "itemIds": [ "def-inference-llm", "mcq-rw-l3-p2-align", "cz-llm-101-def-inference-llm", "tf-d-llm-101-def-inference-llm" ], "requiredCorrect": 3 }, { "id": "d-flops", "label": "FLOPs / compute", "itemIds": [ "def-flops", "tf-df-llm-101-def-flops" ] } ], "objectiveTests": [ { "id": "test-pretraining-base", "objectiveId": "obj-pretraining-base", "title": "Pretraining & the Base Model", "mcqIds": [ "mcq-rw-l3-p1-pretrain", "ot-pretrain-1", "ot-pretrain-2", "ot-pretrain-3", "ot-pretrain-4" ] }, { "id": "test-sft", "objectiveId": "obj-sft", "title": "Supervised Fine-Tuning", "mcqIds": [ "ot-sft-1", "ot-sft-2", "ot-sft-3", "ot-sft-4", "ot-sft-5" ] }, { "id": "test-rlhf", "objectiveId": "obj-rlhf", "title": "RLHF, RLAIF & Constitutional AI", "mcqIds": [ "ot-rlhf-1", "ot-rlhf-2", "ot-rlhf-3", "ot-rlhf-4", "ot-rlhf-5" ] }, { "id": "test-instruct-model", "objectiveId": "obj-instruct-model", "title": "The Instruct / Chat Model", "mcqIds": [ "ot-instruct-1", "ot-instruct-2", "ot-instruct-3", "ot-instruct-4", "ot-instruct-5" ] }, { "id": "test-training-vs-inference", "objectiveId": "obj-training-vs-inference", "title": "Training vs. Inference & FLOPs", "mcqIds": [ "mcq-rw-l3-p2-align", "ot-infer-1", "ot-infer-2", "ot-infer-3", "ot-infer-4" ] } ] }, { "id": "l4-prompting", "title": "Prompting, RAG and Using LLMs", "studyGuidePath": "/packs/llm-101/guides/l4-p2-rag.md", "parts": [ { "id": "l4-p1-prompt", "title": "Prompting techniques", "studyGuideAnchor": "prompting-vs-rag-vs-fine-tuning", "itemIds": [ "def-prompt", "def-system-prompt", "def-zero-shot", "def-few-shot", "def-icl", "def-cot", "mcq-rw-l4-p1-prompt", "proc-prompt-engineer", "cz-llm-101-def-prompt", "tf-df-llm-101-def-prompt", "tf-df-llm-101-def-system-prompt", "tf-d-llm-101-def-few-shot", "tf-df-llm-101-def-icl", "ot-prompt-basics-1", "ot-prompt-basics-2", "ot-prompt-basics-3", "ot-prompt-basics-4", "ot-shots-1", "ot-shots-2", "ot-shots-3", "ot-shots-4", "ot-shots-5" ], "order": 1 }, { "id": "l4-p2-rag", "title": "RAG, tools and agents", "studyGuideAnchor": "rag-tools-and-agents", "itemIds": [ "def-rag", "def-vector-db", "def-chunking", "proc-rag", "f-prompt-rag-ft", "def-tool-use", "def-agent", "mcq-rw-l4-p2-rag", "proc-rag-pipeline", "tf-df-llm-101-def-rag", "tf-df-llm-101-def-vector-db", "tf-d-llm-101-def-chunking", "tf-df-llm-101-def-tool-use", "concept-rw-llm-101-rag", "concept-rw-llm-101-function-calling", "concept-rw-llm-101-llm-agent", "concept-rw-llm-101-vector-database", "concept-rw-llm-101-fine-tuning", "ot-rag-1", "ot-rag-2", "ot-rag-3", "ot-rag-4", "ot-rag-5", "ot-tools-1", "ot-tools-2", "ot-tools-3", "ot-tools-4", "ot-ftrag-1", "ot-ftrag-2", "ot-ftrag-3", "ot-ftrag-4", "ot-ftrag-5", "fi-ot-rag-2", "fi-ot-rag-3", "fi-mcq-rw-l4-p2-rag", "fi-ot-tools-1" ], "order": 2, "studyGuidePath": "/packs/llm-101/guides/l4-p2-rag.md" } ], "order": 4, "objectives": [ { "id": "obj-prompt-basics", "statement": "Define a prompt and the system prompt that sets a model's persistent behavior, and name the limit of prompting alone.", "demonstrationIds": [ "d-prompt", "d-prompt-tf" ] }, { "id": "obj-advanced-prompting", "statement": "Distinguish zero-shot, few-shot, in-context learning, and chain-of-thought prompting.", "demonstrationIds": [ "d-shots", "d-shots-tf" ] }, { "id": "obj-rag", "statement": "Explain RAG and its supporting components — vector databases and chunking — for grounding a model in external knowledge.", "demonstrationIds": [ "d-rag-def", "d-rag-tf" ] }, { "id": "obj-tools-agents", "statement": "Describe function calling and LLM agents, and how an agent's planning loop differs from a single-prompt interaction.", "demonstrationIds": [ "d-tools", "d-tools-things" ] }, { "id": "obj-fine-tuning-vs-rag", "statement": "Contrast fine-tuning with RAG as ways to add knowledge or behavior, one baking into weights and the other into context.", "demonstrationIds": [ "d-ft-rag" ] } ], "demonstrations": [ { "id": "d-prompt", "label": "Prompt and system prompt", "itemIds": [ "def-prompt", "def-system-prompt", "cz-llm-101-def-prompt", "mcq-rw-l4-p1-prompt" ] }, { "id": "d-prompt-tf", "label": "Prompt/system-prompt checks", "itemIds": [ "tf-df-llm-101-def-prompt", "tf-df-llm-101-def-system-prompt" ] }, { "id": "d-shots", "label": "Zero-shot, few-shot, ICL, CoT", "itemIds": [ "def-zero-shot", "def-few-shot", "def-icl", "def-cot" ], "requiredCorrect": 4 }, { "id": "d-shots-tf", "label": "Few-shot / ICL checks", "itemIds": [ "tf-d-llm-101-def-few-shot", "tf-df-llm-101-def-icl" ] }, { "id": "d-rag-def", "label": "RAG, vector DB, chunking", "itemIds": [ "def-rag", "def-vector-db", "def-chunking", "concept-rw-llm-101-rag" ] }, { "id": "d-rag-tf", "label": "RAG component checks", "itemIds": [ "tf-df-llm-101-def-rag", "tf-df-llm-101-def-vector-db", "tf-d-llm-101-def-chunking", "concept-rw-llm-101-vector-database" ], "requiredCorrect": 3 }, { "id": "d-tools", "label": "Function calling and agents", "itemIds": [ "def-tool-use", "def-agent", "mcq-rw-l4-p2-rag", "tf-df-llm-101-def-tool-use" ], "requiredCorrect": 3 }, { "id": "d-tools-things", "label": "Tool-use and agent concepts", "itemIds": [ "concept-rw-llm-101-function-calling", "concept-rw-llm-101-llm-agent" ] }, { "id": "d-ft-rag", "label": "Fine-tuning vs RAG", "itemIds": [ "concept-rw-llm-101-fine-tuning", "concept-rw-llm-101-rag", "def-rag" ] } ], "objectiveTests": [ { "id": "test-prompt-basics", "objectiveId": "obj-prompt-basics", "title": "Prompts and system prompts", "mcqIds": [ "mcq-rw-l4-p1-prompt", "ot-prompt-basics-1", "ot-prompt-basics-2", "ot-prompt-basics-3", "ot-prompt-basics-4" ] }, { "id": "test-advanced-prompting", "objectiveId": "obj-advanced-prompting", "title": "Zero-shot, few-shot and CoT", "mcqIds": [ "ot-shots-1", "ot-shots-2", "ot-shots-3", "ot-shots-4", "ot-shots-5" ] }, { "id": "test-rag", "objectiveId": "obj-rag", "title": "RAG, vector DBs and chunking", "mcqIds": [ "ot-rag-1", "ot-rag-2", "ot-rag-3", "ot-rag-4", "ot-rag-5" ] }, { "id": "test-tools-agents", "objectiveId": "obj-tools-agents", "title": "Function calling and agents", "mcqIds": [ "mcq-rw-l4-p2-rag", "ot-tools-1", "ot-tools-2", "ot-tools-3", "ot-tools-4" ] }, { "id": "test-fine-tuning-vs-rag", "objectiveId": "obj-fine-tuning-vs-rag", "title": "Fine-tuning vs. RAG", "mcqIds": [ "ot-ftrag-1", "ot-ftrag-2", "ot-ftrag-3", "ot-ftrag-4", "ot-ftrag-5" ] } ] }, { "id": "l5-people", "title": "Companies, People and Models", "studyGuidePath": "/packs/llm-101/guides/l5-people.md", "parts": [ { "id": "l5-p1-companies", "title": "The companies", "studyGuideAnchor": "the-companies", "itemIds": [ "pair-openai", "pair-anthropic", "pair-deepmind", "pair-meta", "pair-mistral", "pair-xai", "pair-cohere", "mcq-rw-l5-p1-companies", "concept-rw-llm-101-openai", "concept-rw-llm-101-anthropic", "concept-rw-llm-101-google-deepmind", "concept-rw-llm-101-meta-ai", "concept-rw-llm-101-mistral-ai", "ot-frontier-1", "ot-frontier-2", "ot-frontier-3", "ot-frontier-4", "ot-frontier-5", "ot-intl-1", "ot-intl-2", "ot-intl-3", "ot-intl-4", "ot-cov-frontier-labs-1", "ot-cov-frontier-labs-2", "ot-cov-second-tier-1", "ot-cov-second-tier-2", "ot-cov-second-tier-3", "ot-cov-second-tier-4" ], "order": 1 }, { "id": "l5-p2-researchers", "title": "The godfathers and key researchers", "studyGuideAnchor": "the-godfathers-and-key-researchers", "itemIds": [ "pair-vaswani", "pair-hinton", "pair-bengio", "pair-lecun", "pair-sutskever-llm", "pair-karpathy", "mcq-rw-l5-p2-researchers", "ot-god-1", "ot-god-2", "ot-god-3", "ot-god-4", "ot-res-1", "ot-res-2", "ot-res-3", "ot-res-4", "ot-res-5" ], "order": 2 }, { "id": "l5-p3-founders", "title": "Founders, leaders and the ecosystem", "studyGuideAnchor": "founders-leaders-and-the-ecosystem", "itemIds": [ "pair-altman-llm", "pair-dario", "pair-daniela", "pair-hassabis", "pair-nvidia", "pair-huggingface", "mcq-rw-l5-p3-founders", "ot-lead-1", "ot-lead-2", "ot-lead-3", "ot-lead-4", "ot-cov-leaders-ecosystem-1", "ot-cov-leaders-ecosystem-2", "ot-cov-leaders-ecosystem-3", "ot-cov-leaders-ecosystem-4", "ot-cov-leaders-ecosystem-5", "ot-cov-leaders-ecosystem-6" ], "order": 3 } ], "order": 5, "objectives": [ { "id": "obj-frontier-labs", "statement": "Match the US frontier labs (OpenAI, Anthropic, Meta, xAI) to their flagship model families.", "demonstrationIds": [ "d-us-labs", "d-us-things" ] }, { "id": "obj-second-tier", "statement": "Identify the international and second-tier labs (Google DeepMind, Mistral, Cohere) and the lab that drove open-source LLMs.", "demonstrationIds": [ "d-intl-labs", "d-intl-things" ] }, { "id": "obj-godfathers", "statement": "Recognize the three 'godfathers of deep learning' who shared the 2018 Turing Award.", "demonstrationIds": [ "d-godfathers" ] }, { "id": "obj-key-researchers", "statement": "Identify pivotal LLM researchers — Vaswani, Sutskever, and Karpathy — and their contributions.", "demonstrationIds": [ "d-researchers" ] }, { "id": "obj-leaders-ecosystem", "statement": "Match today's lab leaders (Altman, the Amodeis, Hassabis) and the infrastructure players (Nvidia, Hugging Face).", "demonstrationIds": [ "d-leaders", "d-ecosystem" ] } ], "demonstrations": [ { "id": "d-us-labs", "label": "US frontier labs", "itemIds": [ "pair-openai", "pair-anthropic", "pair-meta", "pair-xai", "ot-cov-frontier-labs-1", "ot-cov-frontier-labs-2" ], "requiredCorrect": 3 }, { "id": "d-us-things", "label": "US lab concepts", "itemIds": [ "concept-rw-llm-101-openai", "concept-rw-llm-101-anthropic", "concept-rw-llm-101-meta-ai" ] }, { "id": "d-intl-labs", "label": "International/second-tier labs", "itemIds": [ "pair-deepmind", "pair-mistral", "pair-cohere", "mcq-rw-l5-p1-companies", "ot-cov-second-tier-1", "ot-cov-second-tier-2", "ot-cov-second-tier-3", "ot-cov-second-tier-4" ], "requiredCorrect": 3 }, { "id": "d-intl-things", "label": "DeepMind and Mistral concepts", "itemIds": [ "concept-rw-llm-101-google-deepmind", "concept-rw-llm-101-mistral-ai" ] }, { "id": "d-godfathers", "label": "Godfathers of deep learning", "itemIds": [ "pair-hinton", "pair-bengio", "pair-lecun", "mcq-rw-l5-p2-researchers" ], "requiredCorrect": 3 }, { "id": "d-researchers", "label": "Vaswani, Sutskever, Karpathy", "itemIds": [ "pair-vaswani", "pair-sutskever-llm", "pair-karpathy" ] }, { "id": "d-leaders", "label": "Lab leaders", "itemIds": [ "pair-altman-llm", "pair-dario", "pair-daniela", "pair-hassabis", "ot-cov-leaders-ecosystem-1", "ot-cov-leaders-ecosystem-2", "ot-cov-leaders-ecosystem-3", "ot-cov-leaders-ecosystem-4", "ot-cov-leaders-ecosystem-5", "ot-cov-leaders-ecosystem-6" ], "requiredCorrect": 3 }, { "id": "d-ecosystem", "label": "Infrastructure ecosystem", "itemIds": [ "pair-nvidia", "pair-huggingface", "mcq-rw-l5-p3-founders" ] } ], "objectiveTests": [ { "id": "test-frontier-labs", "objectiveId": "obj-frontier-labs", "title": "US Frontier Labs & Their Models", "mcqIds": [ "ot-frontier-1", "ot-frontier-2", "ot-frontier-3", "ot-frontier-4", "ot-frontier-5", "ot-cov-frontier-labs-1", "ot-cov-frontier-labs-2" ] }, { "id": "test-second-tier", "objectiveId": "obj-second-tier", "title": "International & Second-Tier Labs", "mcqIds": [ "mcq-rw-l5-p1-companies", "ot-intl-1", "ot-intl-2", "ot-intl-3", "ot-intl-4", "ot-cov-second-tier-1", "ot-cov-second-tier-2", "ot-cov-second-tier-3", "ot-cov-second-tier-4" ] }, { "id": "test-godfathers", "objectiveId": "obj-godfathers", "title": "Godfathers of Deep Learning", "mcqIds": [ "mcq-rw-l5-p2-researchers", "ot-god-1", "ot-god-2", "ot-god-3", "ot-god-4" ] }, { "id": "test-key-researchers", "objectiveId": "obj-key-researchers", "title": "Pivotal LLM Researchers", "mcqIds": [ "ot-res-1", "ot-res-2", "ot-res-3", "ot-res-4", "ot-res-5" ] }, { "id": "test-leaders-ecosystem", "objectiveId": "obj-leaders-ecosystem", "title": "Lab Leaders & the Ecosystem", "mcqIds": [ "mcq-rw-l5-p3-founders", "ot-lead-1", "ot-lead-2", "ot-lead-3", "ot-lead-4", "ot-cov-leaders-ecosystem-1", "ot-cov-leaders-ecosystem-2", "ot-cov-leaders-ecosystem-3", "ot-cov-leaders-ecosystem-4", "ot-cov-leaders-ecosystem-5" ] } ] }, { "id": "l6-numbers", "title": "Key Numbers and Milestones", "studyGuidePath": "/packs/llm-101/guides/l6-numbers.md", "parts": [ { "id": "l6-p1-sizes", "title": "Model sizes and dates", "studyGuideAnchor": "model-sizes-and-dates", "itemIds": [ "num-gpt1-llm", "num-gpt2", "num-gpt3-llm", "num-gpt3-year", "num-chatgpt-year", "num-chatgpt-100m", "num-gpt4-year", "num-gpt3-data", "mcq-rw-l6-p1-sizes", "num-llm-bert-large", "num-gpt1-params", "num-gpt2-params", "num-gpt3-params", "num-llama-7b", "num-llama-70b", "num-mistral-7b", "num-palm-540b", "num-bert-released", "num-gpt2-released", "num-chatgpt-released", "num-llama-released", "num-gpt4-released", "ot-gpt-sizes-1", "ot-gpt-sizes-2", "ot-gpt-sizes-3", "ot-gpt-sizes-4", "ot-gpt-sizes-5", "ot-other-sizes-1", "ot-other-sizes-2", "ot-other-sizes-3", "ot-other-sizes-4", "ot-other-sizes-5", "ot-dates-1", "ot-dates-2", "ot-dates-3", "ot-dates-4", "ot-chatgpt-1", "ot-chatgpt-2", "ot-chatgpt-3", "ot-chatgpt-4", "ot-chatgpt-5", "ot-cov-other-model-sizes-1", "ot-cov-other-model-sizes-2", "ot-cov-milestone-dates-1", "ot-cov-milestone-dates-2", "ot-cov-milestone-dates-3", "ot-cov-milestone-dates-4", "ot-cov-milestone-dates-5", "ot-cov-milestone-dates-6" ], "order": 1 }, { "id": "l6-p2-scaling", "title": "Scaling, context and misconceptions", "studyGuideAnchor": "scaling-context-and-misconceptions", "itemIds": [ "num-chinchilla", "num-kaplan-year", "num-ctx-gpt3", "num-ctx-gpt4", "mcq-rw-l6-p2-scaling", "concept-rw-llm-101-instructgpt", "ot-cov-scaling-context-1", "ot-cov-scaling-context-2", "ot-cov-scaling-context-3", "ot-cov-scaling-context-4" ], "order": 2 } ], "order": 6, "objectives": [ { "id": "obj-gpt-sizes", "statement": "Recall the parameter counts of the GPT line (GPT-1, GPT-2, GPT-3) and GPT-3's training-data size.", "demonstrationIds": [ "d-gpt-params", "d-gpt-params-2" ] }, { "id": "obj-other-model-sizes", "statement": "Recall the parameter counts of other landmark models — BERT-large, LLaMA, Mistral 7B, and PaLM.", "demonstrationIds": [ "d-other-params", "d-palm-bert" ] }, { "id": "obj-milestone-dates", "statement": "Place the key release years in order: BERT, GPT-2, GPT-3, LLaMA, ChatGPT, and GPT-4.", "demonstrationIds": [ "d-dates", "d-dates-2" ] }, { "id": "obj-chatgpt-adoption", "statement": "Recall the ChatGPT and GPT-4 launch years and ChatGPT's record two-month climb to 100 million users.", "demonstrationIds": [ "d-chatgpt" ] }, { "id": "obj-scaling-context", "statement": "Recall the scaling-law figures (Kaplan year, Chinchilla's ~20 tokens/parameter) and context-window growth, and why bigger isn't always better.", "demonstrationIds": [ "d-scaling-nums", "d-misconception" ] } ], "demonstrations": [ { "id": "d-gpt-params", "label": "GPT-1/2/3 sizes and data", "itemIds": [ "num-gpt1-llm", "num-gpt2", "num-gpt3-llm", "num-gpt3-data" ], "requiredCorrect": 4 }, { "id": "d-gpt-params-2", "label": "GPT parameter counts", "itemIds": [ "num-gpt1-params", "num-gpt2-params", "num-gpt3-params" ], "requiredCorrect": 2 }, { "id": "d-other-params", "label": "BERT, LLaMA, Mistral sizes", "itemIds": [ "num-llm-bert-large", "num-llama-7b", "num-llama-70b", "num-mistral-7b", "ot-cov-other-model-sizes-1", "ot-cov-other-model-sizes-2" ], "requiredCorrect": 3 }, { "id": "d-palm-bert", "label": "PaLM and BERT-large sizes", "itemIds": [ "num-palm-540b", "num-llm-bert-large" ] }, { "id": "d-dates", "label": "Model release years", "itemIds": [ "num-bert-released", "num-gpt2-released", "num-gpt3-year", "num-gpt4-released", "ot-cov-milestone-dates-1", "ot-cov-milestone-dates-2", "ot-cov-milestone-dates-3", "ot-cov-milestone-dates-4", "ot-cov-milestone-dates-5", "ot-cov-milestone-dates-6" ], "requiredCorrect": 3 }, { "id": "d-dates-2", "label": "ChatGPT/LLaMA years and ordering", "itemIds": [ "num-chatgpt-released", "num-llama-released", "mcq-rw-l6-p1-sizes" ] }, { "id": "d-chatgpt", "label": "ChatGPT/GPT-4 launch and adoption", "itemIds": [ "num-chatgpt-year", "num-chatgpt-100m", "num-gpt4-year" ], "requiredCorrect": 3 }, { "id": "d-scaling-nums", "label": "Scaling and context numbers", "itemIds": [ "num-chinchilla", "num-kaplan-year", "num-ctx-gpt3", "num-ctx-gpt4", "ot-cov-scaling-context-1", "ot-cov-scaling-context-2", "ot-cov-scaling-context-3", "ot-cov-scaling-context-4" ], "requiredCorrect": 3 }, { "id": "d-misconception", "label": "Bigger-isn't-always-better", "itemIds": [ "mcq-rw-l6-p2-scaling", "concept-rw-llm-101-instructgpt" ] } ], "objectiveTests": [ { "id": "test-gpt-sizes", "objectiveId": "obj-gpt-sizes", "title": "GPT line parameter counts", "mcqIds": [ "ot-gpt-sizes-1", "ot-gpt-sizes-2", "ot-gpt-sizes-3", "ot-gpt-sizes-4", "ot-gpt-sizes-5" ] }, { "id": "test-other-model-sizes", "objectiveId": "obj-other-model-sizes", "title": "Landmark model sizes", "mcqIds": [ "ot-other-sizes-1", "ot-other-sizes-2", "ot-other-sizes-3", "ot-other-sizes-4", "ot-other-sizes-5", "ot-cov-other-model-sizes-1", "ot-cov-other-model-sizes-2" ] }, { "id": "test-milestone-dates", "objectiveId": "obj-milestone-dates", "title": "Release years in order", "mcqIds": [ "mcq-rw-l6-p1-sizes", "ot-dates-1", "ot-dates-2", "ot-dates-3", "ot-dates-4", "ot-cov-milestone-dates-1", "ot-cov-milestone-dates-2", "ot-cov-milestone-dates-3", "ot-cov-milestone-dates-4", "ot-cov-milestone-dates-5" ] }, { "id": "test-chatgpt-adoption", "objectiveId": "obj-chatgpt-adoption", "title": "ChatGPT launch and adoption", "mcqIds": [ "ot-chatgpt-1", "ot-chatgpt-2", "ot-chatgpt-3", "ot-chatgpt-4", "ot-chatgpt-5" ] }, { "id": "test-scaling-context", "objectiveId": "obj-scaling-context", "title": "Scaling laws and context windows", "mcqIds": [ "mcq-rw-l6-p2-scaling", "ot-scaling-1", "ot-scaling-2", "ot-scaling-3", "ot-scaling-4", "ot-cov-scaling-context-1", "ot-cov-scaling-context-2", "ot-cov-scaling-context-3", "ot-cov-scaling-context-4" ] } ] }, { "id": "l8-use-cases", "title": "What LLMs Are Good For", "studyGuidePath": "/packs/llm-101/guides/l8-p2-knowledge-code.md", "parts": [ { "id": "l8-p1-language", "title": "Language and writing tasks", "studyGuideAnchor": "knowledge-search-and-code", "itemIds": [ "x-uc-summarization", "x-uc-translation", "x-uc-rewriting", "x-uc-editing", "x-uc-classification", "x-uc-accessibility", "mcq-rw-l8-p1-language", "ot-lang-summarization-catch", "ot-lang-not-a-strength", "ot-lang-classification-catch", "ot-lang-plain-language", "ot-cov-obj-language-tasks-1", "ot-cov-obj-language-tasks-2", "ot-cov-obj-language-tasks-3", "ot-cov-obj-language-tasks-4", "ot-cov-obj-language-tasks-5", "ot-cov-obj-language-tasks-6" ], "order": 1 }, { "id": "l8-p2-knowledge-code", "title": "Knowledge, search and code", "studyGuideAnchor": "knowledge-search-and-code", "itemIds": [ "x-uc-doc-qa", "x-uc-research", "x-uc-semantic-search", "x-uc-extraction", "x-uc-codegen", "x-uc-num-copilot", "x-uc-explanation", "x-uc-debugging", "x-uc-testing", "mcq-rw-l8-p2-knowledge-code", "ot-kc-copilot-speedup", "ot-kc-junior-gain", "ot-kc-semantic-search", "ot-kc-test-writing-catch", "fi-mcq-rw-l8-p2-knowledge-code", "fi-ot-kc-semantic-search", "fi-ot-kc-test-writing-catch", "ot-cov-obj-knowledge-code-1", "ot-cov-obj-knowledge-code-2", "ot-cov-obj-knowledge-code-3", "ot-cov-obj-knowledge-code-4", "ot-cov-obj-knowledge-code-5", "ot-cov-obj-knowledge-code-6" ], "order": 2, "studyGuidePath": "/packs/llm-101/guides/l8-p2-knowledge-code.md" }, { "id": "l8-p3-work", "title": "Work and everyday tasks", "studyGuideAnchor": "knowledge-search-and-code", "itemIds": [ "x-uc-support", "x-uc-drafting", "x-uc-meeting-notes", "x-uc-data-cleaning", "x-uc-tutoring", "x-uc-brainstorming", "mcq-rw-l8-p3-work", "ot-work-meeting-notes", "ot-work-support-catch", "ot-work-tutoring-catch", "ot-work-not-listed", "ot-cov-obj-work-tasks-1", "ot-cov-obj-work-tasks-2", "ot-cov-obj-work-tasks-3", "ot-cov-obj-work-tasks-4", "ot-cov-obj-work-tasks-5", "ot-cov-obj-work-tasks-6" ], "order": 3 }, { "id": "l8-p4-highstakes", "title": "High-stakes domains and choosing well", "studyGuideAnchor": "high-stakes-domains-and-choosing-well", "itemIds": [ "x-uc-healthcare", "x-uc-legal", "x-uc-finance", "x-uc-copilot-def", "x-uc-hitl", "x-uc-proc", "x-uc-good-fit", "x-uc-poor-fit", "mcq-rw-l8-p4-highstakes", "czr-llm-101-x-uc-copilot-def", "tf-df-llm-101-x-uc-copilot-def", "concept-rw-llm-101-healthcare", "concept-rw-llm-101-legal", "concept-rw-llm-101-finance", "ot-cop-definition", "ot-cop-contrast", "ot-cop-apply-automation", "ot-cop-failure-mode", "ot-fit-good-fit", "ot-fit-not-poor-fit", "ot-dom-legal-sanction", "ot-dom-healthcare-review", "ot-dom-finance-practice", "fi-ot-cop-definition", "ot-cov-obj-fit-domains-1", "ot-cov-obj-fit-domains-2", "ot-cov-obj-fit-domains-3", "ot-cov-obj-fit-domains-4", "ot-cov-obj-fit-domains-5", "ot-cov-obj-fit-domains-6" ], "order": 4, "studyGuidePath": "/packs/llm-101/guides/l8-p4-highstakes.md" } ], "order": 7, "objectives": [ { "id": "obj-language-tasks", "statement": "Identify LLM strengths across language tasks — summarization, translation, rewriting, editing, classification, and simplification.", "demonstrationIds": [ "d-transform", "d-improve" ] }, { "id": "obj-knowledge-code", "statement": "Explain LLM knowledge/search use cases (research, semantic search, extraction, RAG) and coding assistance (Copilot speedup, explanation, debugging, tests).", "demonstrationIds": [ "d-knowledge", "d-coding" ] }, { "id": "obj-work-tasks", "statement": "Identify everyday work and personal tasks LLMs assist — support, drafting, meeting notes, data cleaning, tutoring, brainstorming — with their catches.", "demonstrationIds": [ "d-work", "d-personal" ] }, { "id": "obj-copilot-pattern", "statement": "Apply the copilot pattern and the cost-of-verification framework to decide how much automation a task warrants.", "demonstrationIds": [ "d-copilot" ] }, { "id": "obj-fit-domains", "statement": "Judge good vs poor LLM fit and the human-review guardrails required in healthcare, legal, and finance.", "demonstrationIds": [ "d-fit", "d-domains" ] } ], "demonstrations": [ { "id": "d-transform", "label": "Text-transforming tasks", "itemIds": [ "x-uc-summarization", "x-uc-translation", "x-uc-accessibility", "mcq-rw-l8-p1-language", "ot-cov-obj-language-tasks-1", "ot-cov-obj-language-tasks-2", "ot-cov-obj-language-tasks-3", "ot-cov-obj-language-tasks-4", "ot-cov-obj-language-tasks-5", "ot-cov-obj-language-tasks-6" ], "requiredCorrect": 3 }, { "id": "d-improve", "label": "Improve/classify tasks", "itemIds": [ "x-uc-rewriting", "x-uc-editing", "x-uc-classification" ], "requiredCorrect": 2 }, { "id": "d-knowledge", "label": "Knowledge and search", "itemIds": [ "x-uc-research", "x-uc-semantic-search", "x-uc-extraction", "mcq-rw-l8-p2-knowledge-code", "ot-cov-obj-knowledge-code-1", "ot-cov-obj-knowledge-code-2", "ot-cov-obj-knowledge-code-3", "ot-cov-obj-knowledge-code-4", "ot-cov-obj-knowledge-code-5", "ot-cov-obj-knowledge-code-6" ], "requiredCorrect": 3 }, { "id": "d-coding", "label": "Coding assistance", "itemIds": [ "x-uc-num-copilot", "x-uc-explanation", "x-uc-debugging", "x-uc-testing" ], "requiredCorrect": 3 }, { "id": "d-work", "label": "Business/work tasks", "itemIds": [ "x-uc-support", "x-uc-drafting", "x-uc-meeting-notes", "x-uc-data-cleaning", "ot-cov-obj-work-tasks-1", "ot-cov-obj-work-tasks-2", "ot-cov-obj-work-tasks-3", "ot-cov-obj-work-tasks-4", "ot-cov-obj-work-tasks-5", "ot-cov-obj-work-tasks-6" ], "requiredCorrect": 3 }, { "id": "d-personal", "label": "Personal tasks and limits", "itemIds": [ "x-uc-tutoring", "x-uc-brainstorming", "mcq-rw-l8-p3-work" ] }, { "id": "d-copilot", "label": "Copilot pattern and verification", "itemIds": [ "x-uc-copilot-def", "czr-llm-101-x-uc-copilot-def", "tf-df-llm-101-x-uc-copilot-def", "mcq-rw-l8-p4-highstakes" ], "requiredCorrect": 4 }, { "id": "d-fit", "label": "Good vs poor fit", "itemIds": [ "x-uc-good-fit", "x-uc-poor-fit", "ot-cov-obj-fit-domains-1", "ot-cov-obj-fit-domains-2", "ot-cov-obj-fit-domains-3", "ot-cov-obj-fit-domains-4", "ot-cov-obj-fit-domains-5", "ot-cov-obj-fit-domains-6" ] }, { "id": "d-domains", "label": "High-stakes domains", "itemIds": [ "concept-rw-llm-101-healthcare", "concept-rw-llm-101-legal", "concept-rw-llm-101-finance" ], "requiredCorrect": 2 } ], "objectiveTests": [ { "id": "test-language-tasks", "objectiveId": "obj-language-tasks", "title": "Language and Writing Tasks", "mcqIds": [ "mcq-rw-l8-p1-language", "ot-lang-summarization-catch", "ot-lang-not-a-strength", "ot-lang-classification-catch", "ot-lang-plain-language", "ot-cov-obj-language-tasks-1", "ot-cov-obj-language-tasks-2", "ot-cov-obj-language-tasks-3", "ot-cov-obj-language-tasks-4", "ot-cov-obj-language-tasks-5" ] }, { "id": "test-knowledge-code", "objectiveId": "obj-knowledge-code", "title": "Knowledge, Search and Code", "mcqIds": [ "mcq-rw-l8-p2-knowledge-code", "ot-kc-copilot-speedup", "ot-kc-junior-gain", "ot-kc-semantic-search", "ot-kc-test-writing-catch", "ot-cov-obj-knowledge-code-1", "ot-cov-obj-knowledge-code-2", "ot-cov-obj-knowledge-code-3", "ot-cov-obj-knowledge-code-4", "ot-cov-obj-knowledge-code-5" ] }, { "id": "test-work-tasks", "objectiveId": "obj-work-tasks", "title": "Work and Everyday Tasks", "mcqIds": [ "mcq-rw-l8-p3-work", "ot-work-meeting-notes", "ot-work-support-catch", "ot-work-tutoring-catch", "ot-work-not-listed", "ot-cov-obj-work-tasks-1", "ot-cov-obj-work-tasks-2", "ot-cov-obj-work-tasks-3", "ot-cov-obj-work-tasks-4", "ot-cov-obj-work-tasks-5" ] }, { "id": "test-copilot-pattern", "objectiveId": "obj-copilot-pattern", "title": "Copilot Pattern and Verification", "mcqIds": [ "mcq-rw-l8-p4-highstakes", "ot-cop-definition", "ot-cop-contrast", "ot-cop-apply-automation", "ot-cop-failure-mode" ] }, { "id": "test-fit-domains", "objectiveId": "obj-fit-domains", "title": "Fit and High-Stakes Guardrails", "mcqIds": [ "ot-fit-good-fit", "ot-fit-not-poor-fit", "ot-dom-legal-sanction", "ot-dom-healthcare-review", "ot-dom-finance-practice", "ot-cov-obj-fit-domains-1", "ot-cov-obj-fit-domains-2", "ot-cov-obj-fit-domains-3", "ot-cov-obj-fit-domains-4", "ot-cov-obj-fit-domains-5" ] } ] }, { "id": "l9-limits", "title": "Limits and Responsible Use", "studyGuidePath": "/packs/llm-101/guides/l9-p1-reliability.md", "parts": [ { "id": "l9-p1-reliability", "title": "Reliability and staleness", "studyGuideAnchor": "reliability-and-staleness", "itemIds": [ "x-lim-halluc-cost", "x-lim-halluc-fix", "x-lim-confident", "x-lim-cutoff", "x-lim-staleness", "mcq-rw-l9-p1-reliability", "tf-d-llm-101-x-lim-cutoff", "concept-rw-llm-101-knowledge-cutoff", "ot-halluc-cutoff-1", "ot-halluc-confident-2", "ot-halluc-toolkit-3", "ot-halluc-staleness-4", "fi-mcq-rw-l9-p1-reliability" ], "order": 1, "studyGuidePath": "/packs/llm-101/guides/l9-p1-reliability.md" }, { "id": "l9-p2-bias-privacy", "title": "Bias, privacy and your data", "studyGuideAnchor": "bias-privacy-and-your-data", "itemIds": [ "x-lim-bias-source", "x-lim-bias-shows", "x-lim-bias-hard", "x-lim-data-paste", "x-lim-memorization", "q-llm-bias-stages", "q-llm-bias-aave-gap", "q-llm-paste-consumer", "mcq-rw-l9-p2-bias-privacy", "tf-t-llm-101-q-llm-bias-stages", "tf-f-llm-101-q-llm-bias-aave-gap", "ot-bias-stages-1", "ot-bias-aave-2", "ot-bias-forms-3", "ot-bias-source-4", "ot-bias-hard-5", "ot-privacy-paste-1", "ot-privacy-enterprise-2", "ot-privacy-memorize-3", "ot-privacy-rule-4" ], "order": 2, "studyGuidePath": "/packs/llm-101/guides/l9-p2-bias-privacy.md" }, { "id": "l9-p3-security", "title": "Security and societal risks", "studyGuideAnchor": "security-and-societal-risks", "itemIds": [ "x-lim-injection", "x-lim-jailbreak", "x-lim-exfiltration", "x-lim-agents-surface", "x-lim-misinfo", "x-lim-automation-bias", "x-lim-copyright", "mcq-rw-l9-p3-security", "tf-d-llm-101-x-lim-injection", "tf-d-llm-101-x-lim-jailbreak", "tf-d-llm-101-x-lim-automation-bias", "concept-rw-llm-101-prompt-injection", "concept-rw-llm-101-automation-bias", "ot-sec-injection-1", "ot-sec-jailbreak-2", "ot-sec-automation-3", "ot-sec-agent-fix-4", "fi-ot-sec-injection-1" ], "order": 3, "studyGuidePath": "/packs/llm-101/guides/l9-p3-security.md" }, { "id": "l9-p4-governance", "title": "Limits, safeguards and deployment", "studyGuideAnchor": "limits-safeguards-and-deployment", "itemIds": [ "x-lim-math", "x-lim-nondeterminism", "x-lim-cost", "x-lim-when-not", "x-lim-redteam", "x-lim-modelcard", "x-lim-guardrails", "x-lim-proc-deploy", "x-lim-alignment", "mcq-rw-l9-p4-governance", "proc-deploy-llm", "tf-df-llm-101-x-lim-redteam", "concept-rw-llm-101-non-determinism", "ot-gov-redteam-1", "ot-gov-modelcard-2", "ot-gov-nondet-3", "ot-gov-oversight-4" ], "order": 4, "studyGuidePath": "/packs/llm-101/guides/l9-p4-governance.md" } ], "order": 8, "objectives": [ { "id": "obj-hallucination-staleness", "statement": "Explain why LLMs hallucinate and what a knowledge cutoff means for stale, confidently-stated answers.", "demonstrationIds": [ "d-hallucination" ] }, { "id": "obj-bias", "statement": "Identify where LLM bias enters (data collection, pretraining, fine-tuning) and how it shows up, such as the AAVE accuracy gap.", "demonstrationIds": [ "d-bias" ] }, { "id": "obj-privacy", "statement": "Describe what can happen to data pasted into a consumer LLM and the memorization/PII-extraction risk.", "demonstrationIds": [ "d-privacy" ] }, { "id": "obj-security", "statement": "Explain prompt injection and jailbreaks, automation bias, and why tool-enabled agents widen the attack surface.", "demonstrationIds": [ "d-injection", "d-jailbreak-automation" ] }, { "id": "obj-governance", "statement": "Recognize governance safeguards — red-teaming, model cards, and non-determinism — and why filters alone don't make a model safe for high-stakes use.", "demonstrationIds": [ "d-governance", "d-transparency" ] } ], "demonstrations": [ { "id": "d-hallucination", "label": "Hallucination and knowledge cutoff", "itemIds": [ "mcq-rw-l9-p1-reliability", "x-lim-cutoff", "tf-d-llm-101-x-lim-cutoff", "concept-rw-llm-101-knowledge-cutoff" ], "requiredCorrect": 4 }, { "id": "d-bias", "label": "Sources and forms of bias", "itemIds": [ "q-llm-bias-stages", "q-llm-bias-aave-gap", "tf-t-llm-101-q-llm-bias-stages", "tf-f-llm-101-q-llm-bias-aave-gap" ], "requiredCorrect": 3 }, { "id": "d-privacy", "label": "Data handling and PII", "itemIds": [ "q-llm-paste-consumer", "mcq-rw-l9-p2-bias-privacy" ] }, { "id": "d-injection", "label": "Prompt injection and agents", "itemIds": [ "x-lim-injection", "mcq-rw-l9-p3-security", "tf-d-llm-101-x-lim-injection", "concept-rw-llm-101-prompt-injection" ], "requiredCorrect": 4 }, { "id": "d-jailbreak-automation", "label": "Jailbreaks and automation bias", "itemIds": [ "x-lim-jailbreak", "tf-d-llm-101-x-lim-jailbreak", "x-lim-automation-bias", "concept-rw-llm-101-automation-bias" ], "requiredCorrect": 3 }, { "id": "d-governance", "label": "Red-teaming and model cards", "itemIds": [ "x-lim-redteam", "x-lim-modelcard", "mcq-rw-l9-p4-governance", "tf-df-llm-101-x-lim-redteam" ], "requiredCorrect": 4 }, { "id": "d-transparency", "label": "Model cards and non-determinism", "itemIds": [ "x-lim-modelcard", "concept-rw-llm-101-non-determinism" ] } ], "objectiveTests": [ { "id": "test-hallucination-staleness", "objectiveId": "obj-hallucination-staleness", "title": "Hallucination & Knowledge Cutoff", "mcqIds": [ "mcq-rw-l9-p1-reliability", "ot-halluc-cutoff-1", "ot-halluc-confident-2", "ot-halluc-toolkit-3", "ot-halluc-staleness-4" ] }, { "id": "test-bias", "objectiveId": "obj-bias", "title": "Where LLM Bias Enters", "mcqIds": [ "ot-bias-stages-1", "ot-bias-aave-2", "ot-bias-forms-3", "ot-bias-source-4", "ot-bias-hard-5" ] }, { "id": "test-privacy", "objectiveId": "obj-privacy", "title": "Your Data and Memorization", "mcqIds": [ "mcq-rw-l9-p2-bias-privacy", "ot-privacy-paste-1", "ot-privacy-enterprise-2", "ot-privacy-memorize-3", "ot-privacy-rule-4" ] }, { "id": "test-security", "objectiveId": "obj-security", "title": "Injection, Jailbreaks, Agents", "mcqIds": [ "mcq-rw-l9-p3-security", "ot-sec-injection-1", "ot-sec-jailbreak-2", "ot-sec-automation-3", "ot-sec-agent-fix-4" ] }, { "id": "test-governance", "objectiveId": "obj-governance", "title": "Safeguards and Governance", "mcqIds": [ "mcq-rw-l9-p4-governance", "ot-gov-redteam-1", "ot-gov-modelcard-2", "ot-gov-nondet-3", "ot-gov-oversight-4" ] } ] }, { "id": "l7-recall", "title": "Quick Recall", "studyGuidePath": "/packs/llm-101/guides/l7-recall.md", "parts": [ { "id": "l7-p1-recall", "title": "Core terms", "studyGuideAnchor": "core-terms", "itemIds": [ "recall-transformer", "recall-token", "recall-pretraining", "recall-rlhf", "recall-rag", "recall-inference", "recall-hallucination", "recall-context", "recall-chinchilla", "recall-fewshot", "recall-attention", "recall-emergence", "mcq-rw-l7-p1-recall", "tf-t-llm-101-recall-transformer", "tf-t-llm-101-recall-pretraining", "tf-f-llm-101-recall-rlhf", "tf-f-llm-101-recall-rag", "tf-t-llm-101-recall-hallucination", "tf-t-llm-101-recall-context", "concept-rw-llm-101-token", "concept-rw-llm-101-inference", "ot-arch-token-size", "ot-arch-context-window", "ot-arch-self-attention", "ot-arch-odd-one-out", "ot-train-pretraining-def", "ot-train-rlhf-role", "ot-train-chinchilla-ratio", "ot-train-chinchilla-apply", "ot-train-pretraining-not", "ot-usage-rag-apply", "ot-usage-inference-def", "ot-usage-hallucination-def", "ot-usage-fewshot-def", "ot-usage-emergence-def", "ot-cov-architecture-terms-1", "ot-cov-architecture-terms-2", "ot-cov-obj-training-terms-1", "ot-cov-obj-training-terms-2", "ot-cov-obj-usage-terms-1", "ot-cov-obj-usage-terms-2", "ot-cov-obj-usage-terms-3", "ot-cov-obj-usage-terms-4", "ot-cov-obj-usage-terms-5", "ot-cov-obj-usage-terms-6" ], "order": 1 } ], "order": 9, "objectives": [ { "id": "obj-architecture-terms", "statement": "Recall the core architecture terms — Transformer, token, self-attention, and context window — instantly.", "demonstrationIds": [ "d-arch", "d-arch-check" ] }, { "id": "obj-training-terms", "statement": "Recall the training terms — pretraining, RLHF, and Chinchilla's ~20-tokens-per-parameter rule.", "demonstrationIds": [ "d-train", "d-train-check" ] }, { "id": "obj-usage-terms", "statement": "Recall the usage terms — RAG, inference, hallucination, few-shot, and emergence.", "demonstrationIds": [ "d-usage", "d-usage-check" ] } ], "demonstrations": [ { "id": "d-arch", "label": "Architecture terms", "itemIds": [ "recall-transformer", "recall-token", "recall-attention", "recall-context", "ot-cov-architecture-terms-1", "ot-cov-architecture-terms-2" ], "requiredCorrect": 3 }, { "id": "d-arch-check", "label": "Architecture recall checks", "itemIds": [ "tf-t-llm-101-recall-transformer", "tf-t-llm-101-recall-context", "concept-rw-llm-101-token", "mcq-rw-l7-p1-recall" ], "requiredCorrect": 3 }, { "id": "d-train", "label": "Training terms", "itemIds": [ "recall-pretraining", "recall-rlhf", "recall-chinchilla", "ot-cov-obj-training-terms-1", "ot-cov-obj-training-terms-2" ], "requiredCorrect": 2 }, { "id": "d-train-check", "label": "Training recall checks", "itemIds": [ "tf-t-llm-101-recall-pretraining", "tf-f-llm-101-recall-rlhf" ] }, { "id": "d-usage", "label": "Usage terms", "itemIds": [ "recall-rag", "recall-inference", "recall-hallucination", "recall-fewshot", "ot-cov-obj-usage-terms-1", "ot-cov-obj-usage-terms-2", "ot-cov-obj-usage-terms-3", "ot-cov-obj-usage-terms-4", "ot-cov-obj-usage-terms-5", "ot-cov-obj-usage-terms-6" ], "requiredCorrect": 3 }, { "id": "d-usage-check", "label": "Usage recall checks", "itemIds": [ "recall-emergence", "tf-f-llm-101-recall-rag", "tf-t-llm-101-recall-hallucination", "concept-rw-llm-101-inference" ], "requiredCorrect": 3 } ], "objectiveTests": [ { "id": "test-architecture-terms", "objectiveId": "obj-architecture-terms", "title": "Architecture Terms", "mcqIds": [ "mcq-rw-l7-p1-recall", "ot-arch-token-size", "ot-arch-context-window", "ot-arch-self-attention", "ot-arch-odd-one-out", "ot-cov-architecture-terms-1", "ot-cov-architecture-terms-2" ] }, { "id": "test-training-terms", "objectiveId": "obj-training-terms", "title": "Training Terms", "mcqIds": [ "ot-train-pretraining-def", "ot-train-rlhf-role", "ot-train-chinchilla-ratio", "ot-train-chinchilla-apply", "ot-train-pretraining-not", "ot-cov-obj-training-terms-1", "ot-cov-obj-training-terms-2" ] }, { "id": "test-usage-terms", "objectiveId": "obj-usage-terms", "title": "Usage Terms", "mcqIds": [ "ot-usage-rag-apply", "ot-usage-inference-def", "ot-usage-hallucination-def", "ot-usage-fewshot-def", "ot-usage-emergence-def", "ot-cov-obj-usage-terms-1", "ot-cov-obj-usage-terms-2", "ot-cov-obj-usage-terms-3", "ot-cov-obj-usage-terms-4", "ot-cov-obj-usage-terms-5" ] } ] }, { "id": "l-key-concepts", "title": "Key Concepts", "order": 99, "parts": [ { "id": "lc-people", "title": "Key People", "order": 0, "itemIds": [ "concept-yoshua-bengio", "concept-geoffrey-hinton", "concept-yann-lecun", "concept-ilya-sutskever", "concept-andrew-ng", "concept-alec-radford", "mcq-rw-lc-people", "ot-godfathers-hinton", "ot-godfathers-lecun", "ot-godfathers-bengio", "ot-godfathers-not", "ot-openai-sutskever-ssi", "ot-openai-radford-gpt", "ot-openai-ng-brain", "ot-openai-ng-coursera", "ot-openai-sutskever-seq2seq", "ot-cov-openai-figures-1", "ot-cov-openai-figures-2", "ot-cov-openai-figures-3", "ot-cov-openai-figures-4", "ot-cov-openai-figures-5", "ot-cov-openai-figures-6" ] }, { "id": "lc-things", "title": "Key Things", "order": 1, "itemIds": [ "concept-word2vec", "concept-bert", "concept-gpt-3", "concept-transformer", "concept-lstm", "concept-chatgpt", "mcq-rw-lc-things", "ot-found-word2vec-def", "ot-found-word2vec-origin", "ot-found-transformer-attn", "ot-found-lstm-gradient", "ot-found-lstm-era", "ot-models-bert-training", "ot-models-gpt3-fewshot", "ot-models-chatgpt-rlhf", "ot-models-recency", "ot-cov-foundational-things-1", "ot-cov-foundational-things-2", "ot-cov-foundational-things-3", "ot-cov-foundational-things-4", "ot-cov-foundational-things-5", "ot-cov-foundational-things-6", "ot-cov-landmark-models-1", "ot-cov-landmark-models-2", "ot-cov-landmark-models-3", "ot-cov-landmark-models-4" ] } ], "studyGuidePath": "/packs/llm-101/guides/l-key-concepts.md", "objectives": [ { "id": "obj-turing-godfathers", "statement": "Recognize the three Turing-Award 'godfathers of deep learning' — Bengio, Hinton, and LeCun.", "demonstrationIds": [ "d-godfathers" ] }, { "id": "obj-openai-figures", "statement": "Identify key researchers behind modern LLMs — Sutskever and Radford at OpenAI, and educator Andrew Ng.", "demonstrationIds": [ "d-people" ] }, { "id": "obj-foundational-things", "statement": "Recognize foundational techniques and architectures — word2vec, LSTM, and the Transformer.", "demonstrationIds": [ "d-foundations" ] }, { "id": "obj-landmark-models", "statement": "Recognize landmark models — BERT, GPT-3, and ChatGPT — including GPT-3's 175-billion-parameter scale.", "demonstrationIds": [ "d-models" ] } ], "demonstrations": [ { "id": "d-godfathers", "label": "Godfathers of deep learning", "itemIds": [ "concept-yoshua-bengio", "concept-geoffrey-hinton", "concept-yann-lecun", "mcq-rw-lc-people" ], "requiredCorrect": 3 }, { "id": "d-people", "label": "OpenAI researchers and educator", "itemIds": [ "concept-ilya-sutskever", "concept-alec-radford", "concept-andrew-ng", "ot-cov-openai-figures-1", "ot-cov-openai-figures-2", "ot-cov-openai-figures-3", "ot-cov-openai-figures-4", "ot-cov-openai-figures-5", "ot-cov-openai-figures-6" ], "requiredCorrect": 2 }, { "id": "d-foundations", "label": "Foundational techniques", "itemIds": [ "concept-word2vec", "concept-lstm", "concept-transformer", "ot-cov-foundational-things-1", "ot-cov-foundational-things-2", "ot-cov-foundational-things-3", "ot-cov-foundational-things-4", "ot-cov-foundational-things-5", "ot-cov-foundational-things-6" ], "requiredCorrect": 2 }, { "id": "d-models", "label": "Landmark models", "itemIds": [ "concept-bert", "concept-gpt-3", "concept-chatgpt", "mcq-rw-lc-things", "ot-cov-landmark-models-1", "ot-cov-landmark-models-2", "ot-cov-landmark-models-3", "ot-cov-landmark-models-4" ], "requiredCorrect": 3 } ], "objectiveTests": [ { "id": "test-turing-godfathers", "objectiveId": "obj-turing-godfathers", "title": "The Deep Learning Godfathers", "mcqIds": [ "mcq-rw-lc-people", "ot-godfathers-hinton", "ot-godfathers-lecun", "ot-godfathers-bengio", "ot-godfathers-not" ] }, { "id": "test-openai-figures", "objectiveId": "obj-openai-figures", "title": "Researchers Behind Modern LLMs", "mcqIds": [ "ot-openai-sutskever-ssi", "ot-openai-radford-gpt", "ot-openai-ng-brain", "ot-openai-ng-coursera", "ot-openai-sutskever-seq2seq", "ot-cov-openai-figures-1", "ot-cov-openai-figures-2", "ot-cov-openai-figures-3", "ot-cov-openai-figures-4", "ot-cov-openai-figures-5" ] }, { "id": "test-foundational-things", "objectiveId": "obj-foundational-things", "title": "Foundational Techniques", "mcqIds": [ "ot-found-word2vec-def", "ot-found-word2vec-origin", "ot-found-transformer-attn", "ot-found-lstm-gradient", "ot-found-lstm-era", "ot-cov-foundational-things-1", "ot-cov-foundational-things-2", "ot-cov-foundational-things-3", "ot-cov-foundational-things-4", "ot-cov-foundational-things-5" ] }, { "id": "test-landmark-models", "objectiveId": "obj-landmark-models", "title": "Landmark Models", "mcqIds": [ "mcq-rw-lc-things", "ot-models-bert-training", "ot-models-gpt3-fewshot", "ot-models-chatgpt-rlhf", "ot-models-recency", "ot-cov-landmark-models-1", "ot-cov-landmark-models-2", "ot-cov-landmark-models-3", "ot-cov-landmark-models-4" ] } ] } ] }