troupe-standup/wizard-work/sphinx/transformer-archive/manifest.json

{
  "artifact": "transformer-archive",
  "curator": "sphinx",
  "built_utc": "2026-10-20T20:47:46+00:00",
  "source": {
    "url": "https://arxiv.org/html/1706.03762v7",
    "version": "v7",
    "snapshot": "SOURCE-text.md",
    "sha256": "8e0403fcc8d11dad28ab64ffa9edc302279c2b818ff45f64f6754b256b983161",
    "bytes": 36700
  },
  "figure": {
    "file": "figure1.png",
    "source_ref_in_text": "1706.03762v7/Figures/ModalNet-21.png",
    "png_width": 1520,
    "png_height": 2239,
    "sha256": "56d69ccc192b90c3b93e5e285ff892f74753f970103939b40291945e106e9f79",
    "bytes": 159152
  },
  "vision_labels": [
    "Inputs",
    "Input Embedding",
    "Positional Encoding",
    "Nx",
    "Multi-Head Attention",
    "Add & Norm",
    "Feed Forward",
    "Outputs (shifted right)",
    "Output Embedding",
    "Masked Multi-Head Attention",
    "Linear",
    "Softmax",
    "Output Probabilities"
  ],
  "vision_labels_present_in_text": [
    "Positional Encoding",
    "Multi-Head Attention",
    "Softmax"
  ],
  "vision_labels_absent_from_text": [
    "Inputs",
    "Input Embedding",
    "Nx",
    "Add & Norm",
    "Feed Forward",
    "Outputs (shifted right)",
    "Output Embedding",
    "Masked Multi-Head Attention",
    "Linear",
    "Output Probabilities"
  ],
  "sections": [
    {
      "level": 6,
      "title": "Abstract"
    },
    {
      "level": 2,
      "title": "1 Introduction"
    },
    {
      "level": 2,
      "title": "2 Background"
    },
    {
      "level": 2,
      "title": "3 Model Architecture"
    },
    {
      "level": 3,
      "title": "3.1 Encoder and Decoder Stacks"
    },
    {
      "level": 5,
      "title": "Encoder:"
    },
    {
      "level": 5,
      "title": "Decoder:"
    },
    {
      "level": 3,
      "title": "3.2 Attention"
    },
    {
      "level": 4,
      "title": "3.2.1 Scaled Dot-Product Attention"
    },
    {
      "level": 4,
      "title": "3.2.2 Multi-Head Attention"
    },
    {
      "level": 4,
      "title": "3.2.3 Applications of Attention in our Model"
    },
    {
      "level": 3,
      "title": "3.3 Position-wise Feed-Forward Networks"
    },
    {
      "level": 3,
      "title": "3.4 Embeddings and Softmax"
    },
    {
      "level": 3,
      "title": "3.5 Positional Encoding"
    },
    {
      "level": 2,
      "title": "4 Why Self-Attention"
    },
    {
      "level": 2,
      "title": "5 Training"
    },
    {
      "level": 3,
      "title": "5.1 Training Data and Batching"
    },
    {
      "level": 3,
      "title": "5.2 Hardware and Schedule"
    },
    {
      "level": 3,
      "title": "5.3 Optimizer"
    },
    {
      "level": 3,
      "title": "5.4 Regularization"
    },
    {
      "level": 5,
      "title": "Residual Dropout"
    },
    {
      "level": 5,
      "title": "Label Smoothing"
    },
    {
      "level": 2,
      "title": "6 Results"
    },
    {
      "level": 3,
      "title": "6.1 Machine Translation"
    },
    {
      "level": 3,
      "title": "6.2 Model Variations"
    },
    {
      "level": 3,
      "title": "6.3 English Constituency Parsing"
    },
    {
      "level": 2,
      "title": "7 Conclusion"
    },
    {
      "level": 5,
      "title": "Acknowledgements"
    },
    {
      "level": 2,
      "title": "References"
    },
    {
      "level": 2,
      "title": "Attention Visualizations"
    }
  ],
  "quotes": [
    {
      "id": "C1",
      "label": "EN-DE headline result",
      "text": "Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task, improving over the existing best results, including ensembles, by over 2 BLEU."
    },
    {
      "id": "C2",
      "label": "first pure-self-attention transduction model",
      "text": "the Transformer is the first transduction model relying entirely on self-attention to compute representations of its input and output without using sequence-aligned RNNs or convolution."
    },
    {
      "id": "C3",
      "label": "encoder stack depth",
      "text": "The encoder is composed of a stack of N=6 identical layers."
    },
    {
      "id": "C4",
      "label": "multi-head width",
      "text": "In this work we employ h=8 parallel attention layers, or heads. For each of these we use d_k=d_v=d_(model)/h=64."
    },
    {
      "id": "C5",
      "label": "training hardware",
      "text": "We trained our models on one machine with 8 NVIDIA P100 GPUs."
    },
    {
      "id": "C6",
      "label": "optimizer",
      "text": "We used the Adam optimizer [20] with β_1=0.9, β_2=0.98 and ϵ=10^(−9)."
    },
    {
      "id": "C7",
      "label": "label smoothing",
      "text": "During training, we employed label smoothing of value ϵ_(ls)=0.1 [36]. This hurts perplexity, as the model learns to be more unsure, but improves accuracy and BLEU score."
    },
    {
      "id": "C8",
      "label": "training data",
      "text": "We trained on the standard WMT 2014 English-German dataset consisting of about 4.5 million sentence pairs."
    },
    {
      "id": "C9",
      "label": "self-attention vs recurrent complexity",
      "text": "a self-attention layer connects all positions with a constant number of sequentially executed operations, whereas a recurrent layer requires O(n) sequential operations."
    },
    {
      "id": "C10",
      "label": "constituency parsing generalization",
      "text": "the Transformer outperforms the BerkeleyParser [29] even when training only on the WSJ training set of 40K sentences."
    },
    {
      "id": "C11",
      "label": "conclusion",
      "text": "we presented the Transformer, the first sequence transduction model based entirely on attention, replacing the recurrent layers most commonly used in encoder-decoder architectures with multi-headed self-attention."
    },
    {
      "id": "C12",
      "label": "future work / other modalities",
      "text": "We plan to extend the Transformer to problems involving input and output modalities other than text"
    }
  ]
}