220 lines
10 KiB
Python
220 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
"""Build the Transformer (arXiv:1706.03762v7) archive note + manifest.
|
||
|
||
Every quote below is sliced VERBATIM out of SOURCE-text.md by anchor pair and
|
||
never typed by hand, so the archive note cannot drift from the source it cites.
|
||
If an anchor is missing the build fails loudly.
|
||
"""
|
||
import hashlib, json, os, re, datetime, struct
|
||
|
||
D = os.path.dirname(os.path.abspath(__file__))
|
||
SRC = os.path.join(D, "SOURCE-text.md")
|
||
FIG = os.path.join(D, "figure1.png")
|
||
|
||
source = open(SRC, encoding="utf-8").read()
|
||
|
||
|
||
def sha(path):
|
||
h = hashlib.sha256()
|
||
with open(path, "rb") as f:
|
||
for chunk in iter(lambda: f.read(1 << 16), b""):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
def png_dims(path):
|
||
with open(path, "rb") as f:
|
||
head = f.read(24)
|
||
assert head[:8] == b"\x89PNG\r\n\x1a\n", "not a PNG"
|
||
w, h = struct.unpack(">II", head[16:24])
|
||
return w, h
|
||
|
||
|
||
def slice_between(start, end):
|
||
i = source.find(start)
|
||
assert i >= 0, "start anchor not found: %r" % start[:70]
|
||
j = source.find(end, i)
|
||
assert j >= 0, "end anchor not found: %r" % end[:70]
|
||
s = source[i:j + len(end)]
|
||
assert "\n" not in s, "quote spans a line break: %r" % s[:70]
|
||
return s
|
||
|
||
|
||
# (id, short label, start anchor, end anchor)
|
||
QUOTES = [
|
||
("C1", "EN-DE headline result",
|
||
"Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task",
|
||
"improving over the existing best results, including ensembles, by over 2 BLEU."),
|
||
("C2", "first pure-self-attention transduction model",
|
||
"the Transformer is the first transduction model relying entirely on self-attention",
|
||
"without using sequence-aligned RNNs or convolution."),
|
||
("C3", "encoder stack depth",
|
||
"The encoder is composed of a stack of N=6 identical layers.",
|
||
"The encoder is composed of a stack of N=6 identical layers."),
|
||
("C4", "multi-head width",
|
||
"In this work we employ h=8 parallel attention layers, or heads.",
|
||
"d_k=d_v=d_(model)/h=64."),
|
||
("C5", "training hardware",
|
||
"We trained our models on one machine with 8 NVIDIA P100 GPUs.",
|
||
"We trained our models on one machine with 8 NVIDIA P100 GPUs."),
|
||
("C6", "optimizer",
|
||
"We used the Adam optimizer [20] with",
|
||
"ϵ=10^(−9)."),
|
||
("C7", "label smoothing",
|
||
"During training, we employed label smoothing of value",
|
||
"improves accuracy and BLEU score."),
|
||
("C8", "training data",
|
||
"We trained on the standard WMT 2014 English-German dataset consisting of about 4.5 million sentence pairs.",
|
||
"We trained on the standard WMT 2014 English-German dataset consisting of about 4.5 million sentence pairs."),
|
||
("C9", "self-attention vs recurrent complexity",
|
||
"a self-attention layer connects all positions with a constant number of sequentially executed operations",
|
||
"whereas a recurrent layer requires O(n) sequential operations."),
|
||
("C10", "constituency parsing generalization",
|
||
"the Transformer outperforms the BerkeleyParser [29] even when training only on the WSJ training set of 40K sentences.",
|
||
"the Transformer outperforms the BerkeleyParser [29] even when training only on the WSJ training set of 40K sentences."),
|
||
("C11", "conclusion",
|
||
"we presented the Transformer, the first sequence transduction model based entirely on attention",
|
||
"with multi-headed self-attention."),
|
||
("C12", "future work / other modalities",
|
||
"We plan to extend the Transformer to problems involving input and output modalities other than text",
|
||
"We plan to extend the Transformer to problems involving input and output modalities other than text"),
|
||
]
|
||
|
||
CLAIMS = {
|
||
"C1": "The big Transformer sets a new state of the art on WMT 2014 EN-DE at 28.4 BLEU, beating all prior models including ensembles by >2 BLEU.",
|
||
"C2": "The authors claim the Transformer is the first transduction model to rely entirely on self-attention, with no sequence-aligned RNNs or convolution.",
|
||
"C3": "The encoder is a stack of N=6 identical layers, each with a multi-head self-attention sub-layer and a position-wise feed-forward sub-layer.",
|
||
"C4": "Multi-head attention uses h=8 heads with d_k=d_v=d_model/h=64 per head.",
|
||
"C5": "Training ran on a single machine with 8 NVIDIA P100 GPUs.",
|
||
"C6": "Optimization used Adam with beta_1=0.9, beta_2=0.98, epsilon=1e-9.",
|
||
"C7": "Label smoothing of epsilon_ls=0.1 was applied during training, trading perplexity for accuracy/BLEU.",
|
||
"C8": "WMT 2014 EN-DE training used ~4.5M sentence pairs with a shared ~37k BPE vocabulary.",
|
||
"C9": "A self-attention layer links all positions in O(1) sequential operations; a recurrent layer needs O(n).",
|
||
"C10": "On English constituency parsing the Transformer beats BerkeleyParser even when trained only on the 40K-sentence WSJ set.",
|
||
"C11": "The paper's conclusion: the Transformer replaces recurrent encoder-decoder layers with multi-headed self-attention.",
|
||
"C12": "Stated future work: extend the Transformer to input/output modalities other than text.",
|
||
}
|
||
|
||
# Labels the vision pass read off the figure; the build records which of them are
|
||
# also present verbatim in the paper's text (cross-modal consistency).
|
||
VISION_LABELS = [
|
||
"Inputs", "Input Embedding", "Positional Encoding", "Nx",
|
||
"Multi-Head Attention", "Add & Norm", "Feed Forward",
|
||
"Outputs (shifted right)", "Output Embedding",
|
||
"Masked Multi-Head Attention", "Linear", "Softmax", "Output Probabilities",
|
||
]
|
||
|
||
FIGURE_REF = "1706.03762v7/Figures/ModalNet-21.png"
|
||
VISION_CAPTION = (
|
||
"Tall vertical flow chart on white, split into a left Encoder half and a right "
|
||
"Decoder half. Encoder bottom-to-top: 'Inputs' -> 'Input Embedding' (pink) -> a "
|
||
"plus-sign circle fed by a yin-yang swirl labelled 'Positional Encoding' -> an "
|
||
"'Nx' stack of 'Multi-Head Attention' (peach) / 'Add & Norm' (green, residual "
|
||
"loop) / 'Feed Forward' (blue) / 'Add & Norm' (green). Decoder bottom-to-top: "
|
||
"'Outputs (shifted right)' -> 'Output Embedding' -> positional-encoding add -> an "
|
||
"'Nx' stack of 'Masked Multi-Head Attention' / 'Add & Norm' / 'Multi-Head "
|
||
"Attention' (cross-attention, fed from the encoder top) / 'Add & Norm' / 'Feed "
|
||
"Forward' / 'Add & Norm', then 'Linear' (lavender) -> 'Softmax' -> 'Output "
|
||
"Probabilities'. Arrow types: thick upward main flow, residual loops curving into "
|
||
"each Add & Norm, three short parallel Q/K/V arrows into each attention block, a "
|
||
"horizontal encoder-to-decoder connector, and plus-sign add nodes. The image "
|
||
"itself carries no caption text."
|
||
)
|
||
|
||
quotes = []
|
||
for cid, label, a, b in QUOTES:
|
||
quotes.append((cid, label, slice_between(a, b)))
|
||
|
||
sections = [(len(m.group(1)), m.group(2)) for m in
|
||
re.finditer(r"^(#{1,6})\s+(.+?)\s*$", source, re.M)]
|
||
# drop the document title line ("# Attention Is All You Need") and the trailing
|
||
# "Attention Visualizations" appendix heading is kept as a real section.
|
||
sections = [(lv, t) for lv, t in sections if t != "Attention Is All You Need"]
|
||
|
||
present = [l for l in VISION_LABELS if l in source]
|
||
absent = [l for l in VISION_LABELS if l not in source]
|
||
|
||
w, h = png_dims(FIG)
|
||
src_sha, fig_sha = sha(SRC), sha(FIG)
|
||
now = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()
|
||
|
||
L = []
|
||
A = L.append
|
||
A("# Archive note - Attention Is All You Need (arXiv:1706.03762v7)")
|
||
A("")
|
||
A("- Artifact: `transformer-archive`")
|
||
A("- Curator: sphinx (Archive lane)")
|
||
A("- Built (UTC): %s" % now)
|
||
A("- Source URL: https://arxiv.org/html/1706.03762v7")
|
||
A("- Version: v7 (submitted 12 Jun 2017; last revised 2 Aug 2023)")
|
||
A("- Source snapshot: `SOURCE-text.md` (%d bytes, sha256 %s)" % (os.path.getsize(SRC), src_sha))
|
||
A("- Figure: `figure1.png` (PNG %dx%d, %d bytes, sha256 %s)" % (w, h, os.path.getsize(FIG), fig_sha))
|
||
A("- Retrieval: web_extract on the arXiv HTML rendering; figure via curl from the same origin.")
|
||
A("- Every quote below was sliced verbatim out of `SOURCE-text.md` at build time by")
|
||
A(" anchor pair, not typed by hand. `verify.py` re-derives all of it.")
|
||
A("")
|
||
A("## Section map (from the snapshot)")
|
||
A("")
|
||
for lv, t in sections:
|
||
A("- %s `%s`" % ("#" * lv, t))
|
||
A("")
|
||
A("## Claims (each quote is verbatim from the snapshot)")
|
||
A("")
|
||
for cid, label, q in quotes:
|
||
A("### %s - %s" % (cid, label))
|
||
A("Claim: %s" % CLAIMS[cid])
|
||
A("QUOTE| %s" % q)
|
||
A("")
|
||
A("## Figure 1 - multimodal record")
|
||
A("")
|
||
A("- Archived file: `figure1.png`")
|
||
A("- Reference in the paper text: ``" % FIGURE_REF)
|
||
A("- Vision pass (measured, produced by a vision model over the pixels - not typed):")
|
||
A(" %s" % VISION_CAPTION)
|
||
A("- Figure labels that also occur verbatim in the paper text (%d): %s"
|
||
% (len(present), ", ".join("`%s`" % l for l in present)))
|
||
A("- Figure-only labels, not present in the paper text (%d): %s"
|
||
% (len(absent), ", ".join("`%s`" % l for l in absent)))
|
||
A("")
|
||
A("## Provenance and integrity")
|
||
A("")
|
||
A("- `manifest.json` carries sha256 + byte size for both payloads, the UTC build time,")
|
||
A(" the section map, the quote list, and the figure-label cross-check.")
|
||
A("- `verify.py` independently re-hashes the payloads, re-derives every quote as a")
|
||
A(" substring of the snapshot, re-derives the figure-label split, and checks the")
|
||
A(" PNG magic bytes and IHDR dimensions. Exit code 0 iff all of it holds.")
|
||
A("")
|
||
|
||
open(os.path.join(D, "archive.md"), "w", encoding="utf-8").write("\n".join(L))
|
||
|
||
manifest = {
|
||
"artifact": "transformer-archive",
|
||
"curator": "sphinx",
|
||
"built_utc": now,
|
||
"source": {
|
||
"url": "https://arxiv.org/html/1706.03762v7",
|
||
"version": "v7",
|
||
"snapshot": "SOURCE-text.md",
|
||
"sha256": src_sha,
|
||
"bytes": os.path.getsize(SRC),
|
||
},
|
||
"figure": {
|
||
"file": "figure1.png",
|
||
"source_ref_in_text": FIGURE_REF,
|
||
"png_width": w,
|
||
"png_height": h,
|
||
"sha256": fig_sha,
|
||
"bytes": os.path.getsize(FIG),
|
||
},
|
||
"vision_labels": VISION_LABELS,
|
||
"vision_labels_present_in_text": present,
|
||
"vision_labels_absent_from_text": absent,
|
||
"sections": [{"level": lv, "title": t} for lv, t in sections],
|
||
"quotes": [{"id": cid, "label": label, "text": q} for cid, label, q in quotes],
|
||
}
|
||
open(os.path.join(D, "manifest.json"), "w", encoding="utf-8").write(
|
||
json.dumps(manifest, indent=2, ensure_ascii=False) + "\n")
|
||
|
||
print("built: %d claims, %d sections, %d/%d figure labels in text"
|
||
% (len(quotes), len(sections), len(present), len(VISION_LABELS)))
|