diagrams/assets/dropout/dropout.typ at main · janosh/diagrams · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
#import "@preview/cetz:0.5.2": canvas, draw
#import draw: circle, content, line

#set page(width: auto, height: auto, margin: 8pt, fill: none)

#canvas({
  let spacing = (layer: 2.5, node: 1.5)
  let arrow-style = (mark: (end: "stealth", scale: 0.7), fill: black)

  // Helper function to draw a layer of nodes
  let draw-layer(x, nodes, prefix: "") = {
    for ii in range(nodes) {
      circle(
        (x, spacing.node * (ii + 1)),
        radius: 0.3,
        name: prefix + str(ii + 1),
      )
    }
  }

  // Helper to connect all nodes between layers
  let connect-layers(from-prefix, to-prefix, from-nodes, to-nodes) = {
    for ii in range(from-nodes) {
      for jj in range(to-nodes) {
        line(
          (from-prefix + str(ii + 1)),
          (to-prefix + str(jj + 1)),
          ..arrow-style,
        )
      }
    }
  }

  // Left network (fully connected)
  // Draw all layers
  draw-layer(0, 5, prefix: "i") // Input layer
  draw-layer(spacing.layer, 5, prefix: "h1") // First hidden layer
  draw-layer(2 * spacing.layer, 5, prefix: "h2") // Second hidden layer

  // Draw output nodes
  circle((3 * spacing.layer, 2 * spacing.node), radius: 0.3, name: "o1")
  circle((3 * spacing.layer, 4 * spacing.node), radius: 0.3, name: "o2")

  // Connect all layers
  connect-layers("i", "h1", 5, 5)
  connect-layers("h1", "h2", 5, 5)

  // Connect to output nodes
  for ii in range(5) {
    line(("h2" + str(ii + 1)), "o1", ..arrow-style)
    line(("h2" + str(ii + 1)), "o2", ..arrow-style)
  }

  // Draw dropout arrow
  let mid-x = 4 * spacing.layer
  line(
    (3.5 * spacing.layer, 3 * spacing.node),
    (4.5 * spacing.layer, 3 * spacing.node),
    ..arrow-style,
    name: "dropout-arrow",
  )
  content(
    "dropout-arrow.mid",
    text(weight: "bold", size: 1.2em)[dropout],
    anchor: "south",
    padding: 3pt,
  )

  // Right network (with dropout)
  // Draw all layers
  draw-layer(mid-x + spacing.layer, 5, prefix: "di")
  draw-layer(mid-x + 2 * spacing.layer, 5, prefix: "dh1")
  draw-layer(mid-x + 3 * spacing.layer, 5, prefix: "dh2")

  // Draw output nodes
  circle(
    (mid-x + 4 * spacing.layer, 2 * spacing.node),
    radius: 0.3,
    name: "do1",
  )
  circle(
    (mid-x + 4 * spacing.layer, 4 * spacing.node),
    radius: 0.3,
    name: "do2",
  )

  // Add dropout X marks
  let x-style = (fill: red, weight: "bold", size: 4em, baseline: -4pt)
  content("di1", text(..x-style)[×])
  content("di3", text(..x-style)[×])
  content("dh11", text(..x-style)[×])
  content("dh13", text(..x-style)[×])
  content("dh14", text(..x-style)[×])
  content("dh22", text(..x-style)[×])
  content("dh24", text(..x-style)[×])

  // Connect remaining nodes (after dropout)
  for ii in (2, 4, 5) {
    for jj in (2, 5) {
      line(("di" + str(ii)), ("dh1" + str(jj)), ..arrow-style)
    }
  }

  for ii in (2, 5) {
    for jj in (1, 3, 5) {
      line(("dh1" + str(ii)), ("dh2" + str(jj)), ..arrow-style)
    }
  }

  for ii in (1, 3, 5) {
    line(("dh2" + str(ii)), "do1", ..arrow-style)
    line(("dh2" + str(ii)), "do2", ..arrow-style)
  }
})