𝑇𝑟𝑖𝑎𝑛𝑔𝑙𝑒 𝑖𝑛𝑒𝑞𝑢𝑎𝑙𝑖𝑡𝑦: ‖𝑥 + 𝑦‖ ≤ ‖𝑥‖ + ‖𝑦‖ 𝑅(𝐴)⨁𝑁(𝐴𝑇 ) = 𝑅𝑚

1⁄
𝑛 𝑝 𝑟𝑎𝑛𝑘(𝐴) = dim𝑅(𝐴𝑇 ) = 𝑑𝑖𝑚𝑅(𝐴)
𝑁𝑜𝑟𝑚: ‖𝑥‖𝑝 ≐ (∑|𝑥𝑘 |𝑝 ) , 1≤𝑝<∞ = 𝑖𝑛𝑑𝑒𝑝𝑒𝑛𝑑𝑒𝑛𝑡 𝑐𝑜𝑙𝑢𝑚𝑛𝑠
𝑘=1 = 𝑖𝑛𝑑𝑒𝑝𝑒𝑛𝑑𝑒𝑛𝑡 𝑟𝑜𝑤𝑠
𝐶ℎ𝑒𝑏𝑦𝑠ℎ𝑒𝑣 𝑛𝑜𝑟𝑚: ‖𝑥‖∞ ≐ max |𝑥𝑘 | 𝑆𝑞𝑢𝑎𝑟𝑒 𝑚𝑎𝑡𝑟𝑖𝑥:
𝑘=1,…,𝑛
𝑁𝑜 𝑜𝑓 𝑛𝑜𝑛𝑧𝑒𝑟𝑜 𝑒𝑙𝑒𝑚𝑒𝑛𝑡𝑠: 𝑐𝑎𝑟𝑑(𝑥) = ‖𝑥‖0 𝑡𝑟𝑎𝑐𝑒𝐴 = 𝑡𝑟𝑎𝑐𝑒𝐴𝑇
𝑛 𝑡𝑟𝑎𝑐𝑒𝐴𝐵 = 𝑡𝑟𝑎𝑐𝑒𝐵𝐴
1 𝑖𝑓 𝑥𝑘 ≠ 0 detA2𝑥2 = 𝑎𝑑 − 𝑏𝑐
≐ ∑ Ι , 𝑤ℎ𝑒𝑟𝑒 Ι ≐ {
0 𝑜𝑡ℎ𝑒𝑟𝑤𝑖𝑠𝑒 𝑑𝑒𝑡𝐴 = 𝑑𝑒𝑡𝐴𝑇
𝑘=1
𝑥𝑇𝑦 𝑑𝑒𝑡𝐴𝐵 = 𝑑𝑒𝑡𝐵𝐴 = 𝑑𝑒𝑡𝐴𝑑𝑒𝑡𝐵
𝐴𝑛𝑔𝑙𝑒: 𝑐𝑜𝑠𝜃 = 𝑑𝑒𝑡𝛼𝐴 = 𝛼 𝑛 𝑑𝑒𝑡𝐴
‖𝑥‖2 ‖𝑦‖2
𝐶𝑎𝑢𝑐ℎ𝑦 − 𝑆𝑐ℎ𝑤𝑎𝑟𝑡𝑧 𝑖𝑛𝑒𝑞𝑢𝑎𝑙𝑖𝑡𝑦: |𝑥 𝑇 𝑦| ≤ ‖𝑥‖2 ‖𝑦‖2 𝑆𝑖𝑛𝑔𝑢𝑙𝑎𝑟 ⇔ 𝑑𝑒𝑡𝐴 = 0 ⇔ 𝑁(𝐴) ≠ {0}
|𝑥 𝑇 𝑦| ≤ ‖𝑥‖1 ‖𝑦‖∞ 𝑂𝑟𝑡ℎ𝑜𝑔𝑜𝑛𝑎𝑙 𝑚𝑎𝑡𝑟𝑖𝑥: 𝑈 𝑇 𝑈 = 𝑈𝑈 𝑇 = Ι𝑛
1 1 1 = 𝑑𝑒𝑡Ι𝑛 = 𝑑𝑒𝑡𝑈 𝑇 𝑈 = (𝑑𝑒𝑡𝑈)2
|𝑥 𝑇 𝑦| ≤ ‖𝑥‖𝑝 ‖𝑦‖𝑞 𝑤ℎ𝑒𝑟𝑒 + = 1 𝐼𝑛𝑣𝑒𝑟𝑠𝑒 ⇔ 𝑛𝑜𝑛 − 𝑠𝑖𝑛𝑔𝑢𝑙𝑎𝑟:
𝑝 𝑞
〈𝑣, 𝑥〉 (𝐴𝐵)−1 = 𝐵 −1 𝐴−1
𝑃𝑟𝑜𝑗𝑒𝑐𝑡𝑖𝑜𝑛: 𝑥𝑣 = 𝛼𝑣, 𝛼 = (𝐴𝑇 )−1 = (𝐴−1 )𝑇
‖𝑣‖2
𝐷𝑖𝑠𝑡𝑎𝑛𝑐𝑒 𝑓𝑟𝑜𝑚 𝑝 𝑡𝑜 𝑡ℎ𝑒 𝑙𝑖𝑛𝑒 𝐿 = { 𝑥0 + 𝑠𝑝𝑎𝑛(𝑢)}: 1
𝑑𝑒𝑡𝐴 = 𝑑𝑒𝑡𝐴𝑇 =
‖𝑝 − 𝑝∗ ‖22 = ‖𝑝 − 𝑥0 ‖22 − 𝜆∗2 , 𝜆∗ = 𝑢𝑇 (𝑝 − 𝑥0 ), 𝑑𝑒𝑡𝐴−1
∗ ∗ 1 𝑑 −𝑏
𝑝 =𝜆 𝑢 𝐴−1 = [ ]
𝐷𝑖𝑠𝑡𝑎𝑛𝑐𝑒 𝑓𝑟𝑜𝑚 𝑝 𝑡𝑜 𝑡ℎ𝑒 ℎ𝑦𝑝𝑒𝑟𝑝𝑙𝑎𝑛𝑒 𝐻 ad − bc −𝑐 𝑎
= { 𝑧 𝜖 ℝ𝑛 , 𝑎 ⊥ 𝐻 ∶ 𝑎𝑇 𝑧 = 𝑏, }:
|𝑎𝑇 𝑝 − 𝑏| 𝑎𝑇 𝑝 − 𝑏
‖𝑝 − 𝑝∗ ‖2 = |𝛼| ∙ ‖𝑎‖2 = , 𝛼= , 𝑝∗
‖𝑎‖2 ‖𝑎‖22
𝑎𝑇 𝑝 − 𝑏
=𝑝− 𝑎
‖𝑎‖22
𝑥1 2 + ⋯ + 𝑥𝑛 2 𝑓(𝛼𝐴) = |𝛼|𝑓(𝐴)
𝑟𝑚𝑠(𝑥) = √
𝑛 𝑓(𝐴 + 𝐵) ≤ 𝑓(𝐴) + 𝑓(𝐵)
𝑓(𝐴𝐵) ≤ 𝑓(𝐴)𝑓(𝐴)𝑓(𝐵)
2 2
(𝑥1 − 𝑎𝑣𝑔(𝑥)) + ⋯ + (𝑥𝑛 − 𝑎𝑣𝑔(𝑥)) 𝑚 𝑛 𝑚
𝑠𝑡𝑑(𝑥) = √
𝑛 ‖𝐴‖𝐹 = √𝑡𝑟𝑎𝑐𝑒𝐴𝐴𝑇 = √∑ ∑ 𝐴2𝑖𝑗 = √∑ 𝜆𝑖 (𝐴𝐴𝑇 )
𝑟𝑚𝑠(𝑥)2 = 𝑎𝑣𝑔(𝑥)2 + 𝑠𝑡𝑑(𝑥)2 𝑖=1 𝑖=1 𝑖=1

𝑃𝑟𝑜𝑗𝑒𝑐𝑡𝑖𝑜𝑛 𝑜𝑛 𝑣𝑒𝑐𝑡𝑜𝑟 𝑠𝑝𝑎𝑛𝑠: 𝑥 ∗ ‖𝐴𝑥‖2 ≤ ‖𝐴‖𝐹 ‖𝑥‖2

𝑑 𝑑
‖𝐴𝑢‖𝑝
= ∑ 𝑎𝑖 𝑥 (𝑖) , ∑ 𝑎𝑖 〈𝑥 (𝑘) , 𝑥 (𝑖) 〉 = 〈𝑥 (𝑘) , 𝑥〉 ‖𝐴‖𝑝 ≐ max = max ‖𝐴𝑢‖𝑝
𝑢≠0 ‖𝑢‖𝑝 ‖𝑢‖=1
𝑖=1 𝑖=1
‖𝐴𝐵𝑢‖𝑝 ≤ ‖𝐴‖𝑝 ‖𝐵𝑢‖𝑝 ≤ ‖𝐴‖𝑝 ‖𝐵‖𝑝 ‖𝑢‖𝑝
𝑃𝑟𝑜𝑗𝑒𝑐𝑡𝑖𝑜𝑛 𝑜𝑛 𝑜𝑟𝑡ℎ𝑜𝑛𝑜𝑟𝑚𝑎𝑙 𝑣𝑒𝑐𝑡𝑜𝑟 𝑠𝑝𝑎𝑛𝑠: 𝑥 ∗ ‖𝐴‖1 = 𝑙𝑎𝑟𝑔𝑒𝑠𝑡 𝑎𝑏𝑠𝑜𝑙𝑢𝑡𝑒 𝑐𝑜𝑙𝑢𝑚𝑛 𝑠𝑢𝑚
𝑑 ‖𝐴‖∞ = 𝑙𝑎𝑟𝑔𝑒𝑠𝑡 𝑎𝑏𝑠𝑜𝑙𝑢𝑡𝑒 𝑟𝑜𝑤 𝑠𝑢𝑚
= ∑〈𝑥 (𝑖) , 𝑥〉𝑥 (𝑖) ‖𝐴‖2 = √𝜆𝑚𝑎𝑥 (𝐴𝑇 𝐴) = 𝜎𝑚𝑎𝑥 (𝐴)
𝑖=1 𝑝(𝜆) ≐ det(𝐴 − 𝜆Ι𝑛 ) = 0
𝑇
𝐺𝑟𝑎𝑚 − 𝑆𝑐ℎ𝑚𝑖𝑑𝑡: 𝑞̃𝑖 = 𝑎𝑖 − (𝑞1𝑇 𝑎𝑖 )𝑞1 − ⋯ − (𝑞𝑖−1 𝑎𝑖 )𝑞𝑖−1 𝐵 = 𝑃 −1 𝐴𝑃 → 𝐵 𝑖𝑠 𝑠𝑖𝑚𝑖𝑙𝑎𝑟 𝑡𝑜 𝐴, 𝑠𝑎𝑚𝑒 𝜆
→ 𝑞𝑖 = 𝑞̃𝑖 ⁄‖𝑞̃𝑖 ‖ 𝑃𝑖 = 𝑁𝑖 ⇔ 𝑑𝑖𝑚𝑁𝑖 = 𝜇𝑖
𝐼𝑛𝑝𝑢𝑡 ⁄𝑜𝑢𝑡𝑝𝑢𝑡 𝑝𝑎𝑖𝑟𝑠 𝑡ℎ𝑎𝑡 𝑓 𝑐𝑎𝑛 𝑎𝑡𝑡𝑎𝑖𝑛: 𝑔𝑟𝑎𝑝ℎ 𝑓 = 𝑡𝑖𝑚𝑒𝑠 𝜆𝑖 𝑎𝑝𝑝𝑒𝑎𝑟 𝑖𝑛 𝑁𝑖 (𝐴 − 𝜆𝑖 Ι𝑛 )
= {(𝑥, 𝑓(𝑥)) ∈ 𝑅𝑛+1 ∶ 𝑥 ∈ 𝑅𝑛 }

𝐼𝑛𝑝𝑢𝑡 ⁄𝑜𝑢𝑡𝑝𝑢𝑡 𝑝𝑎𝑖𝑟𝑠 𝑡ℎ𝑎𝑡 𝑓 𝑐𝑎𝑛 𝑎𝑐ℎ𝑖𝑒𝑣𝑒 𝑎𝑛𝑑 𝑎𝑛𝑦𝑡ℎ𝑖𝑛𝑔 𝑎𝑏𝑜𝑣𝑒:

𝑒𝑝𝑖 𝑓 = {(𝑥, 𝑡) ∈ 𝑅𝑛+1 ∶ 𝑥 ∈ 𝑅𝑛 , 𝑡 ≥ 𝑓(𝑥)}

𝑃𝑜𝑖𝑛𝑡𝑠 𝑡ℎ𝑎𝑡 𝑎𝑐ℎ𝑖𝑒𝑣𝑒 𝑒𝑥𝑎𝑐𝑡𝑙𝑦 𝑠𝑜𝑚𝑒 𝑣𝑎𝑙𝑢𝑒 𝑓𝑜𝑟 𝑡ℎ𝑒 𝑓𝑢𝑛𝑐𝑡𝑖𝑜𝑛:

𝐿𝑒𝑣𝑒𝑙 𝑠𝑒𝑡 (𝑐𝑜𝑛𝑡𝑜𝑢𝑟 𝑙𝑖𝑛𝑒) 𝐶𝑓 (𝑡) = {𝑥 ∈ 𝑅𝑛 ∶ 𝑓(𝑥) = 𝑡}

𝑃𝑜𝑖𝑛𝑡𝑠 𝑡ℎ𝑎𝑡 𝑎𝑐ℎ𝑖𝑒𝑣𝑒 𝑎𝑡 𝑚𝑜𝑠𝑡 𝑎 𝑐𝑒𝑟𝑡𝑎𝑖𝑛 𝑣𝑎𝑙𝑢𝑒 𝑓𝑜𝑟 𝑓:

𝑆𝑢𝑏𝑙𝑒𝑣𝑒𝑙 𝑠𝑒𝑡 𝐿𝑓 (𝑡) = {𝑥 ∈ 𝑅𝑛 ∶ 𝑓(𝑥) ≤ 𝑡}
𝜕𝑓
(𝑧)
𝜕𝑥1
𝑇𝑎𝑦𝑙𝑜𝑟: 𝑓̂(𝑥) = 𝑓(𝑧) + ∇𝑓(𝑧)𝑇 (𝑥 − 𝑧), ∇𝑓(𝑧) = …
𝜕𝑓
(𝑧)
[𝜕𝑥𝑛 ]
𝑁(𝐴)⨁𝑅(𝐴𝑇 ) = 𝑅𝑛
Left Inverse (tall)

Right inverse (wide)

Constrained least square

Complexity

QR = 2𝑚𝑛2
Constrained least norm
RA = 𝑚𝑛2

Ax = 2mn

𝐴𝑇 𝐴 = 𝑚𝑛2 Portfolio
𝑅𝑅𝑇 = 𝑅 −1 = 𝑛3

Linear in the parameters model

Least square model fitting

Straight line fit