148 lines
2.8 KiB
Go
148 lines
2.8 KiB
Go
package microgopt
|
|
|
|
import (
|
|
"fmt"
|
|
"maps"
|
|
"math"
|
|
"slices"
|
|
"strings"
|
|
|
|
"github.com/davecgh/go-spew/spew"
|
|
)
|
|
|
|
func btof(b bool) float64 {
|
|
if b {
|
|
return 1.0
|
|
}
|
|
return 0.0
|
|
}
|
|
|
|
func Run(docs []string) {
|
|
// remove leading and trailing whitespace in documents
|
|
for i := range docs {
|
|
docs[i] = strings.TrimSpace(docs[i])
|
|
}
|
|
fmt.Printf("num docs: %d", len(docs))
|
|
|
|
// construct the vocabulary from the documents: an ordered list of all characters in the dataset,
|
|
// plus a "Beginning Of Sequence" (BOS) token
|
|
set := map[rune]struct{}{}
|
|
for _, doc := range docs {
|
|
for _, c := range doc {
|
|
set[c] = struct{}{}
|
|
}
|
|
}
|
|
uchars := slices.Sorted(maps.Keys(set))
|
|
// BOS := len(uchars)
|
|
vocabSize := len(uchars) + 1
|
|
fmt.Printf("vocab size: %d", vocabSize)
|
|
}
|
|
|
|
type value struct {
|
|
data float64
|
|
grad float64 // implicitly 0 to start
|
|
children []*value
|
|
localGrads []*value
|
|
}
|
|
|
|
func (v *value) toKey() string {
|
|
k := fmt.Sprintf("%+v", v)
|
|
fmt.Println(k)
|
|
return k
|
|
}
|
|
|
|
func (v *value) Add(other *value) *value {
|
|
return &value{
|
|
data: v.data + other.data,
|
|
children: []*value{v, other},
|
|
localGrads: []*value{{data: 1}, {data: 1}},
|
|
}
|
|
}
|
|
|
|
func (v *value) Sub(other *value) *value {
|
|
return v.Add(other.Neg())
|
|
}
|
|
|
|
func (v *value) Div(other *value) *value {
|
|
return v.Mul(other.Pow(&value{data: -1}))
|
|
}
|
|
|
|
func (v *value) Mul(other *value) *value {
|
|
return &value{
|
|
data: v.data * other.data,
|
|
children: []*value{v, other},
|
|
localGrads: []*value{
|
|
{data: other.data},
|
|
{data: v.data},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (v *value) Pow(other *value) *value {
|
|
return &value{
|
|
data: math.Pow(v.data, other.data),
|
|
children: []*value{v},
|
|
localGrads: []*value{
|
|
other.Mul(&value{data: math.Pow(v.data, other.Sub(&value{data: 1}).data)}),
|
|
}}
|
|
}
|
|
|
|
func (v *value) Neg() *value {
|
|
return v.Mul(&value{data: -1})
|
|
}
|
|
|
|
func (v *value) Log() *value {
|
|
return &value{
|
|
data: math.Log(v.data),
|
|
children: []*value{v},
|
|
localGrads: []*value{
|
|
(&value{data: 1}).Div(v),
|
|
},
|
|
}
|
|
}
|
|
|
|
func (v *value) Exp() *value {
|
|
return &value{
|
|
data: math.Exp(v.data),
|
|
children: []*value{v},
|
|
localGrads: []*value{
|
|
{data: math.Exp(v.data)},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (v *value) Relu() *value {
|
|
return &value{
|
|
data: max(v.data, 0),
|
|
children: []*value{v},
|
|
localGrads: []*value{
|
|
{data: btof(v.data > 0)},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (v *value) Backward() {
|
|
topo := []*value{}
|
|
visited := map[string]struct{}{}
|
|
|
|
var buildTopo func(v *value)
|
|
buildTopo = func(v *value) {
|
|
k := v.toKey()
|
|
if _, ok := visited[k]; !ok {
|
|
visited[k] = struct{}{}
|
|
for _, child := range v.children {
|
|
buildTopo(child)
|
|
}
|
|
topo = append(topo, v)
|
|
}
|
|
}
|
|
buildTopo(v)
|
|
spew.Dump(topo)
|
|
v.grad = 1
|
|
for _, v := range slices.Backward(topo) {
|
|
for i := range v.children {
|
|
v.children[i].grad += v.localGrads[i].data * v.grad
|
|
}
|
|
}
|
|
}
|