From 707b8035c397c6ad9e90f5b0279fefe6d5d09a72 Mon Sep 17 00:00:00 2001 From: Garionion Date: Fri, 8 Mar 2024 23:51:11 +0100 Subject: [PATCH] feat: use ripedb for asn lookup --- cmd/cli/cli.go | 10 ++++ flake.lock | 14 +++--- go.mod | 13 +++-- go.sum | 28 +++++++++-- main.go | 55 ++++++++++++++++----- package.nix | 13 ++--- ripe/builder.go | 126 ++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 223 insertions(+), 36 deletions(-) create mode 100644 cmd/cli/cli.go create mode 100644 ripe/builder.go diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go new file mode 100644 index 0000000..591a48d --- /dev/null +++ b/cmd/cli/cli.go @@ -0,0 +1,10 @@ +package main + +import "voc-telemetry/ripe" + +func main() { + err := ripe.BuildNewMMDB("/tmp/ripe.mmdb") + if err != nil { + panic(err) + } +} diff --git a/flake.lock b/flake.lock index d6ced7a..58e6e18 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1701680307, - "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", + "lastModified": 1709126324, + "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=", "owner": "numtide", "repo": "flake-utils", - "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", + "rev": "d465f4819400de7c8d874d50b982301f28a84605", "type": "github" }, "original": { @@ -20,10 +20,10 @@ }, "nixpkgs": { "locked": { - "lastModified": 1703255338, - "narHash": "sha256-Z6wfYJQKmDN9xciTwU3cOiOk+NElxdZwy/FiHctCzjU=", - "path": "/nix/store/55ql4j8d47xjvfa8xggjddgfwny4n9j7-source", - "rev": "6df37dc6a77654682fe9f071c62b4242b5342e04", + "lastModified": 1709703039, + "narHash": "sha256-6hqgQ8OK6gsMu1VtcGKBxKQInRLHtzulDo9Z5jxHEFY=", + "path": "/nix/store/gig8j85kj7ybjy3ksn6k3aich8j2k59y-source", + "rev": "9df3e30ce24fd28c7b3e2de0d986769db5d6225d", "type": "path" }, "original": { diff --git a/go.mod b/go.mod index 5120c07..fba98fa 100644 --- a/go.mod +++ b/go.mod @@ -3,34 +3,41 @@ module voc-telemetry go 1.21 require ( - github.com/labstack/echo v3.3.10+incompatible github.com/labstack/echo-contrib v0.15.0 github.com/labstack/echo/v4 v4.10.2 + github.com/maxmind/mmdbwriter v1.0.0 + github.com/oschwald/maxminddb-golang v1.12.0 + github.com/prometheus/client_golang v1.14.0 + github.com/stretchr/testify v1.8.4 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/text v0.2.0 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.40.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect + go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a984ee8..b1fda50 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -14,11 +14,15 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg= -github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo-contrib v0.15.0 h1:9K+oRU265y4Mu9zpRDv3X+DGTqUALY6oRHCSZZKCRVU= github.com/labstack/echo-contrib v0.15.0/go.mod h1:lei+qt5CLB4oa7VHTE0yEfQSEB9XTJI1LUqko9UWvo4= github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= @@ -32,11 +36,16 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw= +github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs= +github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= @@ -46,12 +55,18 @@ github.com/prometheus/common v0.40.0 h1:Afz7EVRqGg2Mqqf4JuF9vdvp1pi220m55Pi9T2Jn github.com/prometheus/common v0.40.0/go.mod h1:L65ZJPSmfn/UBWLQIHV7dBrKFidB/wPlF1y5TlSt9OE= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d h1:ggxwEf5eu0l8v+87VhX1czFh8zJul3hK16Gmruxn7hw= +go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d/go.mod h1:tgPU4N2u9RByaTN3NC2p9xOzyFpte4jYwsIIRF7XlSc= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= @@ -70,3 +85,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index b9c3f01..3605d3f 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,15 @@ package main import ( + "errors" "flag" "fmt" "github.com/labstack/echo-contrib/echoprometheus" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" + "github.com/oschwald/maxminddb-golang" "github.com/prometheus/client_golang/prometheus" + "net" "net/http" ) @@ -36,6 +39,8 @@ const ( typeRecovery = "recovery" typeError = "error" typeQualitySwitch = "quality_switch" + + mmdbPath = "/tmp/ripe.mmdb" ) func main() { @@ -44,6 +49,11 @@ func main() { panic(err) } + mmdb, err := maxminddb.Open(mmdbPath) + if err != nil { + panic(err) + } + listenAddress := ":2342" flag.StringVar(&listenAddress, "listen", listenAddress, "address to listen on") flag.Parse() @@ -61,36 +71,42 @@ func main() { e.Use(echoprometheus.NewMiddleware("http")) e.GET("/metrics", echoprometheus.NewHandler()) - e.POST("/", telemetryHandler(m)) + e.POST("/", telemetryHandler(m, mmdb)) e.Logger.Fatal(e.Start(listenAddress)) } -func telemetryHandler(m metrics) echo.HandlerFunc { +func telemetryHandler(m metrics, mmdb *maxminddb.Reader) echo.HandlerFunc { return func(c echo.Context) error { t := new([]Telemetry) if err := c.Bind(t); err != nil { return c.String(http.StatusBadRequest, "bad request") } + ip := c.RealIP() + asn, err := queryMMDB(mmdb, ip) + if err != nil { + asn = "unknown" + } + for _, v := range *t { switch v.EventType { case typeBuffering: m.bufferingSlug.WithLabelValues(v.Slug).Inc() - m.bufferingRelay.WithLabelValues(v.Relay).Inc() + m.bufferingRelay.WithLabelValues(v.Relay, asn).Inc() case typeRecovery: m.recoverySlug.WithLabelValues(v.Slug).Inc() - m.recoveryRelay.WithLabelValues(v.Relay).Inc() + m.recoveryRelay.WithLabelValues(v.Relay, asn).Inc() case typeError: m.errorMetricSlug.WithLabelValues(v.Slug).Inc() - m.errorMetricRelay.WithLabelValues(v.Relay).Inc() + m.errorMetricRelay.WithLabelValues(v.Relay, asn).Inc() case typeQualitySwitch: if v.QualityChangeUp { m.qualityUpSlug.WithLabelValues(v.Slug).Inc() - m.qualityUpRelay.WithLabelValues(v.Relay).Inc() + m.qualityUpRelay.WithLabelValues(v.Relay, asn).Inc() } else { m.qualityDownSlug.WithLabelValues(v.Slug).Inc() - m.qualityDownRelay.WithLabelValues(v.Relay).Inc() + m.qualityDownRelay.WithLabelValues(v.Relay, asn).Inc() } } } @@ -110,7 +126,7 @@ func initMetrics() (metrics, error) { Namespace: "telemetry", Subsystem: "relay", Name: "buffering", - }, []string{"relay"}) + }, []string{"relay", "asn"}) metricRecoverySlug := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "telemetry", @@ -122,7 +138,7 @@ func initMetrics() (metrics, error) { Namespace: "telemetry", Subsystem: "relay", Name: "recovery", - }, []string{"relay"}) + }, []string{"relay", "asn"}) metricErrorSlug := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "telemetry", @@ -134,7 +150,7 @@ func initMetrics() (metrics, error) { Namespace: "telemetry", Subsystem: "relay", Name: "error", - }, []string{"relay"}) + }, []string{"relay", "asn"}) metricQualitySwitchUpSlug := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "telemetry", @@ -146,7 +162,7 @@ func initMetrics() (metrics, error) { Namespace: "telemetry", Subsystem: "relay", Name: "quality_switch_up", - }, []string{"relay"}) + }, []string{"relay", "asn"}) metricQualitySwitchDownSlug := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "telemetry", @@ -158,7 +174,7 @@ func initMetrics() (metrics, error) { Namespace: "telemetry", Subsystem: "relay", Name: "quality_switch_down", - }, []string{"relay"}) + }, []string{"relay", "asn"}) if err := prometheus.Register(metricBufferingSlug); err != nil { return metrics{}, fmt.Errorf("failed to register buffering slug metric: %w", err) @@ -213,3 +229,18 @@ func initMetrics() (metrics, error) { qualityDownRelay: metricQualitySwitchDownRelay, }, nil } + +func queryMMDB(mmdb *maxminddb.Reader, ip string) (string, error) { + var record map[string]interface{} + + err := mmdb.Lookup(net.ParseIP(ip), &record) + if err != nil { + return "", fmt.Errorf("error looking up IP in MMDB: %w", err) + } + + asn, ok := record["asn"].(string) + if !ok { + return "", errors.New("asn is not a string") + } + return asn, nil +} diff --git a/package.nix b/package.nix index 5540945..8db584d 100644 --- a/package.nix +++ b/package.nix @@ -1,11 +1,11 @@ { pkgs ? import { } }: with pkgs; let - version = "0.0.1"; + version = "0.0.2"; deps = [ ]; nativeDeps = [ ]; in -pkgs.buildGo121Module { +pkgs.buildGo122Module { pname = "voc-telemetry"; inherit version; src = ./.; @@ -17,16 +17,9 @@ pkgs.buildGo121Module { ] ++nativeDeps; tags = [ ]; - CGO_ENABLED = 0; - ldflags = [ - "-s" "-w" - "-linkmode external" - "-extldflags '-static -L${musl}/lib'" - ]; - allowGoReference = false; #vendorSha256 = lib.fakeSha256; - vendorHash = "sha256-RNGVk+X1TpWch1y3JV/RUSao48m19lawkHfwAh7xi2Y="; + vendorHash = "sha256-/cxpaZRQy77T5iilEaXsgXji1MMdk8503OpYvPmrVZc="; meta = { description = "TBA"; diff --git a/ripe/builder.go b/ripe/builder.go new file mode 100644 index 0000000..778da96 --- /dev/null +++ b/ripe/builder.go @@ -0,0 +1,126 @@ +package ripe + +import ( + "bufio" + "compress/gzip" + "errors" + "fmt" + "net" + "net/http" + "os" + "strings" + + "github.com/maxmind/mmdbwriter" + "github.com/maxmind/mmdbwriter/mmdbtype" +) + +type MMDBEntry struct { + Range string `json:"range",maxminddb:"range"` + ASN string `json:"asn",maxminddb:"asn"` +} + +const ( + ripe4_download_url = "https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz" + ripe6_download_url = "https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz" +) + +func BuildNewMMDB(filepath string) error { + writer, err := mmdbwriter.New( + mmdbwriter.Options{ + DatabaseType: "VOC-ASN-DB", + RecordSize: 32, + }, + ) + if err != nil { + return fmt.Errorf("error creating mmdb writer: %w", err) + } + + if err := fillMMDB(writer, ripe4_download_url); err != nil { + return fmt.Errorf("error building IPv4 mmdb: %w", err) + } + + if err := fillMMDB(writer, ripe6_download_url); err != nil { + return fmt.Errorf("error building IPv6 mmdb: %w", err) + } + + fh, err := os.Create(filepath) + if err != nil { + return fmt.Errorf("error creating mmdb output file: %w", err) + } + + if _, err = writer.WriteTo(fh); err != nil { + return fmt.Errorf("error writing mmdb to file: %w", err) + } + + return nil +} + +func fillMMDB(mmdb *mmdbwriter.Tree, url string) error { + // Fetch data from URL + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("error fetching data from %s: %w", url, err) + } + defer resp.Body.Close() + + var s *bufio.Scanner + + // Check if response is gzip compressed + reader, err := gzip.NewReader(resp.Body) + if err != nil { + s = bufio.NewScanner(resp.Body) + } else { + s = bufio.NewScanner(reader) + defer reader.Close() + } + + var ( + entries []MMDBEntry + entry MMDBEntry + ) + + for tok := s.Scan(); tok; tok = s.Scan() { + //remove all spaces + line := strings.ReplaceAll(s.Text(), " ", "") + + tokens := strings.Split(line, ":") + + if len(tokens) != 2 { + continue + } + + key := strings.TrimSpace(tokens[0]) + if key == "route" { + entry = MMDBEntry{ + Range: strings.TrimSpace(tokens[1]), + } + } else if key == "origin" { + entry.ASN = strings.TrimSpace(tokens[1]) + entries = append(entries, entry) + } + + } + + if err := s.Err(); err != nil { + fmt.Printf("error scanning ripe file: %s", err) + } + + var entrieErrors error + for _, e := range entries { + record := mmdbtype.Map{ + "range": mmdbtype.String(e.Range), + "asn": mmdbtype.String(e.ASN), + } + + _, network, err := net.ParseCIDR(e.Range) + if err != nil { + errors.Join(entrieErrors, fmt.Errorf("error parsing CIDR: %w", err)) + continue + } + + err = mmdb.Insert(network, record) + errors.Join(entrieErrors, fmt.Errorf("error inserting record: %w", err)) + } + + return entrieErrors +}