diff --git a/API.md b/API.md index b7ea3de..f221d09 100644 --- a/API.md +++ b/API.md @@ -1,16 +1,122 @@ # dn42regsrv API Description +## Route Origin Authorisation (ROA) API + +Route Origin Authorisation (ROA) data can be obtained from the server in +JSON and bird formats. + +### JSON format output + +``` +GET /api/roa/json +``` + +Provides IPv4 and IPv6 ROAs in JSON format, suitable for use with +[gortr](https://github.com/cloudflare/gortr). + +Example Output: +``` +wget -O - -q http://localhost:8042/api/roa/json | jq +``` + +``` +{ + "metadata": { + "counts": 1564, + "generated": 1550402199, + "valid": 1550445399 + }, + "roas": [ + { + "prefix": "172.23.128.0/26", + "maxLength": 29, + "asn": "AS4242422747" + }, + { + "prefix": "172.22.129.192/26", + "maxLength": 29, + "asn": "AS4242423976" + }, + { + "prefix": "10.110.0.0/16", + "maxLength": 24, + "asn": "AS65110" + }, + +... and so on +``` + +### Bird format output + +``` +GET /api/roa/bird/{bird version}/{IP family} +``` + +Provides ROA data suitable for including in to bird. + +{bird version} must be either 1 or 2 + +{IP family} can be 4, 6 or 46 to provide both IPv4 and IPv6 results + + +Example Output: +``` +wget -O - -q http://localhost:8042/api/roa/bird/1/4 +``` + +``` +# +# dn42regsrv ROA Generator +# Last Updated: 2019-02-17 11:16:39.668799525 +0000 GMT m=+0.279049704 +# Commit: 3cbc349bf770493c016888ff785227ded2a7d866 +# +roa 172.23.128.0/26 max 29 as 4242422747; +roa 172.22.129.192/26 max 29 as 4242423976; +roa 10.110.0.0/16 max 24 as 65110; +roa 172.20.164.0/26 max 29 as 4242423023; +roa 172.20.135.200/29 max 29 as 4242420448; +roa 10.65.0.0/20 max 24 as 4242420420; +roa 172.20.149.136/29 max 29 as 4242420234; +roa 10.160.0.0/13 max 24 as 65079; +roa 10.169.0.0/16 max 24 as 65534; + +... and so on +``` + +``` +wget -O - -q http://localhost:8042/api/roa/bird/2/6 +``` + +``` +# +# dn42regsrv ROA Generator +# Last Updated: 2019-02-17 11:16:39.668799525 +0000 GMT m=+0.279049704 +# Commit: 3cbc349bf770493c016888ff785227ded2a7d866 +# +route fdc3:10cd:ae9d::/48 max 64 as 4242420789; +route fd41:9805:7b69:4000::/51 max 64 as 4242420846; +route fd41:9805:7b69:4000::/51 max 64 as 4242420845; +route fd41:9805:7b69:4000::/51 max 64 as 4242420847; +route fddf:ebfd:a801:2331::/64 max 64 as 65530; +route fd42:1a2b:de57::/48 max 64 as 4242422454; +route fd42:7879:7879::/48 max 64 as 4242421787; + +... and so on +``` + ## Registry API The general form of the registry query API is: +``` GET /api/registry/{type}/{object}/{key}/{attribute}?raw +``` * Prefixing with a '*' performs a case insensitive, substring match * A '*' on its own means match everything * Otherwise an exact, case sensitive match is performed -By default results are returned as JSON objects, and the registry data is decorated +By default, results are returned as JSON objects, and the registry data is decorated with markdown style links depending on relations defined in the DN42 schema. For object results, a 'Backlinks' section is also added providing an array of registry objects that reference this one. diff --git a/README.md b/README.md index 5605c1a..01bd353 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ A public instance of the API and explorer web app can be accessed via: basic web applications utilising the API (such as the included DN42 Registry Explorer) * Automatic pull from the DN42 git repository to keep the registry up to date * Includes a responsive web app for exploring the registry +* API endpoints for ROA data in JSON, and bird formats ## Building @@ -60,8 +61,6 @@ Please feel free to raise issues or create pull requests for the project git rep ### Server - Add WHOIS interface -- Add endpoints for ROA data -- Add attribute searches ### DN42 Registry Explorer Web App diff --git a/dn42regsrv.go b/dn42regsrv.go index 37c8ed3..5ac0a66 100644 --- a/dn42regsrv.go +++ b/dn42regsrv.go @@ -8,6 +8,7 @@ package main import ( "context" + "encoding/json" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" flag "github.com/spf13/pflag" @@ -40,6 +41,27 @@ func (bus SimpleEventBus) Fire(event string, params ...interface{}) { } } +////////////////////////////////////////////////////////////////////////// +// utility func for returning JSON from an API endpoint + +func ResponseJSON(w http.ResponseWriter, v interface{}) { + + // for response time testing + //time.Sleep(time.Second) + + // marshal the JSON string + data, err := json.Marshal(v) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Error("Failed to marshal JSON") + } + + // write back to http handler + w.Header().Set("Content-Type", "application/json") + w.Write(data) +} + ////////////////////////////////////////////////////////////////////////// // utility function to set the log level diff --git a/regapi.go b/regapi.go index b27ebaa..d94fd82 100644 --- a/regapi.go +++ b/regapi.go @@ -7,7 +7,6 @@ package main ////////////////////////////////////////////////////////////////////////// import ( - "encoding/json" // "fmt" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" @@ -47,27 +46,6 @@ func InitRegistryAPI(params ...interface{}) { log.Info("Registry API installed") } -////////////////////////////////////////////////////////////////////////// -// handler utility funcs - -func responseJSON(w http.ResponseWriter, v interface{}) { - - // for response time testing - //time.Sleep(time.Second) - - // marshal the JSON string - data, err := json.Marshal(v) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - }).Error("Failed to marshal JSON") - } - - // write back to http handler - w.Header().Set("Content-Type", "application/json") - w.Write(data) -} - ////////////////////////////////////////////////////////////////////////// // filter functions @@ -313,7 +291,7 @@ func regRootHandler(w http.ResponseWriter, r *http.Request) { for _, rType := range RegistryData.Types { response[rType.Ref] = len(rType.Objects) } - responseJSON(w, response) + ResponseJSON(w, response) } @@ -346,7 +324,7 @@ func regTypeHandler(w http.ResponseWriter, r *http.Request) { response[rtype.Ref] = objects } - responseJSON(w, response) + ResponseJSON(w, response) } ////////////////////////////////////////////////////////////////////////// @@ -411,7 +389,7 @@ func regObjectHandler(w http.ResponseWriter, r *http.Request) { } } - responseJSON(w, response) + ResponseJSON(w, response) } else { // provide a response with just the raw registry data @@ -429,7 +407,7 @@ func regObjectHandler(w http.ResponseWriter, r *http.Request) { } } - responseJSON(w, response) + ResponseJSON(w, response) } } @@ -480,7 +458,7 @@ func regKeyHandler(w http.ResponseWriter, r *http.Request) { return } - responseJSON(w, amap) + ResponseJSON(w, amap) } ////////////////////////////////////////////////////////////////////////// @@ -530,7 +508,7 @@ func regAttributeHandler(w http.ResponseWriter, r *http.Request) { return } - responseJSON(w, amap) + ResponseJSON(w, amap) } diff --git a/registry.go b/registry.go index e551b7a..9ad497c 100644 --- a/registry.go +++ b/registry.go @@ -63,6 +63,7 @@ type RegTypeSchema struct { // the registry itself type Registry struct { + Commit string Schema map[string]*RegTypeSchema Types map[string]*RegType } @@ -187,12 +188,13 @@ func (object *RegObject) addBacklink(ref *RegObject) { ////////////////////////////////////////////////////////////////////////// // reload the registry -func reloadRegistry(path string) { +func reloadRegistry(path string, commit string) { log.Debug("Reloading registry") // r will become the new registry data registry := &Registry{ + Commit: commit, Schema: make(map[string]*RegTypeSchema), Types: make(map[string]*RegType), } @@ -215,6 +217,9 @@ func reloadRegistry(path string) { // mark relationships registry.decorate() + // trigger updates in any other modules + EventBus.Fire("RegistryUpdate", registry, path) + // swap in the new registry data RegistryData = registry } @@ -643,7 +648,7 @@ func InitialiseRegistryData(regDir string, refresh time.Duration, // initialise the previous commit hash // and do initial load from registry previousCommit = getCommitHash(regDir, gitPath) - reloadRegistry(dataPath) + reloadRegistry(dataPath, previousCommit) go func() { @@ -667,7 +672,7 @@ func InitialiseRegistryData(regDir string, refresh time.Duration, }).Info("Registry has changed, refresh started") // refresh - reloadRegistry(dataPath) + reloadRegistry(dataPath, currentCommit) // update commit previousCommit = currentCommit diff --git a/roaapi.go b/roaapi.go new file mode 100644 index 0000000..83bfe68 --- /dev/null +++ b/roaapi.go @@ -0,0 +1,405 @@ +////////////////////////////////////////////////////////////////////////// +// DN42 Registry API Server +////////////////////////////////////////////////////////////////////////// + +package main + +////////////////////////////////////////////////////////////////////////// + +import ( + "fmt" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" + // "math/big" + "bufio" + "net" + "net/http" + "os" + "sort" + "strconv" + "strings" + "time" +) + +////////////////////////////////////////////////////////////////////////// +// register the api + +func init() { + EventBus.Listen("APIEndpoint", InitROAAPI) + EventBus.Listen("RegistryUpdate", ROAUpdate) +} + +////////////////////////////////////////////////////////////////////////// +// data model + +type PrefixROA struct { + Prefix string `json:"prefix"` + MaxLen uint8 `json:"maxLength"` + ASN string `json:"asn"` +} + +type ROAFilter struct { + Number uint + Action string + Prefix string + MinLen uint8 + MaxLen uint8 + Network *net.IPNet +} + +type ROA struct { + CTime time.Time + Commit string + Filters []*ROAFilter + IPv4 []*PrefixROA + IPv6 []*PrefixROA +} + +var ROAData *ROA + +type ROAMetaData struct { + Counts uint `json:"counts"` + Generated uint32 `json:"generated"` + Valid uint32 `json:"valid"` + Signature string `json:"signature,omitempty"` + SignatureDate string `json:"signatureDate,omitempty"` +} + +type ROAJSON struct { + MetaData ROAMetaData `json:"metadata"` + Roas []*PrefixROA `json:"roas"` +} + +var ROAJSONResponse *ROAJSON + +////////////////////////////////////////////////////////////////////////// +// called from main to initialise the API routing + +func InitROAAPI(params ...interface{}) { + + router := params[0].(*mux.Router) + + s := router. + Methods("GET"). + PathPrefix("/roa"). + Subrouter() + + s.HandleFunc("/json", roaJSONHandler) + s.HandleFunc("/bird/{birdv}/{ipv}", roaBirdHandler) + + log.Info("ROA API installed") +} + +////////////////////////////////////////////////////////////////////////// +// api handlers + +// return JSON formatted ROA data suitable for use with GoRTR +func roaJSONHandler(w http.ResponseWriter, r *http.Request) { + ResponseJSON(w, ROAJSONResponse) +} + +// return the roa in bird format +func roaBirdHandler(w http.ResponseWriter, r *http.Request) { + + vars := mux.Vars(r) + birdv := vars["birdv"] + ipv := vars["ipv"] + + // bird 1 or bird 2 format + birdf := "roa %s max %d as %s;\n" + if birdv == "2" { + birdf = "route %s max %d as %s;\n" + } + + var roa []*PrefixROA + if strings.ContainsRune(ipv, '4') { + roa = append(roa, ROAData.IPv4...) + } + if strings.ContainsRune(ipv, '6') { + roa = append(roa, ROAData.IPv6...) + } + + w.Header().Set("Content-Type", "text/plain") + fmt.Fprintf(w, "#\n# dn42regsrv ROA Generator\n# Last Updated: %s\n"+ + "# Commit: %s\n#\n", ROAData.CTime.String(), ROAData.Commit) + + for _, r := range roa { + fmt.Fprintf(w, birdf, r.Prefix, r.MaxLen, r.ASN[2:]) + } + +} + +////////////////////////////////////////////////////////////////////////// +// called whenever the registry is updated + +func ROAUpdate(params ...interface{}) { + + registry := params[0].(*Registry) + path := params[1].(string) + + // initiate new ROA data + roa := &ROA{ + CTime: time.Now(), + Commit: registry.Commit, + } + + // load filter{,6}.txt files + if roa.loadFilter(path+"/filter.txt") != nil { + // error loading IPv4 filter, don't update + return + } + + if roa.loadFilter(path+"/filter6.txt") != nil { + // error loading IPv6 filter, don't update + return + } + + // compile ROA prefixes + roa.IPv4 = roa.CompileROA(registry, "route") + roa.IPv6 = roa.CompileROA(registry, "route6") + + // swap in the new data + ROAData = roa + + log.WithFields(log.Fields{ + "ipv4": len(roa.IPv4), + "ipv6": len(roa.IPv6), + }).Debug("ROA data updated") + + // pre-compute the JSON return struct + + utime := uint32(roa.CTime.Unix()) + + response := &ROAJSON{ + MetaData: ROAMetaData{ + Generated: utime, + Valid: utime + (12 * 3600), // valid for 12 hours + }, + } + + response.Roas = append(roa.IPv4, roa.IPv6...) + response.MetaData.Counts = uint(len(response.Roas)) + + ROAJSONResponse = response +} + +////////////////////////////////////////////////////////////////////////// +// load network filter definitions from a filter file + +func (roa *ROA) loadFilter(path string) error { + + // open the file for reading + file, err := os.Open(path) + if err != nil { + log.WithFields(log.Fields{ + "path": path, + "error": err, + }).Error("Unable to open filter file") + return err + } + defer file.Close() + + // helper closure to convert strings to numbers + var cerr error + convert := func(s string) int { + if cerr != nil { + return 0 + } + val, cerr := strconv.Atoi(s) + if cerr != nil { + log.WithFields(log.Fields{ + "number": s, + "error": err, + }).Error("Unable to parse number in filter file") + return 0 + } + return val + } + + filters := make([]*ROAFilter, 0) + + // read the file line by line + scanner := bufio.NewScanner(file) + for scanner.Scan() { + + line := strings.TrimSpace(scanner.Text()) + + // remove any comments + if ix := strings.IndexRune(line, '#'); ix != -1 { + line = line[:ix] + } + + fields := strings.Fields(line) + if len(fields) >= 5 { + + // parse the prefix in to a NetIP structure + prefix := fields[2] + _, network, err := net.ParseCIDR(prefix) + if err != nil { + log.WithFields(log.Fields{ + "path": path, + "prefix": prefix, + "error": err, + }).Error("Unable to parse CIDR in filter file") + + } else { + + // construct the filter object + roaf := &ROAFilter{ + Number: uint(convert(fields[0])), + Action: fields[1], + Prefix: prefix, + MinLen: uint8(convert(fields[3])), + MaxLen: uint8(convert(fields[4])), + Network: network, + } + + // add to list if no strconv error + if cerr == nil { + filters = append(filters, roaf) + } + } + } + } + + // did something go wrong ? + if err := scanner.Err(); err != nil { + log.WithFields(log.Fields{ + "path": path, + "error": err, + }).Error("Scanner error reading filter file") + return err + } + + // sort the filters based on prefix length (largest first) + sort.Slice(filters, func(i, j int) bool { + leni, _ := filters[i].Network.Mask.Size() + lenj, _ := filters[j].Network.Mask.Size() + return leni > lenj + }) + + // add to the roa object + roa.Filters = append(roa.Filters, filters...) + return nil +} + +////////////////////////////////////////////////////////////////////////// +// return the filter object that matches an IP address + +func (roa *ROA) MatchFilter(ip net.IP) *ROAFilter { + for _, filter := range roa.Filters { + if filter.Network.Contains(ip) { + return filter + } + } + + log.WithFields(log.Fields{ + "IP": ip, + }).Error("Couldn't match address to filter !") + + return nil +} + +////////////////////////////////////////////////////////////////////////// +// compile ROA data + +func (roa *ROA) CompileROA(registry *Registry, + tname string) []*PrefixROA { + + // prepare indices to the route object keys + stype := registry.Schema[tname] + routeIX := stype.KeyIndex[tname] + originIX := stype.KeyIndex["origin"] + mlenIX := stype.KeyIndex["max-length"] + + roalist := make([]*PrefixROA, 0, len(routeIX.Objects)) + + // for each object that has a route key + for object, rattribs := range routeIX.Objects { + + if len(rattribs) > 1 { + log.WithFields(log.Fields{ + "object": object.Ref, + }).Warn("Found object with multiple route attributes") + } + + // extract the prefix + prefix := rattribs[0].RawValue + _, pnet, err := net.ParseCIDR(prefix) + if err != nil { + log.WithFields(log.Fields{ + "object": object.Ref, + "prefix": prefix, + "error": err, + }).Error("Unable to parse CIDR in ROA") + continue + } + + // match the prefix to the prefix filters + filter := roa.MatchFilter(pnet.IP) + if filter == nil { + continue + } + + if filter.Action == "deny" { + log.WithFields(log.Fields{ + "object": object.Ref, + "prefix": prefix, + "filter": filter.Prefix, + }).Warn("Denied ROA through filter rule") + continue + } + + // calculate the max-length for this object + + mlen := filter.MaxLen + + // check if the attribute has max-length defined + mattrib := mlenIX.Objects[object] + if mattrib != nil { + + // use the local max-length value + tmp, err := strconv.ParseUint(mattrib[0].RawValue, 10, 8) + if err != nil { + log.WithFields(log.Fields{ + "object": object.Ref, + "max-length": mattrib[0].RawValue, + "error": err, + }).Warn("Unable to convert max-length attribute") + } else { + + // filter rules still have precedence over local values + if (uint8(tmp) < mlen) && (uint8(tmp) > filter.MinLen) { + mlen = uint8(tmp) + } + + } + } + + // look up the origin key for this object + oattribs := originIX.Objects[object] + if oattribs == nil { + log.WithFields(log.Fields{ + "object": object.Ref, + }).Warn("Route Object without Origin") + } else { + + // then for origin that can announce this prefix + for _, oattrib := range oattribs { + + // add the ROA + roalist = append(roalist, &PrefixROA{ + Prefix: prefix, + MaxLen: mlen, + ASN: oattrib.RawValue, + }) + + } + } + } + + return roalist +} + +////////////////////////////////////////////////////////////////////////// +// end of code