burble/pipeline-test
1
0
Fork 0
Code Issues Pull Requests Releases5 Wiki Activity

blah

Browse Source
This commit is contained in:
Simon Marsh 2020-08-16 14:07:29 +01:00
parent c2b0f9a3eb
commit ac6145392c
No known key found for this signature in database
GPG Key ID: 30B29A716A54DBB3
25 changed files with 3883 additions and 14 deletions

21
.drone.yml

@ -1,17 +1,14 @@
--- ---
kind: pipeline kind: pipeline
type: exec type: docker
name: just a test name: default
workspace:
base: /go
path: src/dn42regsrv
steps: steps:
- name: environment - name: build
image: golang
commands: commands:
- pwd - go get
- env
- df -h
- ls -la
- ls -la ../
- id
- ps -ef

500
API.md Normal file

@ -0,0 +1,500 @@
# dn42regsrv API Description
## Registry API
The general form of the registry query API is:
```
GET /api/registry/{type}/{object}/{key}/{attribute}?raw
```
* Prefixing with a '*' performs a case insensitive, substring match
* A '*' on its own means match everything
* Otherwise an exact, case sensitive match is performed
By default, results are returned as JSON objects, and the registry data is decorated
with markdown style links depending on relations defined in the DN42 schema. For object
results, a 'Backlinks' section is also added providing an array of registry objects that
reference this one.
If the 'raw' parameter is provided, attributes are returned un-decorated exactly
as contained in the registry.
Some examples will help clarify:
* Return a JSON object, with keys for each registry type and values containing a count
of the number of registry objects for each type
```
wget -O - -q http://localhost:8042/api/registry/ | jq
{
"as-block": 8,
"as-set": 34,
"aut-num": 1486,
"domain": 451,
"inet6num": 746,
"inetnum": 1276,
"key-cert": 7,
"mntner": 1379,
"organisation": 275,
"person": 1388,
"registry": 4,
"role": 14,
"route": 892,
"route-set": 2,
"route6": 596,
"schema": 18,
"tinc-key": 25,
"tinc-keyset": 3
}
```
* Return a list of all objects in the role type
```
wget -O - -q http://localhost:8042/api/registry/role | jq
{
"role": [
"ORG-NETRAVNEN-DN42",
"PACKETPUSHERS-DN42",
"CCCKC-DN42",
"FLHB-ABUSE-DN42",
"NIXNODES-DN42",
"ORG-SHACK-ABUSE-DN42",
"ORG-SHACK-TECH-DN42",
"ORG-YANE-DN42",
"SOURIS-DN42",
"CCCHB-ABUSE-DN42",
"MAGLAB-DN42",
"NL-ZUID-DN42",
"ORG-SHACK-ADMIN-DN42",
"ALENAN-DN42"
]
}
```
* Returns a list of all objects in types that match 'route'
```
wget -O - -q http://localhost:8042/api/registry/*route | jq
{
"route": [
"172.20.28.0_27",
"172.23.220.0_24",
"172.23.82.0_25",
"10.149.0.0_16",
...
"172.20.128.0_27",
"172.22.127.32_27"
],
"route-set": [
"RS-DN42",
"RS-DN42-NATIVE"
],
"route6": [
"fd42:df42::_48",
"fd5c:0f0f:39fc::_48",
...
"fd16:c638:3d7c::_48",
"fd23::_48"
]
}
```
* Returns the mntner/BURBLE-MNT object (in decorated format)
```
wget -O - -q http://localhost:8042/api/registry/mntner/BURBLE-MNT | jq
{
"mntner/BURBLE-MNT": {
"Attributes": [
[
"mntner",
"BURBLE-MNT"
],
[
"descr",
"burble.dn42 https://dn42.burble.com/"
],
[
"admin-c",
"[BURBLE-DN42](person/BURBLE-DN42)"
],
[
"tech-c",
"[BURBLE-DN42](person/BURBLE-DN42)"
],
[
"auth",
"pgp-fingerprint 1C08F282095CCDA432AECC657B9FE8780CFB6593"
],
[
"mnt-by",
"[BURBLE-MNT](mntner/BURBLE-MNT)"
],
[
"source",
"[DN42](registry/DN42)"
]
],
"Backlinks": [
"aut-num/AS4242422602",
"aut-num/AS4242422601",
"mntner/BURBLE-MNT",
"route/172.20.129.160_27",
"as-set/AS4242422601:AS-DOWNSTREAM",
"as-set/AS4242422601:AS-TRANSIT",
"person/BURBLE-DN42",
"inet6num/fd42:4242:2601::_48",
"domain/burble.dn42",
"domain/collector.dn42",
"route6/fd42:4242:2601::_48",
"inetnum/172.20.129.160_27"
]
}
}
```
* Returns error 404, exact searches are case sensitive
```
wget -O - -q http://localhost:8042/api/registry/mntner/burble-mnt | jq
```
* Returns domain names matching 'burble' in raw format
```
wget -O - -q http://localhost:8042/api/registry/domain/*burble?raw | jq
{
"domain/burble.dn42": [
[
"domain",
"burble.dn42"
],
[
"descr",
"burble.dn42 https://dn42.burble.com/"
],
[
"admin-c",
"BURBLE-DN42"
],
[
"tech-c",
"BURBLE-DN42"
],
[
"mnt-by",
"BURBLE-MNT"
],
[
"nserver",
"ns1.burble.dn42 172.20.129.161"
],
[
"nserver",
"ns1.burble.dn42 fd42:4242:2601:ac53::1"
],
[
"ds-rdata",
"61857 13 2 bd35e3efe3325d2029fb652e01604a48b677cc2f44226eeabee54b456c67680c"
],
[
"source",
"DN42"
]
]
}
```
* Returns all objects matching 172.20.0
```
wget -O - -q http://localhost:8042/api/registry/*/*172.20.0 | jq
{
"inetnum/172.20.0.0_14": {
"Attributes": [
[
"inetnum",
"172.20.0.0 - 172.23.255.255"
],
[
"cidr",
"172.20.0.0/14"
],
... and so on
```
* Returns the nic-hdl attribute for all person objects
```
wget -O - -q http://localhost:8042/api/registry/person/*/nic-hdl | jq
{
"person/0RIGO-DN42": {
"nic-hdl": [
"0RIGO-DN42"
]
},
"person/0XDRAGON-DN42": {
"nic-hdl": [
"0XDRAGON-DN42"
]
},
"person/1714-DN42": {
"nic-hdl": [
"1714-DN42"
]
},
... and so on
```
* return raw contact (-c) attributes in aut-num objects that contain 'burble'
```
wget -O - -q http://localhost:8042/api/registry/aut-num/*/*-c/*burble?raw | jq
{
"aut-num/AS4242422601": {
"admin-c": [
"BURBLE-DN42"
],
"tech-c": [
"BURBLE-DN42"
]
},
"aut-num/AS4242422602": {
"admin-c": [
"BURBLE-DN42"
],
"tech-c": [
"BURBLE-DN42"
]
}
}
```
A special query exists to return metadata about the registry
```
GET /api/registry/.meta
```
Example Output (JSON format):
```
wget -O - -q http://localhost:8042/api/dns/.meta | jq
```
```
{
"Commit": "fa89d022d0c2a48bcfbee405e2f3685f3b9cf063"
}
```
## Route Origin Authorisation (ROA) API
Route Origin Authorisation (ROA) data can be obtained from the server in
JSON and bird formats.
### JSON format output
```
GET /api/roa/json
```
Provides IPv4 and IPv6 ROAs in JSON format, suitable for use with
[gortr](https://github.com/cloudflare/gortr).
Example Output:
```
wget -O - -q http://localhost:8042/api/roa/json | jq
```
```
{
"metadata": {
"counts": 1564,
"generated": 1550402199,
"valid": 1550445399
},
"roas": [
{
"prefix": "172.23.128.0/26",
"maxLength": 29,
"asn": "AS4242422747"
},
{
"prefix": "172.22.129.192/26",
"maxLength": 29,
"asn": "AS4242423976"
},
{
"prefix": "10.110.0.0/16",
"maxLength": 24,
"asn": "AS65110"
},
... and so on
```
### Bird format output
```
GET /api/roa/bird/{bird version}/{IP family}
```
Provides ROA data suitable for including in to bird.
{bird version} must be either 1 or 2
{IP family} can be 4, 6 or 46 to provide both IPv4 and IPv6 results
Example Output:
```
wget -O - -q http://localhost:8042/api/roa/bird/1/4
```
```
#
# dn42regsrv ROA Generator
# Last Updated: 2019-02-17 11:16:39.668799525 +0000 GMT m=+0.279049704
# Commit: 3cbc349bf770493c016888ff785227ded2a7d866
#
roa 172.23.128.0/26 max 29 as 4242422747;
roa 172.22.129.192/26 max 29 as 4242423976;
roa 10.110.0.0/16 max 24 as 65110;
roa 172.20.164.0/26 max 29 as 4242423023;
roa 172.20.135.200/29 max 29 as 4242420448;
roa 10.65.0.0/20 max 24 as 4242420420;
roa 172.20.149.136/29 max 29 as 4242420234;
roa 10.160.0.0/13 max 24 as 65079;
roa 10.169.0.0/16 max 24 as 65534;
... and so on
```
```
wget -O - -q http://localhost:8042/api/roa/bird/2/6
```
```
#
# dn42regsrv ROA Generator
# Last Updated: 2019-02-17 11:16:39.668799525 +0000 GMT m=+0.279049704
# Commit: 3cbc349bf770493c016888ff785227ded2a7d866
#
route fdc3:10cd:ae9d::/48 max 64 as 4242420789;
route fd41:9805:7b69:4000::/51 max 64 as 4242420846;
route fd41:9805:7b69:4000::/51 max 64 as 4242420845;
route fd41:9805:7b69:4000::/51 max 64 as 4242420847;
route fddf:ebfd:a801:2331::/64 max 64 as 65530;
route fd42:1a2b:de57::/48 max 64 as 4242422454;
route fd42:7879:7879::/48 max 64 as 4242421787;
... and so on
```
### filter{,6}.txt
```
GET /api/roa/filter/{IP family}
```
Provides the contents of filter.txt and filter6.txt in json format.
{IP family} can be 4, 6 or 46 to provide both IPv4 and IPv6 results
Example Output:
```
wget -O - -q http://localhost:8042/api/roa/filter/6 | jq
```
```
[
{
"nr": 1001,
"action": "permit",
"prefix": "fd00::/8",
"minlen": 44,
"maxlen": 64
},
{
"nr": 9999,
"action": "deny",
"prefix": "::/0",
"minlen": 0,
"maxlen": 128
}
]
```
## DNS Root Zone API
The DNS API provides a list of resource records that can be used to create a root zone for DN42
related domains. By polling the API, DNS servers are able to keep their root zone delegations and
DNSSEC records up to date.
```
GET /api/dns/root-zone?format={[json|bind]}
```
Format may either 'json' or 'bind' to provide resource records in either format. The default
output format is JSON.
Example Output (JSON format):
```
wget -O - -q http://localhost:8042/api/dns/root-zone?format=json | jq
```
```
{
"Records": [
{
"Name": "dn42",
"Type": "NS",
"Content": "b.delegation-servers.dn42.",
"Comment": "DN42 Authoritative Zone"
},
{
"Name": "dn42",
"Type": "NS",
"Content": "j.delegation-servers.dn42.",
"Comment": "DN42 Authoritative Zone"
},
... and so on
```
Example Output (BIND format):
```
wget -O - -q http://localhost:8042/api/dns/root-zone?format=bind
```
```
;; DN42 Root Zone Records
;; Commit Reference: 2cc95d9101268ce82239dee1f947e4a8273524a9
;; Generated: 2019-03-08 19:40:51.264803795 +0000 GMT m=+0.197704585
dn42 IN NS b.delegation-servers.dn42. ; DN42 Authoritative Zone
dn42 IN NS j.delegation-servers.dn42. ; DN42 Authoritative Zone
dn42 IN NS y.delegation-servers.dn42. ; DN42 Authoritative Zone
dn42 IN DS 64441 10 2 6dadda00f5986bd26fe4f162669742cf7eba07d212b525acac9840ee06cb2799 ; DN42 Authoritative Zone
dn42 IN DS 56676 10 2 4b559c949eb796f5502f05bd5bb2143672e7ef935286db552955f291bb81093e ; DN42 Authoritative Zone
d.f.ip6.arpa IN NS b.delegation-servers.dn42. ; DN42 Authoritative Zone
d.f.ip6.arpa IN NS j.delegation-servers.dn42. ; DN42 Authoritative Zone
d.f.ip6.arpa IN NS y.delegation-servers.dn42. ; DN42 Authoritative Zone
d.f.ip6.arpa IN DS 64441 10 2 9057500a3b6e09bf45a60ed8891f2e649c6812d5d149c45a3c560fa0a619
5c49 ; DN42 Authoritative Zone
d.f.ip6.arpa IN DS 56676 10 2 d93cfd941025aaa445283d33e27157bb9a2df0a9c1389fdf5e36a377fc31
4736 ; DN42 Authoritative Zone
... and so on
```

21
LICENSE Normal file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 Simon Marsh
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

90
README.md

@ -1,3 +1,89 @@
# pipeline-test # dn42regsrv
A REST API for the DN42 registry, written in Go, to provide a bridge between
interactive applications and registry data.
A public instance of the API and explorer web app can be accessed via:
* [https://explorer.burble.com/](https://explorer.burble.com/) (Internet link)
* [http://explorer.collector.dn42/](http://explorer.collector.dn42/) (DN42 Link)
## Features
* REST API for querying DN42 registry objects
* Able to decorate objects with relationship information based on SCHEMA type definitions
* Includes a simple webserver for delivering static files which can be used to deliver
basic web applications utilising the API (such as the included DN42 Registry Explorer)
* Automatic pull from the DN42 git repository to keep the registry up to date
* Includes a responsive web app for exploring the registry
* API endpoints for ROA data in JSON, and bird formats
* API endpoint to support the creation of DNS root zone records
## Building
#### Using locally installed go
Requires [git](https://git-scm.com/) and [go](https://golang.org)
```
go get -insecure git.dn42.us/burble/dn42regsrv
```
#### Without installing go
Using container runtime to build with the golang container:
```
docker run -v ${PWD}:/go/bin golang go get -insecure git.dn42.us/burble/dn42regsrv
```
Or use the *contrib/build.sh* script after cloning the repo.
## Running
#### As a service
Use --help to view configurable options
```
${GOPATH}/bin/dn42regsrv --help
```
The server requires access to a clone of the DN42 registry and for
the git executable to be accessible.
If you want to use the auto pull feature then the registry must
also be writable by the server.
```
cd ${GOPTH}/src/git.dn42.us/burble/dn42regsrv
git clone http://git.dn42.us/dn42/registry.git
${GOPATH}/dn42regsrv
```
A sample service file is included for running the server under systemd
#### Within a container
A container build script (*contrib/buildah.sh*) is included in the
contrib directory. The script uses [buildah](https://buildah.io/).
See the *contrib/entrypoint.sh* script for environment variables that can
be set when running the container.
## Using
By default the server will be listening on port 8042.
See the [API.md](API.md) file for a detailed description of the API.
## Support
Please feel free to raise issues or create pull requests for the project git repository.
## #ToDo
### Server
- Add WHOIS interface
### DN42 Registry Explorer Web App
- Allow for attribute searches
Test pipeline

1
StaticRoot/anchorme.min.js vendored Normal file

File diff suppressed because one or more lines are too long

12
StaticRoot/bootstrap.min.css vendored Normal file

File diff suppressed because one or more lines are too long

BIN
StaticRoot/dn42_logo.png Normal file

Binary file not shown.
Side by Side

After

(image error) Size: 2.7 KiB

352
StaticRoot/explorer.js Normal file

@ -0,0 +1,352 @@
//////////////////////////////////////////////////////////////////////////
// DN42 Registry Explorer
//////////////////////////////////////////////////////////////////////////
// global store for data that is loaded once
const GlobalStore = {
data: {
RegStats: null,
Index: null
}
}
//////////////////////////////////////////////////////////////////////////
// registry object component
Vue.component('reg-object', {
template: '#reg-object-template',
props: [ 'link', 'content' ],
data() {
return { }
},
methods: {
updateSearch: function(str) {
vm.updateSearch(str)
}
},
computed: {
rtype: function() {
var ix = this.link.indexOf("/")
return this.link.substring(0, ix)
},
obj: function() {
var ix = this.link.indexOf("/")
return this.link.substring(ix + 1)
}
}
})
//////////////////////////////////////////////////////////////////////////
// reg-attribute component
Vue.component('reg-attribute', {
template: '#reg-attribute-template',
props: [ 'content' ],
data() {
return { }
},
methods: {
isRegObject: function(str) {
return (this.content.match(/^\[.*?\]\(.*?\)/) != null)
}
},
computed: {
objectLink: function() {
reg = this.content.match(/^\[(.*?)\]\((.*?)\)/)
return reg[2]
},
decorated: function() {
var c = this.content
// an attribute terminated with \n indicates a blank
// trailing line, however a single trailing <br/> will
// not be rendered in HTML, this hack doubles up the
// trailing newline so it creates a <br/> pair
// which renders as the blank line
if (c.substr(c.length-1) == "\n") {
c = c + "\n"
}
// replace newlines with line breaks
c = c.replace(/\n/g, "<br/>")
// decorate
c = anchorme(c, {
truncate: 40,
ips: false,
attributes: [ { name: "target", value: "_blank" } ]
})
// and return the final decorated content
return c
}
}
})
//////////////////////////////////////////////////////////////////////////
// search input component
Vue.component('search-input', {
template: '#search-input-template',
data() {
return {
search: '',
searchTimeout: 0
}
},
methods: {
debounceSearch: function(value) {
if (this.searchTimeout) {
clearTimeout(this.searchTimeout)
}
// link should be an absolute path
value = '/' + value
this.searchTimeout = setTimeout(
this.$router.push.bind(this.$router, value), 500
)
}
},
mounted() {
// listen to search updates and set the search text appropriately
this.$root.$on('SearchChanged', value => {
this.search = value
})
}
})
//////////////////////////////////////////////////////////////////////////
// registry-stats component
Vue.component('registry-stats', {
template: '#registry-stats-template',
data() {
return {
state: null,
error: null,
store: GlobalStore.data
}
},
methods: {
// just fetch the stats from the API server via axios
reload(event) {
this.store.RegStats = null
this.state = "loading"
axios
.get('/api/registry/')
.then(response => {
this.store.RegStats = response.data
this.state = 'complete'
})
.catch(error => {
this.error = error
this.state = 'error'
console.log(error)
})
}
},
mounted() {
if (this.store.RegStats == null) {
this.reload()
}
}
})
//////////////////////////////////////////////////////////////////////////
// root component
Vue.component('app-root', {
template: '#app-root-template',
mounted() {
this.$root.$emit('SearchChanged', '')
}
})
//////////////////////////////////////////////////////////////////////////
// search results
Vue.component('app-search', {
template: '#app-search-template',
data() {
return {
state: null,
search: null,
results: null,
store: GlobalStore.data
}
},
// trigger a search on route update and on mount
beforeRouteUpdate(to, from, next) {
this.search = to.params.pathMatch
this.doSearch()
next()
},
mounted() {
// store the search for later
this.search = this.$route.params.pathMatch
// the index must be loaded before any real work takes place
if (this.store.Index == null) {
this.loadIndex()
}
else {
// index was already loaded, go search
this.doSearch()
}
},
methods: {
// load the search index from the API
loadIndex() {
this.state = 'loading'
this.$root.$emit('SearchChanged', 'Initialising ...')
axios
.get('/api/registry/*')
.then(response => {
this.store.Index = response.data
// if a query parameter has been passed,
// then go search
if (this.search != null) {
this.$nextTick(this.doSearch())
}
})
.catch(error => {
this.state = 'error'
console.log(error)
})
},
// substring match object names against a search
matchObjects(objects, rtype, term) {
var results = [ ]
// check each object
for (const obj in objects) {
// matches are all lower case
var s = objects[obj].toLowerCase()
var pos = s.indexOf(term)
if (pos != -1) {
if ((pos == 0) && (s == term)) {
// exact match, return just this result
return [[ rtype, objects[obj] ]]
}
results.push([ rtype, objects[obj] ])
}
}
return results
},
// filter the index to find matches
searchFilter() {
var results = [ ]
var index = this.store.Index
// comparisons are lowercase
var term = this.search.toLowerCase()
// check if search includes a '/'
var slash = term.indexOf('/')
if (slash != -1) {
// match only on the specific type
var rtype = term.substring(0, slash)
var term = term.substring(slash + 1)
objects = index[rtype]
if (objects != null) {
results = this.matchObjects(objects, rtype, term)
}
}
else {
// walk though the entire index
for (const rtype in index) {
var objlist = this.matchObjects(index[rtype], rtype, term)
results = results.concat(objlist)
}
}
return results
},
// perform the search and present results
doSearch() {
// notify other components that the search is updated
this.$root.$emit('SearchChanged', this.search)
// filter matches against the index
filtered = this.searchFilter()
// got nothing ?
if (filtered.length == 0) {
this.state = "noresults"
}
// just one result
else if (filtered.length == 1) {
var objname = filtered[0]
// load the object from the API
this.state = 'loading'
query = '/api/registry/' + objname[0] + '/' + objname[1]
axios
.get(query)
.then(response => {
this.state = 'result'
this.result = response.data
})
.catch(error => {
this.error = error
this.state = 'error'
})
}
// lots of results
else {
this.state = "resultlist"
this.result = filtered
}
}
}
})
//////////////////////////////////////////////////////////////////////////
// main vue application starts here
// initialise the Vue Router
const router = new VueRouter({
routes: [
{ path: '/', component: Vue.component('app-root') },
{ path: '/*', component: Vue.component('app-search') }
]
})
// and the main app instance
const vm = new Vue({
el: '#explorer_app',
data: {
},
router
})
//////////////////////////////////////////////////////////////////////////
// end of code

218
StaticRoot/index.html Normal file

@ -0,0 +1,218 @@
<!doctype html>
<html lang="en">
<head>
<title>DN42 Registry Explorer</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<link rel="stylesheet" href="bootstrap.min.css">
<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
<!-- Style overrides -->
<style>
.material-icons { display:inline-flex;vertical-align:middle }
body { box-shadow: inset 0 2em 10em rgba(0,0,0,0.4); min-height: 100vh }
</style>
</head>
<body>
<div id="explorer_app">
<nav class="navbar navbar-fixed-top navbar-expand-md navbar-dark bg-dark">
<search-input></search-input>
<div class="collapse navbar-collapse w-100 ml-auto text-nowrap">
<div class="ml-auto"><router-link class="navbar-brand"
to="/">Registry Explorer</router-link>&nbsp;<a class="pull-right navbar-brand"
href="https://dn42.us/"><img src="/dn42_logo.png" width="173" height="60"/></a>
</div>
</div>
</nav>
<router-view></router-view>
</div>
<footer class="page-footer font-small">
<div style="margin-top: 20px; padding: 5px">
<a href="https://git.dn42.us/burble/dn42regsrv">Source Code</a>.
Powered by
<a href="https://getbootstrap.com/">Bootstrap</a>,
<a href="https://vuejs.org">Vue.js</a>,
<a href="https://github.com/axios/axios">axios</a>,
<a href="http://alexcorvi.github.io/anchorme.js/">Anchorme</a>.
</div>
</footer>
<script type="text/x-template" id="search-input-template">
<form class="form-inline" id="SearchForm">
<input v-bind:value="search"
v-on:input="debounceSearch($event.target.value)"
class="form-control-lg" size="30" type="search"
placeholder="Search the registry" aria-label="Search"/>
</form>
</script>
<script type="text/x-template" id="app-root-template">
<div class="p-3">
<div class="jumbotron">
<h1>DN42 Registry Explorer</h1>
<p class="lead">Just start typing in the search box to start searching the registry</p>
<hr/>
<p>
<p>Search Tips</p>
<ul>
<li>Searches are case independent
<li>No need to hit enter, searches will start immediately
<li>Prefixing the search by a registry type followed by / will narrow the search to
just that type (e.g. <router-link class="text-success"
to="domain/.dn42">domain/.dn42</router-link>)</li>
<li>Searching for <b>type/</b> will return all the objects for that type (e.g.
<router-link class="text-success" to="schema/">schema/</router-link>)</li>
<li>A blank search box will return you to these instructions</li>
<li>Just copy the URL to link to search results</li>
<li>Searches are made on object names; searching the content of objects
is not supported (yet!).</li>
</ul>
<hr/>
<p>
The registry explorer is a simple web app using
<a href="https://git.dn42.us/burble/dn42regsrv">dn42regsrv</a>;
a REST API for the DN42 registry built with <a href="https://golang.org/">Go</a>
<br/>
The DN42 registry contains personal data which is also available through this site.
Please refer to the DN42 registry privacy policy, or contact dn42@burble.com for any
queries.
</p>
</div>
<registry-stats></registry-stats>
</div>
</script>
<script type="text/x-template" id="registry-stats-template">
<div class="container d-flex flex-column w-75">
<h5>Registry Stats</h5>
<section v-if="state == 'error'">
<div class="alert alert-warning clearfix" role="alert">
An error recurred whilst retrieving data from the API
<button type="button" class="float-right btn btn-primary"
v-on:click="reload"><span class="material-icons">refresh</span>
Refresh</button>
</div>
</section>
<section v-else-if="state == 'loading'">
<div class="alert alert-info" role="alert">Loading data ...</div>
</section>
<section v-else>
<p class="p-3 text-center">
<span v-for="(value, key) in store.RegStats"
class="mx-2 text-nowrap d-inline-block">
<router-link v-bind:to="'/'+key+'/'">{{ key }}</router-link>
: <b>{{ value }}</b>
</span>
</p>
</section>
</div>
</script>
<script type="text/x-template" id="app-search-template">
<div class="p-3">
<section v-if="state == 'loading'">
<div class="alert alert-info" role="alert">Loading data ...</div>
</section>
<section v-else-if="state == 'error'">
<div class="alert alert-warning clearfix" role="alert">
An error recurred whilst retrieving search results
</div>
</section>
<section v-else-if="state == 'noresults'">
<h2>Searching for "{{ search }}" ...</h2>
<div class="alert alert-dark" role="alert">
Sorry, no results found
</div>
</section>
<section v-else-if="state == 'resultlist'">
<h2>Listing results for "{{ search }}" ...</h2>
<div class="container d-flex flex-row flex-wrap">
<div class="text-center">
<span v-for="ref in result">
<reg-object class="mx-4 my-2" v-bind:link="ref[0]+'/'+ref[1]"></reg-object>
</span>
</div>
</div>
</section>
<section v-else-if="state == 'result'">
<div v-for="(val, key) in result">
<h2 class="mb-4"><reg-object v-bind:link="key"></reg-object></h2>
<div class="pl-4">
<table class="table">
<thead>
<tr><th scope="col">Key</th><th scope="col">Value</th></tr>
</thead>
<tbody>
<tr v-for="a in val.Attributes">
<th scope="row" class="border-0 py-1 pr-3 text-primary text-nowrap">
{{ a[0] }}
</th>
<td class="border-0 p-1">
<reg-attribute v-bind:content="a[1]"></reg-attribute>
</td>
</tr>
</tbody>
</table>
</div>
<section v-if="val.Backlinks.length != 0">
<p>Referenced by</p>
<div class="pl-4">
<table class="table">
<tbody>
<tr v-for="rlink in val.Backlinks">
<td class="border-0 p-1">
<reg-object v-bind:link="rlink"></reg-object>
</td>
</tr>
</tbody>
</table>
</div>
</section>
</div>
</section>
<section v-else>
<div class="alert alert-warning" role="alert">
Something went wrong, an invalid search state was encountered ({{ state }})
<p>{{ $route.params.pathMatch }}</p>
</div>
</section>
</div>
</script>
<script type="text/x-template" id="reg-object-template">
<span class="text-nowrap d-inline-block">
<router-link class="text-success mr-2" v-bind:to="'/'+link">
{{ ( content ? content : obj ) }}
</router-link>
<span class="badge back-pill badge-dark test-muted">{{ rtype }}</span>
</span>
</script>
<script type="text/x-template" id="reg-attribute-template">
<span style="word-break: break-all; white-space: pre-wrap">
<reg-object v-if="isRegObject()" v-bind:link="objectLink"></reg-object>
<span v-else class="text-monospace" v-html="decorated"></span>
</span>
</script>
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js" integrity="sha384-B0UglyR+jN6CkvvICOB2joaf5I4l3gm9GU6Hc1og6Ls7i6U/mkkaduKaBhlAXv9k" crossorigin="anonymous"></script>
<script src="https://unpkg.com/vue"></script -->
<script src="https://unpkg.com/vue-router"></script -->
<script src="https://unpkg.com/axios@0.18.0/dist/axios.min.js"></script>
<!-- script src="https://unpkg.com/vue@2.6.2/dist/vue.min.js"></script -->
<!-- script src="https://unpkg.com/vue-router@3.0.2/dist/vue-router.js"></script -->
<script src="anchorme.min.js"></script>
<script src="explorer.js"></script>
</body>
</html>

26
contrib/build.sh Executable file

@ -0,0 +1,26 @@
#!/bin/bash
##########################################################################
# A small script to build a static dn42regsrv image
# using the golang container image
#
# the binary will be built in to the current directory
##########################################################################
RUNTIME=$(which podman || which docker)
echo "Using container runtime: ${RUNTIME}"
# find the source directory
SCRIPTPATH="$(cd "$(dirname "$0")" ; pwd -P)"
SOURCEPATH="$(cd "${SCRIPTPATH}/../"; pwd -P)"
echo "Source is in: ${SOURCEPATH}"
# do the thing
${RUNTIME} run --rm \
-e CGO_ENABLED=0 \
-v "${SOURCEPATH}:/go/src/dn42regsrv" \
-v "${PWD}:/go/bin" \
-w "/go/src/dn42regsrv" \
docker.io/golang:1.12 \
go get
##########################################################################
# end of code

65
contrib/buildah.sh Executable file

@ -0,0 +1,65 @@
#!/bin/bash
##########################################################################
echo "Building dn42regsrv container"
# find the source directory
SCRIPTPATH="$(cd "$(dirname "$0")" ; pwd -P)"
SOURCEPATH="$(cd "${SCRIPTPATH}/../"; pwd -P)"
echo "Source is in: ${SOURCEPATH}"
##########################################################################
DEPS='git'
B=$(which buildah)
# initialise container
c=$(buildah from --name dn42regsrv-working docker.io/debian:buster)
##########################################################################
# install dependencies and initialise directories
$B run $c -- bash <<EOF
apt-get -y update
apt-get -y install --no-install-recommends $DEPS
rm -r /var/lib/apt/lists
EOF
# mount the container
m=$($B mount $c)
# create directories and copy the web app
mkdir "$m/app" "$m/data" "$m/registry" "$m/data/ssh"
# web app
cp -r "$SOURCEPATH/StaticRoot" "$m/data/webapp"
# add the entrypoint.sh script
cp "$SOURCEPATH/contrib/entrypoint.sh" "$m/app"
chmod +x "$m/app"
# build the binary directly in to the container
pushd "$m/app"
"$SOURCEPATH/contrib/build.sh"
popd
# unmount the container
$B unmount $c
# configure
buildah config \
--author="Simon Marsh" \
--workingdir='/data/registry' \
--cmd='/app/entrypoint.sh' \
$c
##########################################################################
# finally create the image
echo "Committing image ..."
i=$($B commit --squash $c dn42regsrv)
# clean up
$B rm $c
##########################################################################
# end of file

33
contrib/dn42regsrv.service Normal file

@ -0,0 +1,33 @@
##########################################################################
# dn42regsrv example systemd service file
##########################################################################
[Unit]
Description=DN42 Registry API Server
After=network.target
[Install]
WantedBy=multi-user.target
[Service]
User=regsrv
Group=registry
Type=simple
Restart=on-failure
# service hardening
ProtectSystem=strict
ReadOnlyPaths=/home/regsrv/go/src/git.dn42.us/burble/dn42regsrv/StaticRoot
ReadWritePaths=/home/regsrv/registry
NoNewPrivileges=yes
ProtectControlGroups=yes
PrivateTmp=yes
PrivateDevices=yes
DevicePolicy=closed
MemoryDenyWriteExecute=yes
#
ExecStart=/home/regsrv/go/bin/dn42regsrv \
-s /home/regsrv/go/src/git.dn42.us/burble/dn42regsrv/StaticRoot \
-d /home/regsrv/registry
#########################################################################
# end of file

5
contrib/docker/README.md Normal file

@ -0,0 +1,5 @@
# How to run
* Run ./build.sh
* Rename _env to .env and replace REGISTRYDIR with the path to your dn42 registry clone
* docker-compose up

1
contrib/docker/_env Normal file

@ -0,0 +1 @@
REGISTRYDIR=/somedir

6
contrib/docker/build.sh Executable file

@ -0,0 +1,6 @@
#!/bin/bash
docker run -it -v $(dirname "$(dirname $PWD)"):/go/src/dn42regsrv golang:alpine ash -c 'apk add git && cd src/dn42regsrv && go get && cp /go/bin/dn42regsrv .'
cd ../../
docker build -t dn42regsrv -f contrib/docker/build/Dockerfile .
rm -f dn42regsrv

6
contrib/docker/build/Dockerfile Normal file

@ -0,0 +1,6 @@
FROM alpine:latest
WORKDIR /app
RUN apk add git
COPY dn42regsrv /app/
COPY StaticRoot /app/StaticRoot
ENTRYPOINT ["/app/dn42regsrv"]

9
contrib/docker/docker-compose.yml Normal file

@ -0,0 +1,9 @@
version: '2'
services:
dn42regsrv:
image: dn42regsrv
restart: always
ports:
- 127.0.0.1:8042:8042
volumes:
- ${REGISTRYDIR}:/app/registry

22
contrib/entrypoint.sh Executable file

@ -0,0 +1,22 @@
#!/bin/bash
##########################################################################
DN42REGSRV_WEBAPP=${DN42REGSRV_WEBAPP:-/data/webapp}
DN42REGSRV_REGDIR=${DN42REGSRV_REGDIR:-/data/registry}
DN42REGSRV_BRANCH=${DN42REGSRV_BRANCH:-master}
DN42REGSRV_BIND=${DN42REGSRV_BIND:-[::]:8042}
DN42REGSRV_INTERVAL=${DN42REGSRV_INTERVAL:-10m}
DN42REGSRV_LOGLVL=${DN42REGSRV_LOGLVL:-info}
DN42REGSRV_AUTOPULL=${DN42REGSRV_AUTOPULL:-true}
exec /app/dn42regsrv \
-s "$DN42REGSRV_WEBAPP" \
-d "$DN42REGSRV_REGDIR" \
-p "$DN42REGSRV_BRANCH" \
-b "$DN42REGSRV_BIND" \
-i "$DN42REGSRV_INTERVAL" \
-l "$DN42REGSRV_LOGLVL" \
-a "$DN42REGSRV_AUTOPULL"
##########################################################################
# end of file

294
contrib/sync_rootzone.sh Executable file

@ -0,0 +1,294 @@
#!/bin/bash
##########################################################################
#
# This script is intended to be run via cron job to sync a local,
# authoritative DNS server's root zone with the registry information
# provided by dn42regsrv.
#
# As is, the script is specific to updating PDNS within the burble.dn42
# network, however it is also intended to be easily adaptable to other
# DNS servers and networks
#
##########################################################################
##########################################################################
# This array is used to define additional, local networks that the
# DNS server may be authoritative for. The array is used to prevent
# the related resource records from being removed automatically,
# as any records not listed here or in the registry will get deleted.
# Make sure to include '.' here or the local NS and SOA records
# will get removed
IGNORE_RECORDS=(
'.'
'$ORIGIN'
'ns1.burble.dn42'
'burble.dn42'
'collector.dn42'
'1.0.6.2.2.4.2.4.2.4.d.f.ip6.arpa'
'160/27.129.20.172.in-addr.arpa'
'0/27.129.20.172.in-addr.arpa'
)
##########################################################################
# The functions here are used to actually update the DNS server
# Change these to use a different server than PDNS
PDNSUTIL='/usr/bin/pdnsutil'
PDNSCTRL='/usr/bin/pdns_control'
# replace_rr <name> <type> <content>
function replace_rr {
local rr_name=$1; shift
local rr_type=$1; shift
local rr_content=$*
echo "Replace: ${rr_name} ${rr_type} '${rr_content}'"
if [ ${DEBUG} -eq 0 ]
then
${PDNSUTIL} replace-rrset . ${rr_name} ${rr_type} "${rr_content}"
fi
}
# delete_rr <name> <type>
function delete_rr {
local rr_name=$1
local rr_type=$2
echo "Delete: ${rr_name} ${rr_type}"
if [ ${DEBUG} -eq 0 ]
then
${PDNSUTIL} delete-rrset . ${rr_name} ${rr_type}
fi
}
# list the current contents of the root zone
function get_current_root_zone {
local rra rr_name rr_type rr_content
while read -r -a rra
do
rr_name=${rra[0]}
rr_type=${rra[3]}
rr_content=${rra[@]:4}
current_rz["${rr_name} ${rr_type}"]=${rr_content}
done < <(${PDNSUTIL} list-zone .)
}
# update the . SOA record after a change
function update_soa {
echo "Incrementing SOA serial"
if [ ${DEBUG} -eq 0 ]
then
${PDNSUTIL} increase-serial .
fi
}
# used to trigger a notify to any slaves of this server
function notify_slaves {
echo "Notfying slaves"
if [ ${DEBUG} -eq 0 ]
then
${PDNSCTRL} notify .
fi
}
##########################################################################
# No further local customisation should be needed from here
##########################################################################
# initialise script parameters and global vars
function usage {
echo "Usage: $0 [-h] [-d] [-a URL] [-c FILE]"
echo " -h: this help"
echo " -d: enable debugging and don't action changes"
echo " -a: URL to dn42regsrv API"
echo " -c: file in which to store previous commit number"
}
# default options
DEBUG=0
APIURL="http://explorer.burble.dn42/api"
COMMITFILE="/tmp/.sync_rz_commit"
# parse any arguments passed to the script
while getopts ":hda:" opt
do
case ${opt} in
d)
DEBUG=1
;;
a)
APIURL=${OPTARG}
;;
*)
usage
exit 0
;;
esac
done
# global vars
declare -A current_rz
declare -A new_rz
current_commit=''
new_commit=''
deleted_records=0
updated_records=0
##########################################################################
# fetch and parse the root zone data from the API
function fetch_new_root_zone {
local line fields rr_name rr_type rr_content
while read -r line
do
if [[ ${line} == ';; Commit Reference:'* ]]
then
new_commit=${line#;; Commit Reference: }
else
# strip out comments and create array
fields=( ${line%%;*} )
# if the line is a valid record
if [ ${#fields[@]} -ge 4 ]
then
rr_name=${fields[0]}
rr_type=${fields[2]}
rr_content=${fields[@]:3}
new_rz["${rr_name} ${rr_type}"]=${rr_content}
fi
fi
done < <(/usr/bin/wget -O - -q "${APIURL}/dns/root-zone?format=bind")
if [ ${DEBUG} -eq 1 ]; then echo "New Commit: ${new_commit}"; fi
}
##########################################################################
# load and store the previous commit
function load_current_commit {
read -r current_commit < "${COMMITFILE}"
if [ ${DEBUG} -eq 1 ]; then echo "Current Commit: ${current_commit}"; fi
}
function store_current_commit {
if [ ${DEBUG} -eq 0 ]
then
echo "${1}" > "${COMMITFILE}"
fi
}
##########################################################################
# remove records that have been deleted
function is_ignored {
local rr_name=$1
for i in "${IGNORE_RECORDS[@]}"
do
[ "${i}" == "${rr_name}" ] && echo "ignored"
done
echo ""
}
function remove_deleted_records {
local key rr_name ignored
deleted_records=0
# check each record in the old root zone
for key in "${!current_rz[@]}"
do
ignored=$(is_ignored ${key% })
# if record is not ignored, and no new record exists
if [ "${ignored}" == '' -a "${new_rz[${key}]}" == '' ]
then
# then get rid of it
delete_rr ${key}
deleted_records=$((deleted_records + 1))
fi
done
echo "Deleted ${deleted_records} records"
}
##########################################################################
# update records that have been added or changed
function update_new_records {
local key content
updated_records=0
# check each new record
for key in "${!new_rz[@]}"
do
content="${new_rz[${key}]}"
# if old record didn't exist, or content differs
if [ "${current_rz[${key}]}" != "${content}" ]
then
# update the record
replace_rr $key ${content}
updated_records=$((updated_records + 1))
fi
done
echo "Updated ${updated_records} records"
}
##########################################################################
# main flow of the script starts here
echo "DN42 Root Zone Sync"
date
echo
fetch_new_root_zone
# check that the commit was populated
if [ "${new_commit}" == '' ]
then
echo "Unable to fetch new root zone, aborting"
exit 1
fi
load_current_commit
# now check if anything actually needs to be done
if [ "${new_commit}" == "${current_commit}" ]
then
echo "Commits are equal, nothing to do"
exit 0
fi
get_current_root_zone
# apply changes
remove_deleted_records
update_new_records
# bail out if there were no actual differences
if [ $((deleted_records + updated_records)) -eq 0 ]
then
echo "No records were updated, exiting"
else
# update the SOA and send out a notification to slaves
update_soa
notify_slaves
fi
# finally store the new commit to show it's been updated
store_current_commit "${new_commit}"
echo "All done"
##########################################################################
# end of code

202
dn42regsrv.go Normal file

@ -0,0 +1,202 @@
//////////////////////////////////////////////////////////////////////////
// DN42 Registry API Server
//////////////////////////////////////////////////////////////////////////
package main
//////////////////////////////////////////////////////////////////////////
import (
"context"
"encoding/json"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
flag "github.com/spf13/pflag"
"net/http"
"os"
"os/signal"
"time"
)
//////////////////////////////////////////////////////////////////////////
// simple event bus
type NotifyFunc func(...interface{})
type SimpleEventBus map[string][]NotifyFunc
var EventBus = make(SimpleEventBus)
// add a listener to an event
func (bus SimpleEventBus) Listen(event string, nfunc NotifyFunc) {
bus[event] = append(bus[event], nfunc)
}
// fire notifications for an event
func (bus SimpleEventBus) Fire(event string, params ...interface{}) {
funcs := bus[event]
if funcs != nil {
for _, nfunc := range funcs {
nfunc(params...)
}
}
}
//////////////////////////////////////////////////////////////////////////
// utility func for returning JSON from an API endpoint
func ResponseJSON(w http.ResponseWriter, v interface{}) {
// for response time testing
//time.Sleep(time.Second)
// marshal the JSON string
data, err := json.Marshal(v)
if err != nil {
log.WithFields(log.Fields{
"error": err,
}).Error("Failed to marshal JSON")
}
// write back to http handler
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Write(data)
}
//////////////////////////////////////////////////////////////////////////
// utility function to set the log level
func setLogLevel(levelStr string) {
if level, err := log.ParseLevel(levelStr); err != nil {
// failed to set the level
// set a sensible default and, of course, log the error
log.SetLevel(log.InfoLevel)
log.WithFields(log.Fields{
"loglevel": levelStr,
"error": err,
}).Error("Failed to set requested log level")
} else {
// set the requested level
log.SetLevel(level)
}
}
//////////////////////////////////////////////////////////////////////////
// http request logger
func requestLogger(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.WithFields(log.Fields{
"method": r.Method,
"URL": r.URL.String(),
"Remote": r.RemoteAddr,
}).Debug("HTTP Request")
next.ServeHTTP(w, r)
})
}
//////////////////////////////////////////////////////////////////////////
// everything starts here
func main() {
// set a default log level, so that logging can be used immediately
// the level will be overidden later on once the command line
// options are loaded
log.SetLevel(log.InfoLevel)
log.Info("DN42 Registry API Server Starting")
// declare cmd line options
var (
logLevel = flag.StringP("LogLevel", "l", "Info", "Log level")
regDir = flag.StringP("RegDir", "d", "registry", "Registry data directory")
bindAddress = flag.StringP("BindAddress", "b", "[::]:8042", "Server bind address")
staticRoot = flag.StringP("StaticRoot", "s", "StaticRoot", "Static page directory")
refreshInterval = flag.StringP("Refresh", "i", "60m", "Refresh interval")
gitPath = flag.StringP("GitPath", "g", "/usr/bin/git", "Path to git executable")
autoPull = flag.BoolP("AutoPull", "a", true, "Automatically pull the registry")
branch = flag.StringP("Branch", "p", "master", "git branch to pull")
)
flag.Parse()
// now initialise logging properly based on the cmd line options
setLogLevel(*logLevel)
// parse the refreshInterval and start data collection
interval, err := time.ParseDuration(*refreshInterval)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"interval": *refreshInterval,
}).Fatal("Unable to parse registry refresh interval")
}
InitialiseRegistryData(*regDir, interval,
*gitPath, *autoPull, *branch)
// initialise router
router := mux.NewRouter()
// log all access
router.Use(requestLogger)
// add API routes
subr := router.PathPrefix("/api").Subrouter()
EventBus.Fire("APIEndpoint", subr)
// initialise static routes
InstallStaticRoutes(router, *staticRoot)
// initialise http server
server := &http.Server{
Addr: *bindAddress,
WriteTimeout: time.Second * 15,
ReadTimeout: time.Second * 15,
IdleTimeout: time.Second * 60,
Handler: router,
}
// run the server in a non-blocking goroutine
log.WithFields(log.Fields{
"BindAddress": *bindAddress,
}).Info("Starting server")
go func() {
if err := server.ListenAndServe(); err != nil {
log.WithFields(log.Fields{
"error": err,
"BindAddress": *bindAddress,
}).Fatal("Unable to start server")
}
}()
// graceful shutdown via SIGINT (^C)
csig := make(chan os.Signal, 1)
signal.Notify(csig, os.Interrupt)
// and block
<-csig
log.Info("Server shutting down")
// deadline for server to shutdown
ctx, cancel := context.WithTimeout(context.Background(), 10)
defer cancel()
// shutdown the server
server.Shutdown(ctx)
// nothing left to do
log.Info("Shutdown complete, all done")
os.Exit(0)
}
//////////////////////////////////////////////////////////////////////////
// end of code

234
dnsapi.go Normal file

@ -0,0 +1,234 @@
//////////////////////////////////////////////////////////////////////////
// DN42 Registry API Server
//////////////////////////////////////////////////////////////////////////
package main
//////////////////////////////////////////////////////////////////////////
import (
"fmt"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
// "math/big"
"net/http"
"strings"
"time"
)
//////////////////////////////////////////////////////////////////////////
// register the api
func init() {
EventBus.Listen("APIEndpoint", InitDNSAPI)
EventBus.Listen("RegistryUpdate", DNSUpdate)
}
//////////////////////////////////////////////////////////////////////////
// data model
// very simple DNS record data
type DNSRecord struct {
Name string
Type string
Content string
Comment string `json:",omitempty"`
}
type DNSZone struct {
Records []*DNSRecord
Commit string
Generated time.Time
}
var DNSRootZone *DNSZone
//////////////////////////////////////////////////////////////////////////
// fixed set of authoritative zones
var DNSRootAuthZones = map[string]string{
"dn42": "domain/dn42",
"recursive-servers.dn42": "domain/recursive-servers.dn42",
"delegation-servers.dn42": "domain/delegation-servers.dn42",
"d.f.ip6.arpa": "inet6num/fd00::_8",
"20.172.in-addr.arpa": "inetnum/172.20.0.0_16",
"21.172.in-addr.arpa": "inetnum/172.21.0.0_16",
"22.172.in-addr.arpa": "inetnum/172.22.0.0_16",
"23.172.in-addr.arpa": "inetnum/172.23.0.0_16",
"31.172.in-addr.arpa": "inetnum/172.31.0.0_16",
"10.in-addr.arpa": "inetnum/10.0.0.0_8",
}
//////////////////////////////////////////////////////////////////////////
// called from main to initialise the API routing
func InitDNSAPI(params ...interface{}) {
router := params[0].(*mux.Router)
s := router.
Methods("GET").
PathPrefix("/dns").
Subrouter()
s.HandleFunc("/root-zone", dnsRZoneHandler)
log.Info("DNS API installed")
}
//////////////////////////////////////////////////////////////////////////
// api handlers
// return records that should be included in a DN42 root zone
func dnsRZoneHandler(w http.ResponseWriter, r *http.Request) {
var format []string
query := r.URL.Query()
format = query["format"]
if format == nil || len(format) != 1 {
format = []string{"json"}
}
switch format[0] {
case "bind":
DNSRootZone.WriteBindFormat(w)
case "json":
ResponseJSON(w, DNSRootZone)
default:
ResponseJSON(w, DNSRootZone)
}
}
//////////////////////////////////////////////////////////////////////////
// called whenever the registry is updated
func DNSUpdate(params ...interface{}) {
registry := params[0].(*Registry)
// path := params[1].(string)
zone := &DNSZone{
Generated: time.Now(),
Commit: registry.Commit,
}
// add zones that are authoritative within DN42
for name, object := range DNSRootAuthZones {
zone.AddRecords(registry, name, object, "DN42 Authoritative Zone")
}
// search all domain objects and add stub records for each TLD
rtype := registry.Types["domain"]
for name, object := range rtype.Objects {
// domain is a TLD if it doesn't contain a '.'
if strings.IndexRune(name, '.') == -1 {
// don't include zones which are authoritative within DN42
if DNSRootAuthZones[name] == "" {
zone.AddRecords(registry, name, object.Ref, "Forward Zone")
}
}
}
DNSRootZone = zone
}
//////////////////////////////////////////////////////////////////////////
// utility function to add a DNS record to a zone
func (zone *DNSZone) AddRecord(name string, t string,
content string, comment string) {
record := &DNSRecord{
Name: name,
Type: t,
Content: content,
Comment: comment,
}
zone.Records = append(zone.Records, record)
}
//////////////////////////////////////////////////////////////////////////
// add nserver and ds-rdata records from a registry object
func (zone *DNSZone) AddRecords(registry *Registry, name string,
path string, comment string) {
// use the registry metadata key index to find the appropriate values
object := registry.GetObject(path)
if object == nil {
log.WithFields(log.Fields{
"zone": name,
"path": path,
}).Error("DNS: unable to find object in registry")
return
}
nserver := object.GetKey("nserver")
for _, ns := range nserver {
// check if stub record needs to be added
fields := strings.Split(ns.RawValue, " ")
if len(fields) == 2 {
// add a record for the NS, together with a stub A or AAAA record
var stubtype string
if strings.IndexRune(fields[1], ':') == -1 {
// no : so IPv4
stubtype = "A"
} else {
// has : so IPv6
stubtype = "AAAA"
}
zone.AddRecord(name, "NS", fields[0]+".", comment)
zone.AddRecord(fields[0], stubtype, fields[1], comment)
} else {
// no, just add an NS record as it was presented
zone.AddRecord(name, "NS", ns.RawValue+".", comment)
}
}
dsrdata := object.GetKey("ds-rdata")
for _, ds := range dsrdata {
zone.AddRecord(name, "DS", ds.RawValue, comment)
}
}
//////////////////////////////////////////////////////////////////////////
// Functions for outputting zone records in different formats
func (r *DNSRecord) ToBindString() string {
var comment string
if r.Comment == "" {
comment = ""
} else {
comment = "\t; " + r.Comment
}
return fmt.Sprintf("%s\tIN\t%s\t%s%s",
r.Name, r.Type, r.Content, comment,
)
}
func (zone *DNSZone) WriteBindFormat(w http.ResponseWriter) {
w.Header().Set("Content-Type", "text/plain")
w.Header().Set("Access-Control-Allow-Origin", "*")
// provide a header
fmt.Fprintf(w, ";; DN42 Root Zone Records\n"+
";; Commit Reference: %s\n;; Generated: %s\n",
zone.Commit, zone.Generated)
// then simply output each record in turn
for _, record := range zone.Records {
fmt.Fprintln(w, record.ToBindString())
}
}
//////////////////////////////////////////////////////////////////////////
// end of code

536
regapi.go Normal file

@ -0,0 +1,536 @@
//////////////////////////////////////////////////////////////////////////
// DN42 Registry API Server
//////////////////////////////////////////////////////////////////////////
package main
//////////////////////////////////////////////////////////////////////////
import (
// "fmt"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
"net/http"
"strings"
// "time"
)
//////////////////////////////////////////////////////////////////////////
// data structures
type RegMetaReturn struct {
Commit string
}
//////////////////////////////////////////////////////////////////////////
// register the api
func init() {
EventBus.Listen("APIEndpoint", InitRegistryAPI)
}
//////////////////////////////////////////////////////////////////////////
// called from main to initialise the API routing
func InitRegistryAPI(params ...interface{}) {
router := params[0].(*mux.Router)
s := router.
Methods("GET").
PathPrefix("/registry").
Subrouter()
s.HandleFunc("/", regRootHandler)
//s.HandleFunc("/.schema", rTypeListHandler)
//s.HandleFunc("/.meta/", rTypeListHandler)
s.HandleFunc("/.meta", regMetaHandler)
s.HandleFunc("/{type}", regTypeHandler)
s.HandleFunc("/{type}/{object}", regObjectHandler)
s.HandleFunc("/{type}/{object}/{key}", regKeyHandler)
s.HandleFunc("/{type}/{object}/{key}/{attribute}", regAttributeHandler)
log.Info("Registry API installed")
}
//////////////////////////////////////////////////////////////////////////
// return registry metadata
func regMetaHandler(w http.ResponseWriter, r *http.Request) {
rv := RegMetaReturn{
Commit: RegistryData.Commit,
}
ResponseJSON(w, rv)
}
//////////////////////////////////////////////////////////////////////////
// filter functions
// return a list of types that match the filter
func filterTypes(filter string) []*RegType {
var rtypes []*RegType = nil
// check if filter starts with '*'
if filter[0] == '*' {
// try and match the filter against all reg types
filter = strings.ToLower(filter[1:])
// special case, if the filter was '*' return all types
if len(filter) == 0 {
rtypes = make([]*RegType, 0, len(RegistryData.Types))
for _, rtype := range RegistryData.Types {
rtypes = append(rtypes, rtype)
}
} else {
// otherwise substring match the types
for _, rtype := range RegistryData.Types {
lname := strings.ToLower(rtype.Ref)
if strings.Contains(lname, filter) {
// matched, add it to the list
rtypes = append(rtypes, rtype)
}
}
}
} else {
// perform an exact match with one entry
rtype := RegistryData.Types[filter]
if rtype != nil {
// return a single answer
rtypes = []*RegType{rtype}
}
}
return rtypes
}
// return a list of objects from a set of types that match a filter
func filterObjects(rtypes []*RegType, filter string) []*RegObject {
var objects []*RegObject = nil
// check if filter starts with '*'
if filter[0] == '*' {
// try and match objects against the filter
filter = strings.ToLower(filter[1:])
// for each type
for _, rtype := range rtypes {
// special case, if the filter was '*' return all objects
if len(filter) == 0 {
objs := make([]*RegObject, 0, len(rtype.Objects))
for _, object := range rtype.Objects {
objs = append(objs, object)
}
objects = append(objects, objs...)
} else {
// otherwise substring match the object names
for _, object := range rtype.Objects {
lname := strings.ToLower(object.Ref)
if strings.Contains(lname, filter) {
// matched, add it to the list
objects = append(objects, object)
}
}
}
}
} else {
// perform an exact match against one object for each type
for _, rtype := range rtypes {
object := rtype.Objects[filter]
if object != nil {
// add the object
objects = append(objects, object)
}
}
}
return objects
}
// return a list of key indices matching the filter
func filterKeys(rtypes []*RegType, filter string) []*RegKeyIndex {
var ix []*RegKeyIndex = nil
// check if filter starts with '*'
if filter[0] == '*' {
// try and match keys against the filter
filter = strings.ToLower(filter[1:])
// for each type
for _, rtype := range rtypes {
ref := rtype.Ref
schema := RegistryData.Schema[ref]
// special case, if the filter was '*' return all indices
if len(filter) == 0 {
tmp := make([]*RegKeyIndex, 0, len(schema.KeyIndex))
for _, keyix := range schema.KeyIndex {
tmp = append(tmp, keyix)
}
ix = append(ix, tmp...)
} else {
// otherwise substring match the key names
for kname, keyix := range schema.KeyIndex {
kname = strings.ToLower(kname)
if strings.Contains(kname, filter) {
ix = append(ix, keyix)
}
}
}
}
} else {
// perform an exact match, one key for each type
for _, rtype := range rtypes {
ref := rtype.Ref
schema := RegistryData.Schema[ref]
keyix := schema.KeyIndex[filter]
if keyix != nil {
// add the index
ix = append(ix, keyix)
}
}
}
return ix
}
// helper func to determine if an attribute matches a filter
func matchAttribute(attribute *RegAttribute,
filter string, isExact bool) bool {
if isExact {
return filter == attribute.RawValue
} else {
l := strings.ToLower(attribute.RawValue)
return strings.Contains(l, filter)
}
}
// return a map of objects and attribute values that match the filter
func filterAttributes(ix []*RegKeyIndex, objects []*RegObject,
filter string, raw bool) map[string]map[string][]string {
result := make(map[string]map[string][]string)
// pre-calculate the search type
isExact := true
isAll := false
if filter[0] == '*' {
isExact = false
filter = strings.ToLower(filter[1:])
if len(filter) == 0 {
isAll = true
}
}
// for each key index
for _, keyix := range ix {
// for each object
for _, object := range objects {
// attributes in this object that match this key
attributes := keyix.Objects[object]
if attributes != nil {
// this object has at least one relevant key
// match the attributes
for _, attribute := range attributes {
if isAll || matchAttribute(attribute, filter, isExact) {
// match found !
objmap := result[object.Ref]
if objmap == nil {
objmap = make(map[string][]string)
result[object.Ref] = objmap
}
// append the result
var value *string
if raw {
value = &attribute.RawValue
} else {
value = &attribute.Value
}
objmap[keyix.Ref] = append(objmap[keyix.Ref], *value)
}
}
}
}
}
return result
}
//////////////////////////////////////////////////////////////////////////
// root handler, lists all types within the registry
func regRootHandler(w http.ResponseWriter, r *http.Request) {
response := make(map[string]int)
for _, rType := range RegistryData.Types {
response[rType.Ref] = len(rType.Objects)
}
ResponseJSON(w, response)
}
//////////////////////////////////////////////////////////////////////////
// type handler returns list of objects that match the type
func regTypeHandler(w http.ResponseWriter, r *http.Request) {
// request parameters
vars := mux.Vars(r)
tFilter := vars["type"] // type filter
// match registry types against the filter
rtypes := filterTypes(tFilter)
if rtypes == nil {
http.Error(w, "No objects matching '"+tFilter+"' found",
http.StatusNotFound)
return
}
// construct the response
response := make(map[string][]string)
for _, rtype := range rtypes {
objects := make([]string, 0, len(rtype.Objects))
for key := range rtype.Objects {
objects = append(objects, key)
}
response[rtype.Ref] = objects
}
ResponseJSON(w, response)
}
//////////////////////////////////////////////////////////////////////////
// object handler returns object data
// per object response structure
type RegObjectResponse struct {
Attributes [][2]string
Backlinks []string
}
func regObjectHandler(w http.ResponseWriter, r *http.Request) {
// request parameters
vars := mux.Vars(r)
query := r.URL.Query()
tFilter := vars["type"] // type filter
oFilter := vars["object"] // object filter
raw := query["raw"] // raw or decorated results
// select the type(s)
rtypes := filterTypes(tFilter)
if rtypes == nil {
http.Error(w, "No objects matching '"+tFilter+"' found",
http.StatusNotFound)
return
}
// then select the objects
objects := filterObjects(rtypes, oFilter)
if objects == nil {
http.Error(w, "No objects matching '"+tFilter+
"/"+oFilter+"' found", http.StatusNotFound)
return
}
// collate the results in to the response data
if raw == nil {
// provide a decorated response
response := make(map[string]RegObjectResponse)
// for each object in the results
for _, object := range objects {
// copy the raw attributes
attributes := make([][2]string, len(object.Data))
for ix, attribute := range object.Data {
attributes[ix] = [2]string{attribute.Key, attribute.Value}
}
// construct the backlinks
backlinks := make([]string, len(object.Backlinks))
for ix, object := range object.Backlinks {
backlinks[ix] = object.Ref
}
// add to the response
response[object.Ref] = RegObjectResponse{
Attributes: attributes,
Backlinks: backlinks,
}
}
ResponseJSON(w, response)
} else {
// provide a response with just the raw registry data
response := make(map[string][][2]string)
// for each object in the results
for _, object := range objects {
attributes := make([][2]string, len(object.Data))
response[object.Ref] = attributes
// copy the raw attributes
for ix, attribute := range object.Data {
attributes[ix] = [2]string{attribute.Key, attribute.RawValue}
}
}
ResponseJSON(w, response)
}
}
//////////////////////////////////////////////////////////////////////////
// key handler returns attribute data matching the key
func regKeyHandler(w http.ResponseWriter, r *http.Request) {
// request parameters
vars := mux.Vars(r)
query := r.URL.Query()
tFilter := vars["type"] // type filter
oFilter := vars["object"] // object filter
kFilter := vars["key"] // key filter
raw := query["raw"] // raw or decorated results
// select the type(s)
rtypes := filterTypes(tFilter)
if rtypes == nil {
http.Error(w, "No objects matching '"+tFilter+"' found",
http.StatusNotFound)
return
}
// select the key indices
ix := filterKeys(rtypes, kFilter)
if rtypes == nil {
http.Error(w, "No objects matching '"+tFilter+"/*/"+
kFilter+"' found", http.StatusNotFound)
return
}
// select the objects
objects := filterObjects(rtypes, oFilter)
if objects == nil {
http.Error(w, "No objects matching '"+tFilter+
"/"+oFilter+"' found", http.StatusNotFound)
return
}
// select objects that match the keys
amap := filterAttributes(ix, objects, "*", (raw != nil))
if len(amap) == 0 {
http.Error(w, "No attributes matching '"+tFilter+"/"+
oFilter+"/"+kFilter+"' found", http.StatusNotFound)
return
}
ResponseJSON(w, amap)
}
//////////////////////////////////////////////////////////////////////////
// attribute handler returns attribute data matching the attribute
func regAttributeHandler(w http.ResponseWriter, r *http.Request) {
// request parameters
vars := mux.Vars(r)
query := r.URL.Query()
tFilter := vars["type"] // type filter
oFilter := vars["object"] // object filter
kFilter := vars["key"] // key filter
aFilter := vars["attribute"] // attribute filter
raw := query["raw"] // raw or decorated results
// select the type(s)
rtypes := filterTypes(tFilter)
if rtypes == nil {
http.Error(w, "No objects matching '"+tFilter+"' found",
http.StatusNotFound)
return
}
// select the key indices
ix := filterKeys(rtypes, kFilter)
if rtypes == nil {
http.Error(w, "No objects matching '"+tFilter+"/*/"+
kFilter+"' found", http.StatusNotFound)
return
}
// then select the objects
objects := filterObjects(rtypes, oFilter)
if objects == nil {
http.Error(w, "No objects matching '"+tFilter+
"/"+oFilter+"' found", http.StatusNotFound)
return
}
// select objects that match the keys
amap := filterAttributes(ix, objects, aFilter, (raw != nil))
if len(amap) == 0 {
http.Error(w, "No attributes matching '"+tFilter+"/"+
oFilter+"/"+kFilter+"/"+aFilter+"' found", http.StatusNotFound)
return
}
ResponseJSON(w, amap)
}
//////////////////////////////////////////////////////////////////////////
// end of code

717
registry.go Normal file

@ -0,0 +1,717 @@
//////////////////////////////////////////////////////////////////////////
// DN42 Registry API Server
//////////////////////////////////////////////////////////////////////////
package main
//////////////////////////////////////////////////////////////////////////
import (
"bufio"
// "errors"
"fmt"
log "github.com/sirupsen/logrus"
"io/ioutil"
"os"
"os/exec"
"strings"
"time"
)
//////////////////////////////////////////////////////////////////////////
// registry data model
// registry data
// Attributes within Objects
type RegAttribute struct {
Key string
Value string // this is a post-processed, or decorated value
RawValue string // the raw value as read from the registry
}
type RegObject struct {
Ref string // the ref contains the full path for this object
Data []*RegAttribute // the key/value data for this object
Backlinks []*RegObject // other objects that reference this one
}
// types are collections of objects
type RegType struct {
Ref string // full path for this type
Objects map[string]*RegObject // the objects in this type
}
// registry meta data
type RegAttributeSchema struct {
Fields []string
Relations []*RegType
}
type RegKeyIndex struct {
Ref string
Objects map[*RegObject][]*RegAttribute
}
type RegTypeSchema struct {
Ref string
Attributes map[string]*RegAttributeSchema
KeyIndex map[string]*RegKeyIndex
}
// the registry itself
type Registry struct {
Commit string
Schema map[string]*RegTypeSchema
Types map[string]*RegType
}
// and a variable for the actual data
var RegistryData *Registry
// store the current commit has
var previousCommit string
//////////////////////////////////////////////////////////////////////////
// utility and manipulation functions
// general functions
func RegistryMakePath(t string, o string) string {
return t + "/" + o
}
func RegistrySplitPath(p string) (string, string) {
tmp := strings.Split(p, "/")
if len(tmp) != 2 {
return "", ""
}
return tmp[0], tmp[1]
}
func (registry *Registry) GetObject(path string) *RegObject {
rtname, objname := RegistrySplitPath(path)
rtype := registry.Types[rtname]
if rtype == nil {
return nil
}
return rtype.Objects[objname]
}
// attribute functions
// nothing here
// object functions
// return attributes exactly matching a specific key
func (object *RegObject) GetKey(key string) []*RegAttribute {
attributes := make([]*RegAttribute, 0)
for _, a := range object.Data {
if a.Key == key {
attributes = append(attributes, a)
}
}
return attributes
}
// return a single key
func (object *RegObject) GetSingleKey(key string) *RegAttribute {
attributes := object.GetKey(key)
if len(attributes) != 1 {
log.WithFields(log.Fields{
"key": key,
"object": object.Ref,
}).Error("Unable to find unique key in object")
// can't register the object
return nil
}
return attributes[0]
}
// schema functions
// validate a set of attributes against a schema
func (schema *RegTypeSchema) validate(attributes []*RegAttribute) []*RegAttribute {
validated := make([]*RegAttribute, 0, len(attributes))
for _, attribute := range attributes {
// keys beginning with 'x-' are user defined, skip validation
if !strings.HasPrefix(attribute.Key, "x-") {
if schema.Attributes[attribute.Key] == nil {
// couldn't find a schema attribute
log.WithFields(log.Fields{
"key": attribute.Key,
"schema": schema.Ref,
}).Error("Schema validation failed")
// don't add to the validated list
continue
}
}
// all ok
validated = append(validated, attribute)
}
return validated
}
// add an attribute to the key map
func (schema *RegTypeSchema) addKeyIndex(object *RegObject,
attribute *RegAttribute) {
keyix := schema.KeyIndex[attribute.Key]
// create a new object map if it didn't exist
if keyix == nil {
keyix = &RegKeyIndex{
Ref: attribute.Key,
Objects: make(map[*RegObject][]*RegAttribute),
}
schema.KeyIndex[attribute.Key] = keyix
}
// add the object/attribute reference
keyix.Objects[object] = append(keyix.Objects[object], attribute)
}
// object functions
// add a backlink to an object
func (object *RegObject) addBacklink(ref *RegObject) {
// check if the backlink already exists, this could be the case
// if an object is referenced multiple times (e.g. admin-c & tech-c)
for _, blink := range object.Backlinks {
if blink == ref {
// already exists, just return as nothing to do
return
}
}
// didn't find a match, add the backlink
object.Backlinks = append(object.Backlinks, ref)
}
//////////////////////////////////////////////////////////////////////////
// reload the registry
func reloadRegistry(path string, commit string) {
log.Debug("Reloading registry")
// r will become the new registry data
registry := &Registry{
Commit: commit,
Schema: make(map[string]*RegTypeSchema),
Types: make(map[string]*RegType),
}
// bootstrap the schema registry type
registry.Types["schema"] = &RegType{
Ref: "schema",
Objects: make(map[string]*RegObject),
}
registry.loadType("schema", path)
// and parse the schema to get the remaining types
registry.parseSchema()
// now load the remaining types
for _, rType := range registry.Types {
registry.loadType(rType.Ref, path)
}
// mark relationships
registry.decorate()
// trigger updates in any other modules
EventBus.Fire("RegistryUpdate", registry, path)
// swap in the new registry data
RegistryData = registry
}
//////////////////////////////////////////////////////////////////////////
// create and load the raw data for a registry type
func (registry *Registry) loadType(typeName string, path string) {
// the type will already have been created
rType := registry.Types[typeName]
// as will the schema (unless attempting to load the schema itself)
schema := registry.Schema[typeName]
// special case for DNS as the directory
// doesn't match the type name
if typeName == "domain" {
path += "/dns"
} else {
path += "/" + typeName
}
// and load all the objects in this type
rType.loadObjects(schema, path)
}
//////////////////////////////////////////////////////////////////////////
// load all the objects associated with a type
func (rType *RegType) loadObjects(schema *RegTypeSchema, path string) {
entries, err := ioutil.ReadDir(path)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"path": path,
"type": rType.Ref,
}).Error("Failed to read registry type directory")
return
}
// for each entry in the directory
for _, entry := range entries {
// each file maps to a registry object
if !entry.IsDir() {
filename := entry.Name()
// ignore dotfiles
if !strings.HasPrefix(filename, ".") {
// load the attributes from file
attributes := loadAttributes(path + "/" + filename)
// basic validation of attributes against the schema
// schema may be nil if we are actually loading the schema itself
if schema != nil {
attributes = schema.validate(attributes)
}
// make the object
object := &RegObject{
Ref: RegistryMakePath(rType.Ref, filename),
Data: attributes,
Backlinks: make([]*RegObject, 0),
}
// add to type
rType.Objects[filename] = object
}
}
}
log.WithFields(log.Fields{
"ref": rType.Ref,
"path": path,
"count": len(rType.Objects),
}).Debug("Loaded registry type")
}
//////////////////////////////////////////////////////////////////////////
// read attributes from a file
func loadAttributes(path string) []*RegAttribute {
attributes := make([]*RegAttribute, 0)
// open the file to start reading it
file, err := os.Open(path)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"path": path,
}).Error("Failed to read attributes from file")
return attributes
}
defer file.Close()
// read the file line by line using the bufio scanner
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimRight(scanner.Text(), "\r\n")
// lines starting with '+' denote an empty line
if line[0] == '+' {
// concatenate a \n on to the previous attribute value
attributes[len(attributes)-1].RawValue += "\n"
} else {
// look for a : separator in the first 20 characters
ix := strings.IndexByte(line, ':')
if ix == -1 || ix >= 20 {
// couldn't find one
if len(line) <= 20 {
// hmmm, the line was shorter than 20 characters
// something is amiss
log.WithFields(log.Fields{
"length": len(line),
"path": path,
"line": line,
}).Warn("Short line detected")
} else {
// line is a continuation of the previous line, so
// concatenate the value on to the previous attribute value
attributes[len(attributes)-1].RawValue +=
"\n" + string(line[20:])
}
} else {
// found a key and : separator
// is there actually a value ?
var value string
if len(line) <= 20 {
// blank value
value = ""
} else {
value = string(line[20:])
}
// create a new attribute
a := &RegAttribute{
Key: string(line[:ix]),
RawValue: value,
}
attributes = append(attributes, a)
}
}
}
return attributes
}
//////////////////////////////////////////////////////////////////////////
// parse schema files to extract keys and for attribute relations
func (registry *Registry) parseSchema() {
// for each object in the schema type
for _, object := range registry.Types["schema"].Objects {
// look up the ref attribute
ref := object.GetSingleKey("ref")
if ref == nil {
log.WithFields(log.Fields{
"object": object.Ref,
}).Error("Schema record without ref")
// can't process this object
continue
}
// create the type schema object
typeName := strings.TrimPrefix(ref.RawValue, "dn42.")
typeSchema := &RegTypeSchema{
Ref: typeName,
Attributes: make(map[string]*RegAttributeSchema),
KeyIndex: make(map[string]*RegKeyIndex),
}
// ensure the type exists
rType := registry.Types[typeName]
if rType == nil {
rType := &RegType{
Ref: typeName,
Objects: make(map[string]*RegObject),
}
registry.Types[typeName] = rType
}
// for each key attribute in the schema
attributes := object.GetKey("key")
for _, attribute := range attributes {
// split the value on whitespace
fields := strings.Fields(attribute.RawValue)
keyName := fields[0]
typeSchema.Attributes[keyName] = &RegAttributeSchema{
Fields: fields[1:],
}
}
// register the type schema
registry.Schema[typeName] = typeSchema
}
// scan the fields of each schema attribute to determine relationships
// this needs to be second step to allow pre-creation of the types
for _, typeSchema := range registry.Schema {
for attribName, attribSchema := range typeSchema.Attributes {
for _, field := range attribSchema.Fields {
if strings.HasPrefix(field, "lookup=") {
// the relationships may be a multivalue, separated by ,
rels := strings.Split(strings.
TrimPrefix(field, "lookup="), ",")
// map to a regtype
relations := make([]*RegType, 0, len(rels))
for ix := range rels {
relName := strings.TrimPrefix(rels[ix], "dn42.")
relation := registry.Types[relName]
// log if unable to look up the type
if relation == nil {
// log unless this is the schema def lookup=str '>' [spec]...
if typeSchema.Ref != "schema" {
log.WithFields(log.Fields{
"relation": relName,
"attribute": attribName,
"type": typeSchema.Ref,
}).Error("Relation to type that does not exist")
}
} else {
// store the relationship
relations = append(relations, relation)
}
}
// register the relations
attribSchema.Relations = relations
// assume only 1 lookup= per key
break
}
}
}
}
log.Debug("Schema parsing complete")
}
//////////////////////////////////////////////////////////////////////////
// parse all attributes and decorate them
func (registry *Registry) decorate() {
cattribs := 0
cmatched := 0
// walk each attribute value
for _, rType := range registry.Types {
schema := registry.Schema[rType.Ref]
for _, object := range rType.Objects {
for _, attribute := range object.Data {
cattribs += 1
// add this attribute to the key map
schema.addKeyIndex(object, attribute)
attribSchema := schema.Attributes[attribute.Key]
// are there relations defined for this attribute ?
// attribSchema may be null if this attribute is user defined (x-*)
if (attribSchema != nil) &&
attribute.matchRelation(object, attribSchema.Relations) {
// matched
cmatched += 1
} else {
// no match, just copy the attribute data
attribute.Value = attribute.RawValue
}
}
}
}
log.WithFields(log.Fields{
"attributes": cattribs,
"matched": cmatched,
}).Debug("Decoration complete")
}
//////////////////////////////////////////////////////////////////////////
// match an attribute against schema relations
func (attribute *RegAttribute) matchRelation(parent *RegObject,
relations []*RegType) bool {
// it's not going to match if relations is empty
if relations == nil {
return false
}
// check each relation
for _, relation := range relations {
object := relation.Objects[attribute.RawValue]
if object != nil {
// found a match !
// decorate the attribute value
attribute.Value = fmt.Sprintf("[%s](%s)",
attribute.RawValue, object.Ref)
// and add a back reference to the related object
object.addBacklink(parent)
return true
}
}
// didn't find anything
return false
}
//////////////////////////////////////////////////////////////////////////
// fetch the current commit hash
func getCommitHash(regDir string, gitPath string) string {
// run git to get the latest commit hash
cmd := exec.Command(gitPath, "log", "-1", "--format=%H")
cmd.Dir = regDir
// execute
out, err := cmd.Output()
if err != nil {
log.WithFields(log.Fields{
"error": err,
"gitPath": gitPath,
"regDir": regDir,
}).Error("Failed to execute git log")
}
return strings.TrimSpace(string(out))
}
//////////////////////////////////////////////////////////////////////////
// refresh the registry
func refreshRegistry(regDir string, gitPath string, branch string) {
// run git fetch to get the current commits from the master
cmd := exec.Command(gitPath, "fetch")
cmd.Dir = regDir
// execute
if out, err := cmd.Output(); err != nil {
log.WithFields(log.Fields{
"error": err,
"gitPath": gitPath,
"regDir": regDir,
}).Error("Failed to execute git fetch")
} else {
fmt.Printf("Git Fetch: %s", string(out))
}
// then reset hard to match the master
cmd = exec.Command(gitPath, "reset", "--hard", "origin/"+branch)
cmd.Dir = regDir
// execute
if out, err := cmd.Output(); err != nil {
log.WithFields(log.Fields{
"error": err,
"gitPath": gitPath,
"regDir": regDir,
"branch": branch,
}).Error("Failed to execute git reset")
} else {
fmt.Printf("Git Reset: %s", string(out))
}
}
//////////////////////////////////////////////////////////////////////////
// called from main to initialse the registry data and syncing
func InitialiseRegistryData(regDir string, refresh time.Duration,
gitPath string, autoPull bool, branch string) {
// validate that the regDir/data path exists
dataPath := regDir + "/data"
regStat, err := os.Stat(dataPath)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"path": dataPath,
}).Fatal("Unable to find registry directory")
}
// and it is a directory
if !regStat.IsDir() {
log.WithFields(log.Fields{
"error": err,
"path": dataPath,
}).Fatal("Registry path is not a directory")
}
// check that git exists
_, err = os.Stat(gitPath)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"path": gitPath,
}).Fatal("Unable to find git executable")
}
// enforce a minimum update time
minTime := 10 * time.Minute
if refresh < minTime {
log.WithFields(log.Fields{
"interval": refresh,
}).Error("Enforcing minimum update time of 10 minutes")
refresh = minTime
}
// initialise the previous commit hash
// and do initial load from registry
previousCommit = getCommitHash(regDir, gitPath)
reloadRegistry(dataPath, previousCommit)
go func() {
// every refresh interval
for range time.Tick(refresh) {
log.Debug("Refresh Timer")
// automatically try to refresh the registry ?
if autoPull {
refreshRegistry(regDir, gitPath, branch)
}
// get the latest hash
currentCommit := getCommitHash(regDir, gitPath)
// has the registry been updated ?
if currentCommit != previousCommit {
log.WithFields(log.Fields{
"current": currentCommit,
"previous": previousCommit,
}).Info("Registry has changed, refresh started")
// refresh
reloadRegistry(dataPath, currentCommit)
// update commit
previousCommit = currentCommit
}
}
}()
}
//////////////////////////////////////////////////////////////////////////
// end of code

471
roaapi.go Normal file

@ -0,0 +1,471 @@
//////////////////////////////////////////////////////////////////////////
// DN42 Registry API Server
//////////////////////////////////////////////////////////////////////////
package main
//////////////////////////////////////////////////////////////////////////
import (
"fmt"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
// "math/big"
"bufio"
"net"
"net/http"
"os"
"sort"
"strconv"
"strings"
"time"
)
//////////////////////////////////////////////////////////////////////////
// register the api
func init() {
EventBus.Listen("APIEndpoint", InitROAAPI)
EventBus.Listen("RegistryUpdate", ROAUpdate)
}
//////////////////////////////////////////////////////////////////////////
// data model
type PrefixROA struct {
Prefix string `json:"prefix"`
MaxLen uint8 `json:"maxLength"`
ASN string `json:"asn"`
}
type ROAFilter struct {
Number uint `json:"nr"`
Action string `json:"action"`
Prefix string `json:"prefix"`
MinLen uint8 `json:"minlen"`
MaxLen uint8 `json:"maxlen"`
Network *net.IPNet `json:"-"`
IPType uint8 `json:"-"`
}
type ROA struct {
CTime time.Time
Commit string
Filters []*ROAFilter
IPv4 []*PrefixROA
IPv6 []*PrefixROA
}
var ROAData *ROA
// set validity period for one week
// this might appear to be a long time, but is intended to provide
// enough time to prevent expiry of the data between real registry
// updates (which may only happen infrequently)
const ROA_JSON_VALIDITY_PERIOD = (7 * 24)
type ROAMetaData struct {
Counts uint `json:"counts"`
Generated uint32 `json:"generated"`
Valid uint32 `json:"valid"`
Signature string `json:"signature,omitempty"`
SignatureDate string `json:"signatureDate,omitempty"`
}
type ROAJSON struct {
MetaData ROAMetaData `json:"metadata"`
Roas []*PrefixROA `json:"roas"`
}
var ROAJSONResponse *ROAJSON
//////////////////////////////////////////////////////////////////////////
// called from main to initialise the API routing
func InitROAAPI(params ...interface{}) {
router := params[0].(*mux.Router)
s := router.
Methods("GET").
PathPrefix("/roa").
Subrouter()
s.HandleFunc("/filter/{ipv}", roaFilterHandler)
s.HandleFunc("/json", roaJSONHandler)
s.HandleFunc("/bird/{birdv}/{ipv}", roaBirdHandler)
log.Info("ROA API installed")
}
//////////////////////////////////////////////////////////////////////////
// api handlers
// return JSON formatted version of filter{,6}.txt
func roaFilterHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
ipv := vars["ipv"]
// pre-create an array to hold the result
filters := make([]*ROAFilter, 0, len(ROAData.Filters))
// helper closure to select from the filter array
fselect := func(a []*ROAFilter, t uint8) []*ROAFilter {
for _, f := range ROAData.Filters {
if f.IPType == t {
a = append(a, f)
}
}
return a
}
// add ipv4 filters if required
if strings.ContainsRune(ipv, '4') {
filters = fselect(filters, 4)
}
// add ipv6 filters if required
if strings.ContainsRune(ipv, '6') {
filters = fselect(filters, 6)
}
ResponseJSON(w, filters)
}
// return JSON formatted ROA data suitable for use with GoRTR
func roaJSONHandler(w http.ResponseWriter, r *http.Request) {
// check validity period of returned data
tnow := uint32(time.Now().Unix())
valid := ROAJSONResponse.MetaData.Valid
// check if validity period is close to expiry
if (tnow > valid) ||
((valid - tnow) < (ROA_JSON_VALIDITY_PERIOD / 4)) {
// if so extend the validity period
ROAJSONResponse.MetaData.Valid += (ROA_JSON_VALIDITY_PERIOD * 3600)
}
ResponseJSON(w, ROAJSONResponse)
}
// return the roa in bird format
func roaBirdHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
birdv := vars["birdv"]
ipv := vars["ipv"]
// bird 1 or bird 2 format
birdf := "roa %s max %d as %s;\n"
if birdv == "2" {
birdf = "route %s max %d as %s;\n"
}
var roa []*PrefixROA
if strings.ContainsRune(ipv, '4') {
roa = append(roa, ROAData.IPv4...)
}
if strings.ContainsRune(ipv, '6') {
roa = append(roa, ROAData.IPv6...)
}
w.Header().Set("Content-Type", "text/plain")
w.Header().Set("Access-Control-Allow-Origin", "*")
fmt.Fprintf(w, "#\n# dn42regsrv ROA Generator\n# Last Updated: %s\n"+
"# Commit: %s\n#\n", ROAData.CTime.String(), ROAData.Commit)
for _, r := range roa {
fmt.Fprintf(w, birdf, r.Prefix, r.MaxLen, r.ASN[2:])
}
}
//////////////////////////////////////////////////////////////////////////
// called whenever the registry is updated
func ROAUpdate(params ...interface{}) {
registry := params[0].(*Registry)
path := params[1].(string)
// initiate new ROA data
roa := &ROA{
CTime: time.Now(),
Commit: registry.Commit,
}
// load filter{,6}.txt files
if roa.loadFilter(path+"/filter.txt", 4) != nil {
// error loading IPv4 filter, don't update
return
}
if roa.loadFilter(path+"/filter6.txt", 6) != nil {
// error loading IPv6 filter, don't update
return
}
// compile ROA prefixes
roa.IPv4 = roa.CompileROA(registry, "route")
roa.IPv6 = roa.CompileROA(registry, "route6")
// swap in the new data
ROAData = roa
log.WithFields(log.Fields{
"ipv4": len(roa.IPv4),
"ipv6": len(roa.IPv6),
}).Debug("ROA data updated")
// pre-compute the JSON return struct
utime := uint32(roa.CTime.Unix())
response := &ROAJSON{
MetaData: ROAMetaData{
Generated: utime,
Valid: utime + (ROA_JSON_VALIDITY_PERIOD * 3600),
},
}
response.Roas = append(roa.IPv4, roa.IPv6...)
response.MetaData.Counts = uint(len(response.Roas))
ROAJSONResponse = response
}
//////////////////////////////////////////////////////////////////////////
// load network filter definitions from a filter file
func (roa *ROA) loadFilter(path string, iptype uint8) error {
// open the file for reading
file, err := os.Open(path)
if err != nil {
log.WithFields(log.Fields{
"path": path,
"error": err,
}).Error("Unable to open filter file")
return err
}
defer file.Close()
// helper closure to convert strings to numbers
var cerr error
convert := func(s string) int {
if cerr != nil {
return 0
}
val, cerr := strconv.Atoi(s)
if cerr != nil {
log.WithFields(log.Fields{
"number": s,
"error": err,
}).Error("Unable to parse number in filter file")
return 0
}
return val
}
filters := make([]*ROAFilter, 0)
// read the file line by line
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// remove any comments
if ix := strings.IndexRune(line, '#'); ix != -1 {
line = line[:ix]
}
fields := strings.Fields(line)
if len(fields) >= 5 {
// parse the prefix in to a NetIP structure
prefix := fields[2]
_, network, err := net.ParseCIDR(prefix)
if err != nil {
log.WithFields(log.Fields{
"path": path,
"prefix": prefix,
"error": err,
}).Error("Unable to parse CIDR in filter file")
} else {
// construct the filter object
roaf := &ROAFilter{
Number: uint(convert(fields[0])),
Action: fields[1],
Prefix: prefix,
MinLen: uint8(convert(fields[3])),
MaxLen: uint8(convert(fields[4])),
Network: network,
IPType: iptype,
}
// add to list if no strconv error
if cerr == nil {
filters = append(filters, roaf)
}
}
}
}
// did something go wrong ?
if err := scanner.Err(); err != nil {
log.WithFields(log.Fields{
"path": path,
"error": err,
}).Error("Scanner error reading filter file")
return err
}
// filter.txt should be in order,
// but still sort by number just in case
sort.Slice(filters, func(i, j int) bool {
return filters[i].Number < filters[j].Number
})
// add to the roa object
roa.Filters = append(roa.Filters, filters...)
return nil
}
//////////////////////////////////////////////////////////////////////////
// return the filter object that matches an IP address
func (roa *ROA) MatchFilter(ip net.IP) *ROAFilter {
for _, filter := range roa.Filters {
if filter.Network.Contains(ip) {
return filter
}
}
log.WithFields(log.Fields{
"IP": ip,
}).Error("Couldn't match address to filter !")
return nil
}
//////////////////////////////////////////////////////////////////////////
// compile ROA data
func (roa *ROA) CompileROA(registry *Registry,
tname string) []*PrefixROA {
// prepare indices to the route object keys
stype := registry.Schema[tname]
routeIX := stype.KeyIndex[tname]
originIX := stype.KeyIndex["origin"]
mlenIX := stype.KeyIndex["max-length"]
roalist := make([]*PrefixROA, 0, len(routeIX.Objects))
// for each object that has a route key
for object, rattribs := range routeIX.Objects {
if len(rattribs) > 1 {
log.WithFields(log.Fields{
"object": object.Ref,
}).Warn("Found object with multiple route attributes")
}
// extract the prefix
prefix := rattribs[0].RawValue
_, pnet, err := net.ParseCIDR(prefix)
if err != nil {
log.WithFields(log.Fields{
"object": object.Ref,
"prefix": prefix,
"error": err,
}).Error("Unable to parse CIDR in ROA")
continue
}
// match the prefix to the prefix filters
filter := roa.MatchFilter(pnet.IP)
if filter == nil {
continue
}
// don't allow routes that are denied in the filter rules
if filter.Action == "deny" {
log.WithFields(log.Fields{
"object": object.Ref,
"prefix": prefix,
"filter": filter.Prefix,
}).Warn("Denied ROA through filter rule")
continue
}
mlen := filter.MaxLen
// if the prefix is greater than the filter.MaxLen
// then don't emit an ROA route (making the route invalid)
if ones, _ := pnet.Mask.Size(); ones > int(mlen) {
log.WithFields(log.Fields{
"object": object.Ref,
"prefix": prefix,
"filter": filter.Prefix,
}).Debug("Defined ROA: Prefix > filter MaxLen")
continue
}
// calculate the max-length for this object
// check if the attribute has max-length defined
mattrib := mlenIX.Objects[object]
if mattrib != nil {
// use the local max-length value
tmp, err := strconv.ParseUint(mattrib[0].RawValue, 10, 8)
if err != nil {
log.WithFields(log.Fields{
"object": object.Ref,
"max-length": mattrib[0].RawValue,
"error": err,
}).Warn("Unable to convert max-length attribute")
} else {
// filter rules still have precedence over local values
if (uint8(tmp) < mlen) && (uint8(tmp) > filter.MinLen) {
mlen = uint8(tmp)
}
}
}
// look up the origin key for this object
oattribs := originIX.Objects[object]
if oattribs == nil {
log.WithFields(log.Fields{
"object": object.Ref,
}).Warn("Route Object without Origin")
} else {
// then for origin that can announce this prefix
for _, oattrib := range oattribs {
// add the ROA
roalist = append(roalist, &PrefixROA{
Prefix: prefix,
MaxLen: mlen,
ASN: oattrib.RawValue,
})
}
}
}
return roalist
}
//////////////////////////////////////////////////////////////////////////
// end of code

55
static.go Normal file

@ -0,0 +1,55 @@
//////////////////////////////////////////////////////////////////////////
// DN42 Registry API Server
//////////////////////////////////////////////////////////////////////////
package main
//////////////////////////////////////////////////////////////////////////
import (
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
"net/http"
"os"
)
//////////////////////////////////////////////////////////////////////////
// called from main to initialise the API routing
func InstallStaticRoutes(router *mux.Router, staticPath string) {
// an empty path disables static route serving
if staticPath == "" {
log.Info("Disabling static route serving")
return
}
// validate that the staticPath exists
stat, err := os.Stat(staticPath)
if err != nil {
log.WithFields(log.Fields{
"error": err,
"path": staticPath,
}).Fatal("Unable to find static page directory")
}
// and it is a directory
if !stat.IsDir() {
log.WithFields(log.Fields{
"error": err,
"path": staticPath,
}).Fatal("Static path is not a directory")
}
// install a file server for the static route
router.PathPrefix("/").Handler(http.StripPrefix("/",
http.FileServer(http.Dir(staticPath)))).Methods("GET")
log.WithFields(log.Fields{
"path": staticPath,
}).Info("Static route installed")
}
//////////////////////////////////////////////////////////////////////////
// end of code