re-work loadsa stuff

2020-10-18 13:55:13 +01:00 · 2020-10-18 13:55:13 +01:00 · 8cde2ae854
commit 8cde2ae854
parent 3e0802c16b
4 changed files with 384 additions and 150 deletions
--- a/StaticRoot/grc.js
+++ b/StaticRoot/grc.js
@ -7,6 +7,7 @@ const ExplorerURL='https://explorer.burble.com/#'
 //////////////////////////////////////////////////////////////////////////
 // smaller display components

+// display an ASN
 Vue.component('reg-asn', {
    template: '#reg-asn',
    props: [ 'asn' ],
@ -27,6 +28,7 @@ Vue.component('reg-asn', {
    }
 })

+// display a path of ASNs
 Vue.component('reg-path', {
    template: '#reg-path',
    props: [ 'path' ],
@ -40,6 +42,7 @@ Vue.component('reg-path', {
    }
 })

+// display a inet{6,}num
 Vue.component('reg-prefix', {
    template: '#reg-prefix',
    props: [ 'prefix' ],
@ -63,7 +66,88 @@ Vue.component('reg-prefix', {
    }
 })

+// display mntner(s)
+Vue.component('reg-mnts', {
+    template: '#reg-mnts',
+    props: [ 'mnts' ],
+    data() {
+        return { }
+    }
+})

+// display mntner
+Vue.component('reg-mnt', {
+    template: '#reg-mnt',
+    props: [ 'mnt' ],
+    data() {
+        return { }
+    },
+    computed: {
+        url: function() {
+            return ExplorerURL + "/mntner/" + this.mnt
+        }
+    }
+})
+
+
+// display a set of AS Paths
+Vue.component('rt-paths', {
+    template: '#rt-paths',
+    props: [ 'paths' ],
+    data() {
+        return { }
+    },
+    methods: {
+        showPaths: function() {
+            const e = this.$createElement
+
+            // create a new array of reversed paths
+            var paths = [ ]
+            this.paths.forEach(path => {
+                paths.push(path.split(" ").reverse())
+            })
+            // and sort for easier comparing
+            paths.sort(
+                (a,b) => a.join(" ").localeCompare(b.join(" "))
+            )
+            
+            // now build the modal dialog
+
+            // for each path in the list
+            var pathList = [ e('p', { }, 'Paths are reverse sorted') ]
+            paths.forEach(path => {
+                
+                // for each ASN in the path
+                var asnList = [ ]
+                path.forEach(asn => {
+                    asnList.push(e('b-link', {
+                        class: 'px-1 my-0 py-0',
+                        props: {
+                            href: ExplorerURL + "/aut-num/" + asn
+                        }
+                    }, asn))
+                })
+
+                pathList.push(e('li', {
+                    class: 'my-0 py-0'
+                }, asnList))
+            })
+
+            var vnode = e('ul', { }, pathList)
+            
+            this.$bvModal.msgBoxOk([vnode], {
+                title: 'AS Paths',
+                size: 'xl',
+                centered: true
+            })
+        }
+    },
+    computed: {
+        count: function() {
+            return this.paths.length
+        }
+    }
+})

 //////////////////////////////////////////////////////////////////////////
 // flap list component
@ -73,52 +157,47 @@ Vue.component('app-flaps', {
    data() {
        return {
            filter: '',
-            currentPage: 1,
-            total: 0,
-            fields: [
-                {
-                    key: 'prefix',
-                    label: 'Prefix',
-                    sortable: true
-                },
-                {
-                    key: 'path',
-                    label: 'Path',
-                    sortable: true
-                },
-                {
-                    key: 'count',
-                    label: 'Updates',
-                    sortable: true,
-                    sortDirection: 'desc'
-                }
-            ],
-            flaps: [ ]
        }
    },
    computed: {
-        rows: function() {
-            return this.flaps.length;
-        }
-    },
-    methods: {
-        update: function(data) {
-            this.flaps.splice(0);
+
+        total: function() {
+            return this.$root.flapsTotal
+        },
+        
+        // converts the map to an array for iterating,
+        // applying any filtering that is required
+        list: function() {
            
-            this.total = data.total;
-            this.flaps = data.list.map((update) => {
-                var tmp = update.path.split(" ");
-                update.path = tmp.map((asn) => {
-                    return "AS" + asn;
-                }).join(" ")
-                return update;
+            var list = [ ]
+            var flaps = this.$root.flaps
+            var total = this.total
+            
+            Object.keys(flaps).forEach((prefix) => {
+
+                if (this.filter == '' || prefix.includes(this.filter)) {
+                    list.push({
+                        prefix: prefix,
+                        count: flaps[prefix].count,
+                        percent: Math.round(flaps[prefix].count*100 / total),
+                        paths: flaps[prefix].paths,
+                        mntner: flaps[prefix].mntner
+                    })
+                }
            })
+            
+            list.sort(
+                (a,b) => b.count - a.count
+            )
+
+            return list.slice(0, 10)
        }
    },
    mounted() {
        this.$root.$on('flaps-update', data => {
            this.update(data)
        })
+
    }
 })

@ -131,37 +210,53 @@ Vue.component('app-roa', {
    data() {
        return {
            filter4: '',
-            filter6: '',
-            roaFields: [
-                {
-                    key: 'prefix',
-                    label: 'Prefix',
-                    sortable: true
-                },
-                {
-                    key: 'origin',
-                    label: 'Origin',
-                    sortable: true
-                }
-            ],
-            roa4: [ ],
-            roa6: [ ]
+            filter6: ''
        }
    },
-    methods: {
-        update: function(data) {
-            this.roa4.splice(0);
-            this.roa6.splice(0);          
-            
-            data.list.forEach((roa) => {
-                var nroa = {
-                    origin: "AS" + roa.origin,
-                    prefix: roa.prefix
-                };
-                if (roa.prefix.includes(":")) { this.roa6.push(nroa) }
-                else { this.roa4.push(nroa) }
+    computed: {
+        list4: function() {
+            var list = [ ]
+
+            this.$root.roas.forEach((roa) => {
+                if (!roa.prefix.includes(':') &&
+                    (this.filter4 == '' ||
+                     roa.prefix.includes(this.filter4))) {
+                    list.push(roa)
+                }
            })
-        }
+
+            // sort by origin
+            list.sort((a,b) => {
+                var asna = a.origin.substr(2)
+                var asnb = b.origin.substr(2)
+                return asna - asnb
+            })
+            
+            return list
+        },
+
+        list6: function() {
+            var list = [ ]
+
+            this.$root.roas.forEach((roa) => {
+                if (roa.prefix.includes(':') &&
+                    (this.filter6 == '' ||
+                     roa.prefix.includes(this.filter6))) {
+                    list.push(roa)
+                }
+            })
+
+            // sort by origin
+            list.sort((a,b) => {
+                var asna = a.origin.substr(2)
+                var asnb = b.origin.substr(2)
+                return asna - asnb
+            })
+            
+            return list
+        }        
+        
+
    },
    mounted() {
        this.$root.$on('roa-update', data => {
@ -178,7 +273,8 @@ Vue.component('app-timer', {
    data() {
        return {
            seconds: 0,
-            timer: 0
+            timer: 0,
+            roaCounter: 0
        }
    },
    methods: {
@ -186,15 +282,65 @@ Vue.component('app-timer', {
            if (this.seconds <= 0) {
                this.seconds = 61;

+                // refresh flaps data
                axios.get('/api/flaps').then(response => {
-                    this.$root.$emit('flaps-update', response.data)
+                    
+                    // build a new map of the prefixes and total updates
+                    var flaps = { }
+                    var total = 0
+                    
+                    response.data.list.forEach((update) => {
+                        
+                        // initialise if no existing entry for this prefix
+                        if (!(update.prefix in flaps)) {
+                            flaps[update.prefix] = {
+                                count: 0,
+                                paths: [ ],
+                                mntner: this.$root.mntmap[update.prefix] || [ ]
+                            }
+                        }
+                        
+                        // canonicalise the ASN path
+                        var tmp = update.path.split(" ")
+                        var path = tmp.map((asn) => {
+                            return "AS" + asn
+                        }).join(" ")
+                        
+                        // finally update the prefix entry
+                        flaps[update.prefix].count += update.count
+                        flaps[update.prefix].paths.push(path)
+                        
+                        total += update.count
+                    })
+                    
+                    this.$root.flaps = flaps
+                    this.$root.flapsTotal = total
+                    
                });
-                
-                axios.get('/api/roa').then(response => {
-                    this.$root.$emit('roa-update', response.data)
-                });
-                
+
+                if (this.roaCounter == 0) {
+                    this.roaCounter = 59
+                    
+                    // refresh roa data
+                    axios.get('/api/roa').then(response => {
+                        var roas = [ ]
+                        response.data.list.forEach((roa) => {
+                            var asn = "AS" + roa.origin
+                            roas.push({
+                                origin: asn,
+                                prefix: roa.prefix,
+                                mntner: this.$root.mntmap[asn] || [ ]
+                            })
+                        })
+                        this.$root.roas = roas
+                    });
+                }
+                else {
+                    this.roaCounter -= 1
+                }
+                    
            }
+            // countdown every second
            this.seconds -= 1;
            this.timer = setTimeout(() => this.trigger(), 1000);
        }
@ -220,7 +366,46 @@ const router = new VueRouter({
 // and the main app instance
 const vm = new Vue({
    el: '#grc_realtime',
-    data: { },
+    data: {
+        flaps: { },
+        flapsTotal: 0,
+        roas: [ ],
+        mntmap: { }
+    },
+    mounted() {
+        axios.get('https://explorer.burble.com/api/registry/*inet/*/mnt-by?raw').then(response => {
+
+            // put the response in the mntmap hash
+            Object.keys(response.data).forEach(index => {
+                var slash = index.indexOf('/')
+                var prefix = index.substr(slash + 1)
+                var prefix = prefix.replace('_', '/')
+                this.mntmap[prefix] = response.data[index]['mnt-by']
+            })
+
+            // update the flaps list if required
+            Object.keys(this.flaps).forEach(prefix => {
+                this.flaps[prefix].mntner = this.mntmap[prefix]
+            })
+
+        })
+
+        axios.get('https://explorer.burble.com/api/registry/aut-num/*/mnt-by?raw').then(response => {
+
+            // put the response in the mntmap hash
+            Object.keys(response.data).forEach(index => {
+                var slash = index.indexOf('/')
+                var autnum = index.substr(slash + 1)
+                this.mntmap[autnum] = response.data[index]['mnt-by']
+            })
+
+            // and roas
+            this.roas.forEach(roa => {
+                roa.mntner = this.mntmap[roa.origin]
+            })
+            
+        })
+    },
    router
 })

--- a/StaticRoot/index.html
+++ b/StaticRoot/index.html
@ -76,47 +76,43 @@
 <script type="text/x-template" id="app-flaps-template">
 <b-container fluid="sm">
  <b-row class="justify-content-center py-3">
-    <h2>Most Updated Routes</h2>
+    <h2>Most updated prefixes</h2>
  </b-row>
  <b-row class="px-3">
    <b-col>
-      <b-table
-        id="flaps-table"
-        :fields="fields"
-        :items="flaps"
-        :filter="filter"
-        :sort-by="'count'"
-        :sort-desc="true"
-        :per-page="20"
-      >
-        <template v-slot:cell(prefix)="data">
-          <reg-prefix v-bind:prefix="data.value"></reg-prefix>
-        </template>   
-        <template v-slot:cell(path)="data">
-          <reg-path v-bind:path="data.value"></reg-path>
-        </template>
-      </b-table>
+      <b-table-simple responsive>
+        <b-thead>
+          <b-tr>
+            <b-th>Updates/min</b-th>
+            <b-th>Prefix</b-th>
+            <b-th>MNTNER</b-th>    
+            <b-th>Paths</b-th>
+          </b-tr>
+        </b-thead>
+        <b-tbody>
+          <b-tr v-for="item in this.list">
+            <b-td class="py-1 my-0" v-text="item.count + ' - ' + item.percent + '%'"></b-td>
+            <b-td class="py-1 my-0"><reg-prefix v-bind:prefix="item.prefix"></reg-prefix></b-td>
+            <b-td class="py-1 my-0"><reg-mnts v-bind:mnts="item.mntner"></reg-mnts></b-td>
+            <b-td class="py-1 my-0"><rt-paths v-bind:paths="item.paths"></rt-paths></b-td>    
+          </b-tr>
+        </b-tbody>
+      </b-table-simple>
    </b-col>
+    
    <b-col class="text-left pr-5">
-    <p>Total Updates: {{ this.total }}</p>
-    <p><b-pagination class="pb-4"
-      v-model="currentPage"
-      :total-rows="rows"
-      :per-page="20"
-      aria-controls="flaps-table"
-    ></b-pagination>
-    </p>
-    <p><b-input-group size="sm">
-      <b-form-input
-        v-model="filter"
-        type="search"
-        id="filterInput"
-        placeholder="Type to filter results"
-        debounce="200"
-      ></b-form-input>
-    </b-input-group></p>
-  </b-col>
-</b-row>
+      <p>Total Updates: {{ this.total }}<br/>Showing top ten results</p>
+      <p class="mt-5">Type to filter prefixes:<br/><b-input-group size="sm">
+        <b-form-input
+          v-model="filter"
+          type="search"
+          id="filterInput"
+          placeholder="filter"
+          debounce="200"
+        ></b-form-input>
+      </b-input-group></p>
+    </b-col>
+  </b-row>
 </b-container>
 </script>

@ -131,26 +127,28 @@
 </b-row>
 <b-row class="px-3">
  <b-col>
-    <b-table
-      :fields="roaFields"
-      :items="roa4"
-      :filter="filter4"
-    >
-      <template v-slot:cell(prefix)="data">
-        <reg-prefix v-bind:prefix="data.value"></reg-prefix>
-      </template>   
-      <template v-slot:cell(origin)="data">
-        <reg-asn v-bind:asn="data.value"></reg-asn>
-      </template>
-    </b-table>
+    <b-table-simple responsive>
+      <b-thead>
+        <b-th>Prefix</b-th>
+        <b-th>Origin</b-th>
+        <b-th>MNTNER</b-th>    
+      </b-thead>
+      <b-tbody>
+        <b-tr v-for="roa in this.list4">
+          <b-td class="py-1 my-0"><reg-prefix v-bind:prefix="roa.prefix"></reg-prefix></b-td>
+          <b-td class="py-1 my-0"><reg-asn v-bind:asn="roa.origin"></reg-asn></b-td>
+          <b-td class="py-1 my-0"><reg-mnts v-bind:mnts="roa.mntner"></reg-mnts></b-td>    
+        </b-tr>
+      </b-tbody>
+    </b-table-simple>
  </b-col>
  <b-col class="text-left pr-5">
-    <b-input-group size="sm">
+    <p>Type to filter prefixes</p><b-input-group size="sm">
      <b-form-input
        v-model="filter4"
        type="search"
        id="filterInput4"
-        placeholder="Type to filter results"
+        placeholder="filter"
        debounce="200"
      ></b-form-input>
    </b-input-group>
@ -164,26 +162,28 @@
 </b-row>
 <b-row class="px-3">
  <b-col>
-    <b-table
-      :fields="roaFields"
-      :items="roa6"
-      :filter="filter6"
-    >
-      <template v-slot:cell(prefix)="data">
-        <reg-prefix v-bind:prefix="data.value"></reg-prefix>
-      </template>   
-      <template v-slot:cell(origin)="data">
-        <reg-asn v-bind:asn="data.value"></reg-asn>
-      </template>
-    </b-table>
-  </b-col>
+    <b-table-simple responsive>
+      <b-thead>
+        <b-th>Prefix</b-th>
+        <b-th>Origin</b-th>
+        <b-th>MNTNER</b-th>    
+      </b-thead>
+      <b-tbody>
+        <b-tr v-for="roa in this.list6">
+          <b-td class="py-1 my-0"><reg-prefix v-bind:prefix="roa.prefix"></reg-prefix></b-td>
+          <b-td class="py-1 my-0"><reg-asn v-bind:asn="roa.origin"></reg-asn></b-td>
+          <b-td class="py-1 my-0"><reg-mnts v-bind:mnts="roa.mntner"></reg-mnts></b-td>    
+        </b-tr>
+      </b-tbody>
+    </b-table-simple>
+  </b-col>    
  <b-col class="text-left pr-5">
-    <b-input-group size="sm">
+    <p>Type to filter prefixes</p><b-input-group size="sm">
      <b-form-input
        v-model="filter6"
        type="search"
        id="filterInput6"
-        placeholder="Type to filter results"
+        placeholder="filter"
        debounce="200"
      ></b-form-input>
    </b-input-group>
@ -203,6 +203,18 @@
 <b-link v-bind:href="url" v-text="prefix"></b-link>
 </script>

+<script type="text/x-template" id="reg-mnts">
+<div>  
+  <p class="py-0 my-0" v-for="mnt in this.mnts">
+    <reg-mnt v-bind:mnt="mnt"></reg-mnt>
+  </p>
+</div>
+</script>
+
+<script type="text/x-template" id="reg-mnt">
+<b-link class="text-nowrap" v-bind:href="url" v-text="mnt"></b-link>
+</script>
+
 <script type="text/x-template" id="reg-path">
 <ul class="text-nowrap">
  <li class="d-inline p-1" v-for="asn in list">
@ -211,6 +223,10 @@
 </ul>
 </script>

+<script type="text/x-template" id="rt-paths">
+<b-button @click="this.showPaths" variant="dark" size="sm"><span v-text="this.count"></span>&nbsp;paths</b-button>
+</script>
+
 <script type="text/x-template" id="app-timer">
 <p class="p-1 pr-3">Update in {{ this.seconds }}s</p>
 </script>  
--- a/data.go
+++ b/data.go
@ -70,7 +70,8 @@ type SnapshotData struct {
 }

 type DataStruct struct {
-	shutdown context.CancelFunc
+	shutdown   context.CancelFunc
+	roaCounter uint

 	// sets for currently cellecting and previous data
 	active   *ActiveData
@ -119,25 +120,49 @@ func (data *DataStruct) swap(ctx context.Context) {

 		case <-ticker.C:
 			// tiemr expired
-			log.Debug("Data Update")
+			log.Debug("Route Data Update")

 			var upp *unsafe.Pointer
 			var up unsafe.Pointer
 			var oldp unsafe.Pointer

-			// swap in a new empty data set
-			upp = (*unsafe.Pointer)(unsafe.Pointer(&data.active))
-			up = unsafe.Pointer(data.newActiveData())
+			// swap in new route data
+			upp = (*unsafe.Pointer)(unsafe.Pointer(&data.active.route))
+			up = unsafe.Pointer(data.newRouteData())
 			oldp = atomic.SwapPointer(upp, up)

-			// generate the reporting snapshot
-			active := (*ActiveData)(oldp)
-			snapshot := active.snapshot()
+			// create a snapshot of the old data
+			routed := (*RouteData)(oldp)
+			snapshot := routed.snapshot()

 			// then swap in the new snapshot
-			upp = (*unsafe.Pointer)(unsafe.Pointer(&data.snapshot))
+			upp = (*unsafe.Pointer)(unsafe.Pointer(&data.snapshot.route))
 			up = unsafe.Pointer(snapshot)
 			atomic.SwapPointer(upp, up)
+
+			if data.roaCounter == 0 {
+				data.roaCounter = 59
+
+				log.Debug("ROA Data Update")
+
+				// swap in new roa data
+				upp = (*unsafe.Pointer)(unsafe.Pointer(&data.active.roa))
+				up = unsafe.Pointer(data.newROAData())
+				oldp = atomic.SwapPointer(upp, up)
+
+				// create a snapshot of the old data
+				road := (*ROAData)(oldp)
+				snapshot := road.snapshot()
+
+				// then swap in the new snapshot
+				upp = (*unsafe.Pointer)(unsafe.Pointer(&data.snapshot.roa))
+				up = unsafe.Pointer(snapshot)
+				atomic.SwapPointer(upp, up)
+
+			} else {
+				data.roaCounter -= 1
+			}
+
 		}
 	}
 }
@ -171,8 +196,8 @@ func (route *RouteData) add(path string) {
 	route.paths[path] += 1
 }

-func (roa *ROAData) add(path string) {
-	roa.roas[path] += 1
+func (roa *ROAData) add(prefix string) {
+	roa.roas[prefix] += 1
 }

 //////////////////////////////////////////////////////////////////////////
--- a/ingest.go
+++ b/ingest.go
@ -98,8 +98,9 @@ func (ingest *Ingest) tail(ctx context.Context, sockPath string) {

 				// trim the trailing newline
 				line = strings.TrimSuffix(line, "\n")
-
-				if len(line) > 3 {
+				ix := strings.Index(line, "<INFO> ")
+				if (ix != -1) && (len(line) > (ix + 10)) {
+					line = line[ix+7:]

 					log.WithFields(log.Fields{
 						"line": line,
@ -137,7 +138,14 @@ func (ingest *Ingest) route(data string) {
 		"data": data,
 	}).Debug("route update")

-	ingest.data.active.route.add(data)
+	space := strings.IndexByte(data, ' ')
+	if space != -1 {
+
+		prefix := data[:space]
+		path := data[space+7 : len(data)-1]
+
+		ingest.data.active.route.add(prefix + " " + path)
+	}
 }

 //////////////////////////////////////////////////////////////////////////