From 9c38d45c333d3c6e2dbe4cde6a4da57b6434c155 Mon Sep 17 00:00:00 2001 From: Simon Marsh Date: Sat, 4 Apr 2020 22:41:39 +0100 Subject: [PATCH] Update maint log without info on recent outage --- pages/01.home/ipam/default.md | 7 +++--- pages/01.home/maintenance-log/default.md | 29 ++++++++++++++++++++++++ pages/02.peering/default.md | 11 --------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/pages/01.home/ipam/default.md b/pages/01.home/ipam/default.md index 86c0000..c1eaa8f 100755 --- a/pages/01.home/ipam/default.md +++ b/pages/01.home/ipam/default.md @@ -42,7 +42,7 @@ burble.dn42 IP addresses |*unassigned*|172.20.129.173|fd42:4242:2601:3b::1|| |dn42-ch-zur1.burble.dn42|172.20.129.174|fd42:4242:2601:28::1|| |dn42-uk-lon4.burble.dn42|172.20.129.175|fd42:4242:2601:29::1|Private Node| -|dn42-hk-hkg1.burble.dn42|172.20.129.176|fd42:4242:2601:26::1|| +|*unassigned*|172.20.129.176|fd42:4242:2601:26::1|| |dn42-sg-sin1.burble.dn42|172.20.129.177|fd42:4242:2601:25::1|| |dn42-uk-lon2.burble.dn42|172.20.129.178|fd42:4242:2601:24::1|Private Node| |dn42-fr-sbg1.burble.dn42|172.20.129.179|fd42:4242:2601:23::1|| @@ -51,7 +51,7 @@ burble.dn42 IP addresses |dn42-jp-tyo1.burble.dn42|172.20.129.182|fd42:4242:2601:3e::1|| |dn42-in-mum1.burble.dn42|172.20.129.183|fd42:4242:2601:3c::1|| |*unassigned*|*172.20.129.184*|*fd42:4242:2601:22::1*|| -|dn42-at-vie1.burble.dn42|172.20.129.185|fd42:4242:2601:39::1|| +|*unassigned*|172.20.129.185|fd42:4242:2601:39::1|| |dn42-fr-rbx2.burble.dn42|172.20.129.186|fd42:4242:2601:32::1|*Use fr-rbx1*| |dn42-uk-lon1.burble.dn42|172.20.129.187|fd42:4242:2601:35::1|| |dn42-fr-rbx1.burble.dn42|172.20.129.188|fd42:4242:2601:36::1|| @@ -60,7 +60,7 @@ burble.dn42 IP addresses | |172.20.129.191| |Reserved| |dn42-uk-lon3.burble.dn42| |fd42:4242:2601:27::1|Private Node| |dn42-uk-lon4.burble.dn42| |fd42:4242:2601:29::1|Private Node| -|dn42-nl-ams1.burble.dn42| |fd42:4242:2601:33::1|Private Node| +|*unassigned*| |fd42:4242:2601:33::1|Private Node| |dn42-in-mum2.burble.dn42| |fd42:4242:2601:22::1|Private Node| #### burble.dn42 Nodes (Public) @@ -72,7 +72,6 @@ burble.dn42 IP addresses |dn42-uk-lon1.burble.com|185.121.25.242|2a04:92c7:e:bd2::e6b9|| |dn42-de-fra1.burble.com|176.96.138.245|2a0d:5940:1:c3::b35c|| |dn42-lt-vil1.burble.com|195.181.241.93|2a02:7b40:c3b5:f15d::1|| -|dn42-at-vie1.burble.com|185.175.58.235|2a0d:5600:31:7::1|| |dn42-ch-zur1.burble.com|45.91.92.111|2a0e:dc0:6:8::1|| |dn42-ca-bhs2.burble.com|158.69.248.26|2607:5300:120:81a::1|| |dn42-us-nyc1.burble.com|185.213.26.143|2a0d:5600:33:b::1|| diff --git a/pages/01.home/maintenance-log/default.md b/pages/01.home/maintenance-log/default.md index 6915b89..99aac18 100755 --- a/pages/01.home/maintenance-log/default.md +++ b/pages/01.home/maintenance-log/default.md @@ -10,6 +10,35 @@ A log of changes to the burble.dn42 network. ## burble.dn42 Maintenance Log +#### 4th April 2020 + +Well, that was fun; burble.dn42 had a number of outages over this evening, caused by trying +to perform a rolling upgrade across the network. The biggest of these took out the burble.dn42 +DNS service for an extended period, impacting DNS resolution across DN42. + +The plan had been to perform a full upgrade and reboot for every burble.dn42 node. To minimise +disruption I perform updates across groups of servers that are chosen to be independent +so that service resilience should not be impacted. +However, this time there were two key failures: + +- The provider configuration for ca-bhs2 meant that it could not mount all of its disks when +rebooted and it ended up in maintenance mode. The server needed to be recovered via the +IPMI console. Whilst global services continued to be provided by other nodes, peers on +ca-bhs2 lost connectivity whilst the node was recovered. + +- The new pdns-recursor that was implemented at the end of March (see below) had a different +runtime path than the default OS install. This meant that when each of the core nodes was +restarted the pdns-recursor failed to restart as the runtime path was missing. Since the DNS +service is resilient, it continued to operate without problems until the last core node was +restarted, at which point the entire service failed. Without DNS, most of the remaining +burble.dn42 failed or could not be restarted and recovery was also hampered by having to +work without having DNS available. + +#### 1st April 2020 + +at-vie1 will be decommissioned by 14th April. +If you are peered on this node, please contact me to move the peering to another node. + #### 28th March 2020 The patched pdns recursor is now deployed to all core nodes. diff --git a/pages/02.peering/default.md b/pages/02.peering/default.md index 1364c8e..29e30fc 100755 --- a/pages/02.peering/default.md +++ b/pages/02.peering/default.md @@ -251,17 +251,6 @@ The burble.dn42 is fully meshed between nodes using wireguard tunnels. |**Tunnel IPv6 Link Local**|fe80::42:2601:3d:1/64| |**Tunnel IPv6 ULA**|fd42:4242:2601:3d::1/128| -####dn42-at-vie1 -||| -|---|---| -|**Location**|HostHatch, Vienna, Austria| -|**Public Hostname**|dn42-at-vie1.burble.com| -|**Public IPv4 Address**|185.175.58.235| -|**Public IPv6 Address**|2a0d:5600:31:7::1| -|**Tunnel IPv4 Peer Address**|172.20.129.185/32| -|**Tunnel IPv6 Link Local**|fe80::42:2601:39:1/64| -|**Tunnel IPv6 ULA**|fd42:4242:2601:39::1/128| - ####dn42-ch-zur1 ||| |---|---|