From f60a292129349964bda502c5a0540261f734bf9b Mon Sep 17 00:00:00 2001 From: Lan Tian Date: Sun, 9 Jan 2022 01:45:00 -0600 Subject: [PATCH] general: build docker images with correct arch label --- .circleci/config.yml | 202 ++++++++++++++++++++++++++++++----- Makefile | 8 +- Makefile.docker | 77 ------------- README.md | 60 ++--------- API.md => docs/API.md | 0 docs/Telegram.md | 22 ++++ frontend/Dockerfile | 11 ++ frontend/template.Dockerfile | 28 ----- proxy/Dockerfile | 27 +++++ proxy/template.Dockerfile | 64 ----------- 10 files changed, 242 insertions(+), 257 deletions(-) delete mode 100644 Makefile.docker rename API.md => docs/API.md (100%) create mode 100644 docs/Telegram.md create mode 100644 frontend/Dockerfile delete mode 100644 frontend/template.Dockerfile create mode 100644 proxy/Dockerfile delete mode 100644 proxy/template.Dockerfile diff --git a/.circleci/config.yml b/.circleci/config.yml index f55c28d..366feb3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,16 +4,35 @@ workflows: docker: jobs: - build - - deploy: + - docker-frontend: + context: + - docker + requires: + - build + filters: + branches: + ignore: master + - docker-frontend-deploy: + context: + - docker + requires: + - build + filters: + branches: + only: master + - docker-proxy: + context: + - docker + requires: + - build + filters: + branches: + ignore: master + - docker-proxy-deploy: context: - docker requires: - build - matrix: - parameters: - program: [frontend, proxy] - # latest is amd64 arch + push to default latest tag - image_arch: [latest, i386, arm32v7, arm64v8, ppc64le, s390x] filters: branches: only: master @@ -21,39 +40,166 @@ workflows: jobs: build: docker: - - image: circleci/golang:1.16 - working_directory: /go/src/github.com/xddxdd/bird-lg-go + - image: cimg/go:1.17 + working_directory: /home/circleci/go/src/github.com/xddxdd/bird-lg-go steps: - checkout - run: GO111MODULE=auto go get -v -t -d ./... - run: GO111MODULE=auto go test -v ./... - deploy: - docker: - - image: circleci/golang:1.16 - working_directory: /go/src/github.com/xddxdd/bird-lg-go - parameters: - image_arch: - type: string - program: - type: string + + docker-frontend: + machine: + image: ubuntu-2004:202111-02 + environment: + BUILDX_PLATFORMS: linux/amd64,linux/arm64,linux/386,linux/arm/v7 steps: - checkout - - setup_remote_docker: - version: 19.03.13 - run: - name: Install GPP + name: Install buildx command: | - sudo apt-get update && sudo apt-get install -y gpp + BUILDX_BINARY_URL="https://github.com/docker/buildx/releases/download/v0.7.1/buildx-v0.7.1.linux-amd64" + + curl --output docker-buildx \ + --silent --show-error --location --fail --retry 3 \ + "$BUILDX_BINARY_URL" + + mkdir -p ~/.docker/cli-plugins + + mv docker-buildx ~/.docker/cli-plugins/ + chmod a+x ~/.docker/cli-plugins/docker-buildx + + docker buildx install + # Run binfmt + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - run: name: Build Docker image environment: - IMAGE_ARCH: << parameters.image_arch >> - PROGRAM: << parameters.program >> BUILD_ID: << pipeline.number >> command: | - make -f Makefile.docker _crossbuild echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin - make -f Makefile.docker \ - DOCKER_USERNAME=$DOCKER_USERNAME \ - BUILD_ID=circleci-build$BUILD_ID \ - $PROGRAM/$IMAGE_ARCH + docker buildx create --name mybuilder --use + docker buildx build \ + --platform $BUILDX_PLATFORMS \ + -t $DOCKER_USERNAME/bird-lg-go:circleci-build$BUILD_ID \ + --progress plain \ + frontend + + docker-frontend-deploy: + machine: + image: ubuntu-2004:202111-02 + environment: + BUILDX_PLATFORMS: linux/amd64,linux/arm64,linux/386,linux/arm/v7 + steps: + - checkout + - run: + name: Install buildx + command: | + BUILDX_BINARY_URL="https://github.com/docker/buildx/releases/download/v0.7.1/buildx-v0.7.1.linux-amd64" + + curl --output docker-buildx \ + --silent --show-error --location --fail --retry 3 \ + "$BUILDX_BINARY_URL" + + mkdir -p ~/.docker/cli-plugins + + mv docker-buildx ~/.docker/cli-plugins/ + chmod a+x ~/.docker/cli-plugins/docker-buildx + + docker buildx install + # Run binfmt + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - run: + name: Build Docker image + environment: + BUILD_ID: << pipeline.number >> + command: | + echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin + docker buildx create --name mybuilder --use + docker buildx build \ + --platform $BUILDX_PLATFORMS \ + -t $DOCKER_USERNAME/bird-lg-go:circleci-build$BUILD_ID \ + --progress plain \ + --push frontend + docker buildx build \ + --platform $BUILDX_PLATFORMS \ + -t $DOCKER_USERNAME/bird-lg-go:latest \ + --progress plain \ + --push frontend + + docker-proxy: + machine: + image: ubuntu-2004:202111-02 + environment: + BUILDX_PLATFORMS: linux/amd64,linux/arm64,linux/386,linux/arm/v7 + steps: + - checkout + - run: + name: Install buildx + command: | + BUILDX_BINARY_URL="https://github.com/docker/buildx/releases/download/v0.7.1/buildx-v0.7.1.linux-amd64" + + curl --output docker-buildx \ + --silent --show-error --location --fail --retry 3 \ + "$BUILDX_BINARY_URL" + + mkdir -p ~/.docker/cli-plugins + + mv docker-buildx ~/.docker/cli-plugins/ + chmod a+x ~/.docker/cli-plugins/docker-buildx + + docker buildx install + # Run binfmt + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - run: + name: Build Docker image + environment: + BUILD_ID: << pipeline.number >> + command: | + echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin + docker buildx create --name mybuilder --use + docker buildx build \ + --platform $BUILDX_PLATFORMS \ + -t $DOCKER_USERNAME/bird-lgproxy-go:circleci-build$BUILD_ID \ + --progress plain \ + proxy + + docker-proxy-deploy: + machine: + image: ubuntu-2004:202111-02 + environment: + BUILDX_PLATFORMS: linux/amd64,linux/arm64,linux/386,linux/arm/v7 + steps: + - checkout + - run: + name: Install buildx + command: | + BUILDX_BINARY_URL="https://github.com/docker/buildx/releases/download/v0.7.1/buildx-v0.7.1.linux-amd64" + + curl --output docker-buildx \ + --silent --show-error --location --fail --retry 3 \ + "$BUILDX_BINARY_URL" + + mkdir -p ~/.docker/cli-plugins + + mv docker-buildx ~/.docker/cli-plugins/ + chmod a+x ~/.docker/cli-plugins/docker-buildx + + docker buildx install + # Run binfmt + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - run: + name: Build Docker image + environment: + BUILD_ID: << pipeline.number >> + command: | + echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin + docker buildx create --name mybuilder --use + docker buildx build \ + --platform $BUILDX_PLATFORMS \ + -t $DOCKER_USERNAME/bird-lgproxy-go:circleci-build$BUILD_ID \ + --push proxy + docker buildx build \ + --platform $BUILDX_PLATFORMS \ + -t $DOCKER_USERNAME/bird-lgproxy-go:latest \ + --progress plain \ + --push proxy diff --git a/Makefile b/Makefile index a165e7d..98d9448 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,6 @@ proxy: .PHONY: all frontend proxy all: frontend proxy -dockerfiles: - @$(MAKE) -f Makefile.docker dockerfiles - @sh -c "ls -1 */Dockerfile.*" - install: - install -m 755 frontend/frontend /usr/local/bin/frontend - install -m 755 proxy/proxy /usr/local/bin/proxy + install -m 755 frontend/frontend /usr/local/bin/bird-lg-go + install -m 755 proxy/proxy /usr/local/bin/bird-lgproxy-go diff --git a/Makefile.docker b/Makefile.docker deleted file mode 100644 index 62c587d..0000000 --- a/Makefile.docker +++ /dev/null @@ -1,77 +0,0 @@ -# Basic definitions -DOCKER_USERNAME := xddxdd -ARCHITECTURES := amd64 i386 arm32v7 arm64v8 ppc64le s390x -IMAGES := frontend proxy - -# General Purpose Preprocessor config -GPP_INCLUDE_DIR := include -GPP_FLAGS_U := "" "" "(" "," ")" "(" ")" "\#" "" -GPP_FLAGS_M := "\#" "\n" " " " " "\n" "(" ")" -GPP_FLAGS_EXTRA := +c "\\\n" "" -GPP_FLAGS := -I ${GPP_INCLUDE_DIR} --nostdinc -U ${GPP_FLAGS_U} -M ${GPP_FLAGS_M} ${GPP_FLAGS_EXTRA} - -BUILD_ID ?= $(shell date +%Y%m%d%H%M) - -define create-image-arch-target -frontend/Dockerfile.$1: frontend/template.Dockerfile - @gpp ${GPP_FLAGS} -D ARCH_$(shell echo $1 | tr a-z A-Z) -o frontend/Dockerfile.$1 frontend/template.Dockerfile || rm -rf frontend/Dockerfile.$1 - -frontend/$1: frontend/Dockerfile.$1 - @if [ -f frontend/Dockerfile.$1 ]; then \ - docker build --pull --no-cache -t ${DOCKER_USERNAME}/bird-lg-go:$1-${BUILD_ID} -f frontend/Dockerfile.$1 frontend || exit 1; \ - docker push ${DOCKER_USERNAME}/bird-lg-go:$1-${BUILD_ID} || exit 1; \ - docker tag ${DOCKER_USERNAME}/bird-lg-go:$1-${BUILD_ID} ${DOCKER_USERNAME}/bird-lg-go:$1 || exit 1; \ - docker push ${DOCKER_USERNAME}/bird-lg-go:$1 || exit 1; \ - else \ - echo "Dockerfile generation failed, see error above"; \ - exit 1; \ - fi - -proxy/Dockerfile.$1: proxy/template.Dockerfile - @gpp ${GPP_FLAGS} -D ARCH_$(shell echo $1 | tr a-z A-Z) -o proxy/Dockerfile.$1 proxy/template.Dockerfile || rm -rf proxy/Dockerfile.$1 - -proxy/$1: proxy/Dockerfile.$1 - @if [ -f proxy/Dockerfile.$1 ]; then \ - docker build --pull --no-cache -t ${DOCKER_USERNAME}/bird-lgproxy-go:$1-${BUILD_ID} -f proxy/Dockerfile.$1 proxy || exit 1; \ - docker push ${DOCKER_USERNAME}/bird-lgproxy-go:$1-${BUILD_ID} || exit 1; \ - docker tag ${DOCKER_USERNAME}/bird-lgproxy-go:$1-${BUILD_ID} ${DOCKER_USERNAME}/bird-lgproxy-go:$1 || exit 1; \ - docker push ${DOCKER_USERNAME}/bird-lgproxy-go:$1 || exit 1; \ - else \ - echo "Dockerfile generation failed, see error above"; \ - exit 1; \ - fi - -endef - -$(foreach arch,${ARCHITECTURES},$(eval $(call create-image-arch-target,$(arch)))) - -frontend:$(foreach arch,latest ${ARCHITECTURES},frontend/${arch}) - -frontend/latest: frontend/amd64 - @docker tag ${DOCKER_USERNAME}/bird-lg-go:amd64-${BUILD_ID} ${DOCKER_USERNAME}/bird-lg-go:${BUILD_ID} || exit 1 - @docker push ${DOCKER_USERNAME}/bird-lg-go:${BUILD_ID} || exit 1 - @docker tag ${DOCKER_USERNAME}/bird-lg-go:amd64-${BUILD_ID} ${DOCKER_USERNAME}/bird-lg-go:latest || exit 1 - @docker push ${DOCKER_USERNAME}/bird-lg-go:latest || exit 1 - -proxy:$(foreach arch,latest ${ARCHITECTURES},proxy/${arch}) - -proxy/latest: proxy/amd64 - @docker tag ${DOCKER_USERNAME}/bird-lgproxy-go:amd64-${BUILD_ID} ${DOCKER_USERNAME}/bird-lgproxy-go:${BUILD_ID} || exit 1 - @docker push ${DOCKER_USERNAME}/bird-lgproxy-go:${BUILD_ID} || exit 1 - @docker tag ${DOCKER_USERNAME}/bird-lgproxy-go:amd64-${BUILD_ID} ${DOCKER_USERNAME}/bird-lgproxy-go:latest || exit 1 - @docker push ${DOCKER_USERNAME}/bird-lgproxy-go:latest || exit 1 - -.DEFAULT_GOAL := images -.DELETE_ON_ERROR: -.SECONDARY: - -# Target to enable multiarch support -_crossbuild: - @docker run --rm --privileged multiarch/qemu-user-static --reset -p yes >/dev/null - -dockerfiles: $(foreach image,${IMAGES},$(foreach arch,${ARCHITECTURES},$(image)/Dockerfile.$(arch))) - -images: $(foreach image,${IMAGES},$(image)) - -clean: - @rm -rf */Dockerfile.{$(shell echo ${ARCHITECTURES} | sed "s/ /,/g")} diff --git a/README.md b/README.md index 28ad854..cf9317d 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ An alternative implementation for [bird-lg](https://github.com/sileht/bird-lg) w - [Bird-lg-go](#bird-lg-go) - [Table of Contents](#table-of-contents) - [Build Instructions](#build-instructions) - - [Build Docker Images](#build-docker-images) + - [Build Docker Images](#build-docker-images) - [Frontend](#frontend) - [Proxy](#proxy) - [Advanced Features](#advanced-features) @@ -17,8 +17,6 @@ An alternative implementation for [bird-lg](https://github.com/sileht/bird-lg) w - [IP addresses](#ip-addresses) - [API](#api) - [Telegram Bot Webhook](#telegram-bot-webhook) - - [Example of setting the webhook](#example-of-setting-the-webhook) - - [Supported commands](#supported-commands) - [Credits](#credits) - [License](#license) @@ -28,40 +26,11 @@ You need to have **Go 1.16 or newer** installed on your machine. Run `make` to build binaries for both the frontend and the proxy. -Optionally run `make install` to install them to `/usr/local/bin`. +Optionally run `make install` to install them to `/usr/local/bin` (`bird-lg-go` and `bird-lgproxy-go`). -Or, you can manually do the building steps: +### Build Docker Images -```bash -# Build frontend binary -cd frontend -go build -ldflags "-w -s" -o frontend -cd .. - -# Build proxy binary -cd proxy -go build -ldflags "-w -s" -o proxy -cd .. -``` - -## Build Docker Images - -Run `make dockerfile` and you'll get a bunch of Dockerfiles for different architectures: - -- `frontend/Dockerfile.amd64` -- `frontend/Dockerfile.arm32v7` -- `frontend/Dockerfile.arm64v8` -- `frontend/Dockerfile.i386` -- `frontend/Dockerfile.ppc64le` -- `frontend/Dockerfile.s390x` -- `proxy/Dockerfile.amd64` -- `proxy/Dockerfile.arm32v7` -- `proxy/Dockerfile.arm64v8` -- `proxy/Dockerfile.i386` -- `proxy/Dockerfile.ppc64le` -- `proxy/Dockerfile.s390x` - -`cd` into either frontend or proxy directory, rename the Dockerfile for your architecture to `Dockerfile` and run `docker build .` as usual. In most cases you'll want `Dockerfile.amd64`. +Use the Dockerfiles in `frontend` and `proxy` directory. ## Frontend @@ -194,30 +163,13 @@ These three servers are displayed as "Prod", "Test1" and "Test2" in the user int The frontend provides an API for running BIRD/traceroute/whois queries. -See [API docs](API.md) for detailed information. +See [API docs](docs/API.md) for detailed information. ### Telegram Bot Webhook The frontend can act as a Telegram Bot webhook endpoint, to add BGP route/traceroute/whois lookup functionality to your tech group. -There is no configuration necessary on the frontend, just start it up normally. - -Set your Telegram Bot webhook URL to `https://your.frontend.com/telegram/alpha+beta+gamma`, where `alpha+beta+gamma` is the list of servers to be queried on Telegram commands, separated by `+`. - -You may omit `alpha+beta+gamma` to use all your servers, but it is not recommended when you have lots of servers, or the message would be too long and hard to read. - -#### Example of setting the webhook - -```bash -curl "https://api.telegram.org/bot${BOT_TOKEN}/setWebhook?url=https://your.frontend.com:5000/telegram/alpha+beta+gamma" -``` - -#### Supported commands - -- `path`: Show bird's ASN path to target IP -- `route`: Show bird's preferred route to target IP -- `trace`: Traceroute to target IP/domain -- `whois`: Whois query +See [Telegram docs](docs/Telegram.md) for detailed information. ## Credits diff --git a/API.md b/docs/API.md similarity index 100% rename from API.md rename to docs/API.md diff --git a/docs/Telegram.md b/docs/Telegram.md new file mode 100644 index 0000000..c894d31 --- /dev/null +++ b/docs/Telegram.md @@ -0,0 +1,22 @@ +# Telegram Bot Webhook + +The frontend can act as a Telegram Bot webhook endpoint, to add BGP route/traceroute/whois lookup functionality to your tech group. + +There is no configuration necessary on the frontend, just start it up normally. + +Set your Telegram Bot webhook URL to `https://your.frontend.com/telegram/alpha+beta+gamma`, where `alpha+beta+gamma` is the list of servers to be queried on Telegram commands, separated by `+`. + +You may omit `alpha+beta+gamma` to use all your servers, but it is not recommended when you have lots of servers, or the message would be too long and hard to read. + +## Example of setting the webhook + +```bash +curl "https://api.telegram.org/bot${BOT_TOKEN}/setWebhook?url=https://your.frontend.com:5000/telegram/alpha+beta+gamma" +``` + +## Supported commands + +- `path`: Show bird's ASN path to target IP +- `route`: Show bird's preferred route to target IP +- `trace`: Traceroute to target IP/domain +- `whois`: Whois query diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 0000000..ab590c8 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,11 @@ +FROM golang:buster AS step_0 +ENV CGO_ENABLED=0 GO111MODULE=on +WORKDIR /root +COPY . . +RUN go build -ldflags "-w -s" -o /frontend + +################################################################################ + +FROM scratch AS step_1 +COPY --from=step_0 /frontend / +ENTRYPOINT ["/frontend"] diff --git a/frontend/template.Dockerfile b/frontend/template.Dockerfile deleted file mode 100644 index 127dd21..0000000 --- a/frontend/template.Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM golang:buster AS step_0 - -#if defined(ARCH_AMD64) -ENV GOOS=linux GOARCH=amd64 -#elif defined(ARCH_I386) -ENV GOOS=linux GOARCH=386 -#elif defined(ARCH_ARM32V7) -ENV GOOS=linux GOARCH=arm -#elif defined(ARCH_ARM64V8) -ENV GOOS=linux GOARCH=arm64 -#elif defined(ARCH_PPC64LE) -ENV GOOS=linux GOARCH=ppc64le -#elif defined(ARCH_S390X) -ENV GOOS=linux GOARCH=s390x -#else -#error "Architecture not set" -#endif - -ENV CGO_ENABLED=0 GO111MODULE=on -WORKDIR /root -COPY . . -RUN go build -ldflags "-w -s" -o /frontend - -################################################################################ - -FROM scratch AS step_1 -COPY --from=step_0 /frontend / -ENTRYPOINT ["/frontend"] diff --git a/proxy/Dockerfile b/proxy/Dockerfile new file mode 100644 index 0000000..4dcb39c --- /dev/null +++ b/proxy/Dockerfile @@ -0,0 +1,27 @@ +FROM golang:buster AS step_0 + +ENV CGO_ENABLED=0 GO111MODULE=on +WORKDIR /root +COPY . . +RUN go build -ldflags "-w -s" -o /proxy + +################################################################################ + +FROM alpine:edge AS step_1 + +WORKDIR /root +RUN apk add --no-cache build-base linux-headers +RUN wget https://sourceforge.net/projects/traceroute/files/traceroute/traceroute-2.1.0/traceroute-2.1.0.tar.gz/download \ + -O traceroute-2.1.0.tar.gz +RUN tar xvf traceroute-2.1.0.tar.gz \ + && cd traceroute-2.1.0 \ + && make -j4 LDFLAGS="-static" \ + && strip /root/traceroute-2.1.0/traceroute/traceroute + +################################################################################ + +FROM scratch AS step_2 +ENV PATH=/ +COPY --from=step_0 /proxy / +COPY --from=step_1 /root/traceroute-2.1.0/traceroute/traceroute / +ENTRYPOINT ["/proxy"] diff --git a/proxy/template.Dockerfile b/proxy/template.Dockerfile deleted file mode 100644 index 0b44bca..0000000 --- a/proxy/template.Dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -FROM golang:buster AS step_0 - -#if defined(ARCH_AMD64) -ENV GOOS=linux GOARCH=amd64 -#elif defined(ARCH_I386) -ENV GOOS=linux GOARCH=386 -#elif defined(ARCH_ARM32V7) -ENV GOOS=linux GOARCH=arm -#elif defined(ARCH_ARM64V8) -ENV GOOS=linux GOARCH=arm64 -#elif defined(ARCH_PPC64LE) -ENV GOOS=linux GOARCH=ppc64le -#elif defined(ARCH_S390X) -ENV GOOS=linux GOARCH=s390x -#else -#error "Architecture not set" -#endif - -ENV CGO_ENABLED=0 GO111MODULE=on -WORKDIR /root -COPY . . -RUN go build -ldflags "-w -s" -o /proxy - -################################################################################ - -#if defined(ARCH_AMD64) -FROM amd64/debian:sid AS step_1 -ENV TARGET_ARCH=x86_64 -#elif defined(ARCH_I386) -FROM i386/debian:sid AS step_1 -ENV TARGET_ARCH=i386 -#elif defined(ARCH_ARM32V7) -FROM arm32v7/debian:sid AS step_1 -ENV TARGET_ARCH=arm -#elif defined(ARCH_ARM64V8) -FROM arm64v8/debian:sid AS step_1 -ENV TARGET_ARCH=aarch64 -#elif defined(ARCH_PPC64LE) -FROM ppc64le/debian:sid AS step_1 -ENV TARGET_ARCH=ppc64le -#elif defined(ARCH_S390X) -FROM s390x/debian:sid AS step_1 -ENV TARGET_ARCH=s390 -#else -#error "Architecture not set" -#endif - -WORKDIR /root -RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y \ - build-essential musl-dev musl-tools tar wget git -RUN git clone https://github.com/sabotage-linux/kernel-headers.git -RUN wget https://sourceforge.net/projects/traceroute/files/traceroute/traceroute-2.1.0/traceroute-2.1.0.tar.gz/download \ - -O traceroute-2.1.0.tar.gz -RUN tar xvf traceroute-2.1.0.tar.gz \ - && cd traceroute-2.1.0 \ - && make -j4 CC=musl-gcc CFLAGS="-I/root/kernel-headers/${TARGET_ARCH}/include" LDFLAGS="-static" - -################################################################################ - -FROM scratch AS step_2 -ENV PATH=/ -COPY --from=step_0 /proxy / -COPY --from=step_1 /root/traceroute-2.1.0/traceroute/traceroute / -ENTRYPOINT ["/proxy"]