Commit c185a956 authored by Kevin Daudt's avatar Kevin Daudt 💻

merge: implement secdb in go

This implementation does not suffer from the issue that lua has, where it
cannot distinguish from an empty list and an empty array, without having to
resort to post processing to fix issues.

See merge request !4
parents 826dbc0b 2a6744d0
Pipeline #80371 passed with stages
in 9 minutes and 42 seconds
FROM alpinelinux/mqtt-exec
FROM alpinelinux/golang as build
ENV NQDIR=/tmp
COPY --chown=build:build src /home/build/src
WORKDIR /home/build/src
RUN go build -v secdb
RUN apk --no-cache add lua5.3 lua5.3-lyaml lua5.3-optarg lua5.3-cjson nq git
FROM alpinelinux/mqtt-exec
RUN apk --no-cache add nq git
COPY --from=build /home/build/src/secdb /usr/local/bin/secdb
COPY scripts /usr/local/bin
ENV NQDIR=/tmp
CMD [ "/usr/bin/nq", "-c", "generate_secdb.sh" ]
......@@ -11,25 +11,9 @@ services:
- htdocs:/var/www/html
nginx:
image: nginx:1-alpine
restart: always
volumes:
- htdocs:/var/www/html
- ./config/nginx.conf:/etc/nginx/conf.d/default.conf:ro
labels:
traefik.enable: "true"
traefik.http.routers.secdb.rule: Host(`secdb.alpinelinux.org`)
traefik.http.routers.secdb.entrypoints: http
traefik.http.routers.secdb.middlewares: redirect
traefik.http.routers.secdb-tls.rule: Host(`secdb.alpinelinux.org`)
traefik.http.routers.secdb-tls.entrypoints: https
traefik.http.routers.secdb-tls.tls: true
traefik.http.middlewares.redirect.redirectscheme.scheme: https
traefik.http.services.secdb.loadbalancer.server.port: 80
networks:
web:
volumes:
aports:
htdocs:
networks:
web:
external: true
version: '3.7'
services:
nginx:
restart: always
labels:
traefik.enable: "true"
traefik.http.routers.secdb.rule: Host(`secdb.alpinelinux.org`)
traefik.http.routers.secdb.entrypoints: https
traefik.http.routers.secdb-tls.tls: true
networks:
web:
networks:
web:
external: true
......@@ -6,7 +6,8 @@ REPOS="main community"
BRANCH=${1##*/}
case $BRANCH in
master|commits) exit 0 ;;
commits) exit 0 ;;
master) RELEASE=edge;;
*-stable) RELEASE=v${BRANCH/-stable/} ;;
*) echo "unknown branch: \"$BRANCH\"" ; exit 1 ;;
esac
......@@ -26,15 +27,17 @@ for REPO in $REPOS; do
fi
echo "Generating secdb for $RELEASE/$REPO"
TMPFILE=$(mktemp)
secfixes.lua \
secdb \
--repo "$REPO" --release "$RELEASE" \
"$HOME"/aports/"$REPO"/*/APKBUILD > "$TMPFILE"
if cmp -s "$TMPFILE" /var/www/html/"$RELEASE"/"$REPO".yaml; then
-outYaml "$TMPFILE".yaml \
-outJson "$TMPFILE".json \
"$HOME"/aports/"$REPO"/*/APKBUILD
if cmp -s "$TMPFILE".yaml /var/www/html/"$RELEASE"/"$REPO".yaml; then
echo "No changes found yml secfixes, skipping."
else
secfixes.lua --verify "$TMPFILE"
install -D "$TMPFILE" /var/www/html/"$RELEASE"/"$REPO".yaml
yaml2json.lua "$TMPFILE" > /var/www/html/"$RELEASE"/"$REPO".json
for ext in yaml json; do
install -D "$TMPFILE.$ext" /var/www/html/"$RELEASE/$REPO.$ext"
done
fi
rm -f "$TMPFILE"
rm -f "$TMPFILE.yaml" "$TMPFILE.json"
done
module secdb
go 1.16
require (
gitlab.alpinelinux.org/alpine/go v0.2.1
gopkg.in/yaml.v2 v2.4.0
)
github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
gitlab.alpinelinux.org/alpine/go v0.2.1 h1:seYzsntFNE/v0H4pd+r+kNnOICYFnIhjGlBjNbkLg84=
gitlab.alpinelinux.org/alpine/go v0.2.1/go.mod h1:auOw3SnxDQBo1vzPh8q6gjvKsYgKxYet03lgSKtA3Q4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
package main
import (
"encoding/json"
"flag"
"fmt"
"os"
"strings"
"gitlab.alpinelinux.org/alpine/go/pkg/apkbuild"
"gitlab.alpinelinux.org/alpine/go/pkg/releases"
"gopkg.in/yaml.v2"
)
var (
repo string
release string
debug bool
outJson string
outYaml string
)
func errOut(msg string, args ...interface{}) {
fmt.Fprintf(os.Stderr, msg+"\n", args...)
}
func main() {
flag.StringVar(&repo, "repo", "", "The repository to process (required)")
flag.StringVar(&release, "release", "", "The release that's being generated (required)")
flag.BoolVar(&debug, "debug", false, "Turn on debug output")
flag.StringVar(&outJson, "outJson", "", "File to write json output to")
flag.StringVar(&outYaml, "outYaml", "", "File to write yaml output to")
flag.Parse()
if release == "" {
errOut("-release argument is required")
flag.Usage()
os.Exit(1)
}
if repo == "" {
errOut("-repo argument is required")
flag.Usage()
os.Exit(1)
}
releases, err := releases.Fetch()
if err != nil {
panic(err)
}
releaseBranch := releases.GetRelBranch(release)
if releaseBranch == nil {
errOut("No release branch found called: %s\n", release)
os.Exit(1)
}
secfixDB := SecfixDB{}
secfixDB.Urlprefix = "https://dl-cdn.alpinelinux.org/alpine"
secfixDB.Apkurl = `{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-{{pkg.ver}}.apk`
secfixDB.Distroversion = release
secfixDB.Reponame = repo
secfixDB.Archs = releaseBranch.Arches
secfixDB.Packages = []Pkg{}
for _, pkg := range flag.Args() {
pkgComponents := strings.Split(pkg, "/")
pkgName := pkgComponents[len(pkgComponents)-2]
if debug {
fmt.Printf("Checking package %s\n", pkg)
}
file, err := os.Open(pkg)
if err != nil {
errOut("%s\n", err)
continue
}
secfixes, err := apkbuild.ParseSecfixes(file)
file.Close()
if err != nil {
errOut("%s\n", err)
continue
}
if secfixes == nil {
continue
}
secfixDB.Packages = append(secfixDB.Packages, Pkg{
Pkg: Package{
Name: pkgName,
Secfixes: secfixes,
}})
if debug {
secfixesJson, _ := json.MarshalIndent(secfixes, "", " ")
fmt.Println(string(secfixesJson))
}
}
if outYaml != "" {
secfixDBYaml, err := yaml.Marshal(secfixDB)
if err != nil {
panic(err)
}
err = writeToFile(outYaml, secfixDBYaml)
if err != nil {
panic(err)
}
}
if outJson != "" {
secfixDBJson, err := json.Marshal(secfixDB)
if err != nil {
panic(err)
}
err = writeToFile(outJson, secfixDBJson)
if err != nil {
panic(err)
}
}
}
func writeToFile(filename string, output []byte) error {
file, err := os.Create(filename)
if err != nil {
return err
}
defer file.Close()
_, err = file.Write(output)
if err != nil {
return err
}
return nil
}
package main
import (
"gitlab.alpinelinux.org/alpine/go/pkg/apkbuild"
)
type (
Package struct {
Name string `json:"name"`
Secfixes apkbuild.Secfixes `json:"secfixes"`
}
Pkg struct {
Pkg Package `json:"pkg"`
}
SecfixDB struct {
Apkurl string `json:"apkurl"`
Archs []string `json:"archs"`
Reponame string `json:"reponame"`
Urlprefix string `json:"urlprefix"`
Distroversion string `json:"distroversion"`
Packages []Pkg `json:"packages"`
}
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment