From bdef2bc8fc072248ce94319a8aedd063f93a77fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 3 Sep 2020 23:12:35 +0200 Subject: [PATCH] Allow specifying -o submit='fnmatch/*.pattern,another' to build alternate manifests Defaults to ".build.yml,.builds/*.yml" Ref: ~sircmpwn/git.sr.ht#316 --- gitsrht-update-hook/go.mod | 1 + gitsrht-update-hook/go.sum | 2 + gitsrht-update-hook/stage-3.go | 2 +- gitsrht-update-hook/submitter.go | 84 ++++++++++++++++++++++++-------- gitsrht-update-hook/webhooks.go | 2 +- 5 files changed, 70 insertions(+), 21 deletions(-) diff --git a/gitsrht-update-hook/go.mod b/gitsrht-update-hook/go.mod index f32de48..2f9a918 100644 --- a/gitsrht-update-hook/go.mod +++ b/gitsrht-update-hook/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( git.sr.ht/~sircmpwn/core-go v0.0.0-20200820135923-98806e712f5e + github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 github.com/fernet/fernet-go v0.0.0-20191111064656-eff2850e6001 github.com/go-git/go-git/v5 v5.1.0 github.com/go-redis/redis v6.15.9+incompatible diff --git a/gitsrht-update-hook/go.sum b/gitsrht-update-hook/go.sum index 275e184..ce79230 100644 --- a/gitsrht-update-hook/go.sum +++ b/gitsrht-update-hook/go.sum @@ -4,6 +4,8 @@ github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/g github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= diff --git a/gitsrht-update-hook/stage-3.go b/gitsrht-update-hook/stage-3.go index 9320029..46ef1d3 100644 --- a/gitsrht-update-hook/stage-3.go +++ b/gitsrht-update-hook/stage-3.go @@ -156,7 +156,7 @@ func deleteArtifacts(ctx *PushContext, db *sql.DB, payload *WebhookPayload) { logger.Fatalf("Scanning artifact rows: %e", err) } path := filepath.Join(s3prefix, "artifacts", - "~" + ctx.Repo.OwnerName, ctx.Repo.Name, filename) + "~"+ctx.Repo.OwnerName, ctx.Repo.Name, filename) logger.Printf("Deleting S3 object %s", path) err = minioClient.RemoveObject(context.TODO(), s3bucket, path, minio.RemoveObjectOptions{}) diff --git a/gitsrht-update-hook/submitter.go b/gitsrht-update-hook/submitter.go index 6eb085b..e750770 100644 --- a/gitsrht-update-hook/submitter.go +++ b/gitsrht-update-hook/submitter.go @@ -13,8 +13,11 @@ import ( "strings" "unicode/utf8" + "github.com/danwakefield/fnmatch" "github.com/fernet/fernet-go" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" "github.com/pkg/errors" ) @@ -85,32 +88,75 @@ type GitBuildSubmitter struct { } func (submitter GitBuildSubmitter) FindManifests() (map[string]string, error) { - tree, err := submitter.Repository.TreeObject(submitter.Commit.TreeHash) + rootTree, err := submitter.Repository.TreeObject(submitter.Commit.TreeHash) if err != nil { - return nil, errors.Wrap(err, "lookup tree failed") + return nil, errors.Wrap(err, "root tree lookup failed") } var files []*object.File - file, err := tree.File(".build.yml") - if err == nil { - files = append(files, file) - } else { - subtree, err := tree.Tree(".builds") - if err != nil { - return nil, nil - } - entries := subtree.Files() - for { - file, err = entries.Next() - if file == nil || err != nil { - break + loadOptions() + pattern := ".build.yml,.builds/*.yml" + if pat, ok := options["submit"]; ok { + pattern = pat + } + for _, pat := range strings.Split(pattern, ",") { + // If exact match: get to the blob directly + // Otherwise: find the longest prefix and start walking from there + + asterisk := strings.Index(pat, "*") + isWildcard := asterisk != -1 + if !isWildcard { + file, err := rootTree.File(pat) + if err != nil && err != object.ErrFileNotFound { + return nil, errors.Wrap(err, "getting file") } - if strings.HasSuffix(file.Name, ".yml") { + if file != nil { files = append(files, file) } - } - if err != io.EOF { - return nil, errors.Wrap(err, "EOF finding build manifest") + } else { + var tree *object.Tree + var prefix string + for strings.HasPrefix(pat, "/") { + pat = pat[1:] + } + if pref := strings.LastIndex(pat, "/"); pref != -1 { + tree, err = rootTree.Tree(pat[:pref]) + if err != nil && err != object.ErrDirectoryNotFound { + return nil, errors.Wrap(err, "getting pref tree") + } + prefix = pat[:pref+1] + pat = pat[pref+1:] + } else { + tree = rootTree + } + if tree == nil { + continue + } + + traversal := object.NewTreeWalker(tree, true, make(map[plumbing.Hash]bool)) + defer traversal.Close() + for { + name, entry, err := traversal.Next() + if err == io.EOF { + break + } else if err != nil { + return nil, errors.Wrap(err, "iterating worktree") + } + + if fnmatch.Match(pat, name, fnmatch.FNM_PATHNAME) { + if entry.Mode == filemode.Dir || entry.Mode == filemode.Submodule { + continue // Match iteration behaviour of subtree.Files() + } + + file, err := tree.TreeEntryFile(&entry) + if file == nil || err != nil { + return nil, errors.Wrap(err, "getting file for entry") + } + + file.Name = prefix + file.Name + files = append(files, file) + } + } } } diff --git a/gitsrht-update-hook/webhooks.go b/gitsrht-update-hook/webhooks.go index 40d2cd2..5a9a435 100644 --- a/gitsrht-update-hook/webhooks.go +++ b/gitsrht-update-hook/webhooks.go @@ -12,9 +12,9 @@ import ( "time" "unicode/utf8" + "git.sr.ht/~sircmpwn/core-go/crypto" "github.com/google/uuid" "github.com/mattn/go-runewidth" - "git.sr.ht/~sircmpwn/core-go/crypto" ) type WebhookSubscription struct { -- 2.38.4