From 84afd9d7b0298b6b418684daf7feab7f74635fa1 Mon Sep 17 00:00:00 2001 From: Drew DeVault Date: Tue, 19 Nov 2019 18:24:24 -0500 Subject: [PATCH] Rewrite gitsrht-update-hook in Go --- gitsrht-keys/main.go | 4 +- gitsrht-update-hook | 45 ----- gitsrht-update-hook/.gitignore | 1 + gitsrht-update-hook/go.mod | 16 ++ gitsrht-update-hook/go.sum | 77 ++++++++ gitsrht-update-hook/main.go | 67 +++++++ gitsrht-update-hook/manifest.go | 38 ++++ gitsrht-update-hook/post-update.go | 273 +++++++++++++++++++++++++++++ gitsrht-update-hook/stage-3.go | 83 +++++++++ gitsrht-update-hook/submitter.go | 265 ++++++++++++++++++++++++++++ gitsrht-update-hook/types.go | 106 +++++++++++ gitsrht-update-hook/update.go | 28 +++ gitsrht-update-hook/webhooks.go | 137 +++++++++++++++ gitsrht/blueprints/api.py | 1 + gitsrht/submit.py | 178 ------------------- scripts/symlink-update-hook.py | 9 +- setup.py | 1 - 17 files changed, 1101 insertions(+), 228 deletions(-) delete mode 100755 gitsrht-update-hook create mode 100644 gitsrht-update-hook/.gitignore create mode 100644 gitsrht-update-hook/go.mod create mode 100644 gitsrht-update-hook/go.sum create mode 100644 gitsrht-update-hook/main.go create mode 100644 gitsrht-update-hook/manifest.go create mode 100644 gitsrht-update-hook/post-update.go create mode 100644 gitsrht-update-hook/stage-3.go create mode 100644 gitsrht-update-hook/submitter.go create mode 100644 gitsrht-update-hook/types.go create mode 100644 gitsrht-update-hook/update.go create mode 100644 gitsrht-update-hook/webhooks.go delete mode 100644 gitsrht/submit.py diff --git a/gitsrht-keys/main.go b/gitsrht-keys/main.go index f3691a5..9f4cc6c 100644 --- a/gitsrht-keys/main.go +++ b/gitsrht-keys/main.go @@ -211,7 +211,7 @@ func main() { push := uuid.New() shellCommand := fmt.Sprintf("%s '%d' '%s' '%s'", shell, userId, username, b64key) - fmt.Printf(`restrict,command="%s",environment="SRHT_UID=%d",`+ + fmt.Printf(`restrict,command="%s",`+ `environment="SRHT_PUSH=%s" %s %s %s`+"\n", - shellCommand, userId, push.String(), keyType, b64key, username) + shellCommand, push.String(), keyType, b64key, username) } diff --git a/gitsrht-update-hook b/gitsrht-update-hook deleted file mode 100755 index 989f1a9..0000000 --- a/gitsrht-update-hook +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -import json -import os -import sys -from srht.config import cfg -from configparser import ConfigParser -from datetime import datetime, timedelta -from gitsrht.submit import do_post_update -from scmsrht.redis import redis - -op = sys.argv[0] -origin = cfg("git.sr.ht", "origin") - -if op == "hooks/update": - # Stash updated refs for later processing - refname = sys.argv[1] - old = sys.argv[2] - new = sys.argv[3] - - push_uuid = os.environ.get("SRHT_PUSH") - if not push_uuid: - sys.exit(0) - redis.setex(f"update.{push_uuid}.{refname}", - timedelta(minutes=10), f"{old}:{new}") - -if op == "hooks/post-update": - refs = sys.argv[1:] - - config = ConfigParser() - with open("config") as f: - config.read_file(f) - - context = json.loads(os.environ.get("SRHT_PUSH_CTX")) - repo = context["repo"] - - if repo["visibility"] == "autocreated": - print("\n\t\033[93mNOTICE\033[0m") - print("\tWe saved your changes, but this repository does not exist.") - print("\tClick here to create it:") - print() - print("\t{}/create?name={}".format(origin, repo["name"])) - print() - print("\tYour changes will be discarded in 20 minutes.\n") - - do_post_update(context, refs) diff --git a/gitsrht-update-hook/.gitignore b/gitsrht-update-hook/.gitignore new file mode 100644 index 0000000..381bb8e --- /dev/null +++ b/gitsrht-update-hook/.gitignore @@ -0,0 +1 @@ +gitsrht-update-hook diff --git a/gitsrht-update-hook/go.mod b/gitsrht-update-hook/go.mod new file mode 100644 index 0000000..5408742 --- /dev/null +++ b/gitsrht-update-hook/go.mod @@ -0,0 +1,16 @@ +module git.sr.ht/~sircmpwn/git.sr.ht/gitsrht-update-hook + +go 1.13 + +require ( + github.com/go-redis/redis v6.15.6+incompatible + github.com/google/uuid v1.1.1 + github.com/lib/pq v1.2.0 + github.com/mattn/go-runewidth v0.0.6 + github.com/microcosm-cc/bluemonday v1.0.2 + github.com/pkg/errors v0.8.1 + github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec + golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 + gopkg.in/src-d/go-git.v4 v4.13.1 + gopkg.in/yaml.v2 v2.2.7 +) diff --git a/gitsrht-update-hook/go.sum b/gitsrht-update-hook/go.sum new file mode 100644 index 0000000..d5bc92c --- /dev/null +++ b/gitsrht-update-hook/go.sum @@ -0,0 +1,77 @@ +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-redis/redis v6.15.6+incompatible h1:H9evprGPLI8+ci7fxQx6WNZHJSb7be8FqJQRhdQZ5Sg= +github.com/go-redis/redis v6.15.6+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-runewidth v0.0.6 h1:V2iyH+aX9C5fsYCpK60U8BYIvmhqxuOL3JZcqc1NB7k= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= +github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks= +github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw= +github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= +github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= +gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= +gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= +gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/gitsrht-update-hook/main.go b/gitsrht-update-hook/main.go new file mode 100644 index 0000000..87f63bd --- /dev/null +++ b/gitsrht-update-hook/main.go @@ -0,0 +1,67 @@ +package main + +import ( + "log" + "os" + + "github.com/vaughan0/go-ini" +) + +var ( + buildOrigin string + config ini.File + logger *log.Logger + origin string + pgcs string +) + +func main() { + log.SetFlags(0) + // The update hook is run on the update and post-update git hooks, and also + // runs a third stage directly. The first two stages are performance + // critical and take place while the user is blocked at their terminal. The + // third stage is done in the background. + if os.Args[0] == "hooks/update" { + update() + } else if os.Args[0] == "hooks/post-update" { + postUpdate() + } else if os.Args[0] == "stage-3" { + stage3() + } else { + log.Fatalf("Unknown git hook %s", os.Args[0]) + } +} + +func init() { + logf, err := os.OpenFile("/var/log/gitsrht-update-hook", + os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + log.Printf("Warning: unable to open log file: %v "+ + "(using stderr instead)", err) + logger = log.New(os.Stderr, os.Args[0]+" ", log.LstdFlags) + } else { + logger = log.New(logf, os.Args[0]+" ", log.LstdFlags) + } + + for _, path := range []string{"../config.ini", "/etc/sr.ht/config.ini"} { + config, err = ini.LoadFile(path) + if err == nil { + break + } + } + if err != nil { + logger.Fatalf("Failed to load config file: %v", err) + } + + var ok bool + origin, ok = config.Get("git.sr.ht", "origin") + if !ok { + logger.Fatalf("No origin configured for git.sr.ht") + } + pgcs, ok = config.Get("git.sr.ht", "connection-string") + if !ok { + logger.Fatalf("No connection string configured for git.sr.ht: %v", err) + } + + buildOrigin, _ = config.Get("builds.sr.ht", "origin") // Optional +} diff --git a/gitsrht-update-hook/manifest.go b/gitsrht-update-hook/manifest.go new file mode 100644 index 0000000..8ae4da6 --- /dev/null +++ b/gitsrht-update-hook/manifest.go @@ -0,0 +1,38 @@ +package main + +// TODO: Move this into builds.sr.ht + +import ( + "gopkg.in/yaml.v2" +) + +type Manifest struct { + Arch *string `yaml:"arch",omitempty` + Environment map[string]interface{} `yaml:"environment",omitempty` + Image string `yaml:"image"` + Packages []string `yaml:"packages",omitempty` + Repositories map[string]string `yaml:"repositories",omitempty` + Secrets []string `yaml:"secrets",omitempty` + Shell bool `yaml:"shell",omitempty` + Sources []string `yaml:"sources",omitempty` + Tasks []map[string]string `yaml:"tasks"` + Triggers []map[string]interface{} `yaml:"triggers",omitempty` +} + +func ManifestFromYAML(src string) (Manifest, error) { + var m Manifest + if err := yaml.Unmarshal([]byte(src), &m); err != nil { + return m, err + } + // XXX: We could do validation here, but builds.sr.ht will also catch it + // for us later so it's not especially important to + return m, nil +} + +func (manifest Manifest) ToYAML() (string, error) { + bytes, err := yaml.Marshal(&manifest) + if err != nil { + return "", err + } + return string(bytes), nil +} diff --git a/gitsrht-update-hook/post-update.go b/gitsrht-update-hook/post-update.go new file mode 100644 index 0000000..3bba5ac --- /dev/null +++ b/gitsrht-update-hook/post-update.go @@ -0,0 +1,273 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "log" + "os" + "os/exec" + "strings" + + goredis "github.com/go-redis/redis" + _ "github.com/lib/pq" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" +) + +func printAutocreateInfo(context PushContext) { + log.Println("\n\t\033[93mNOTICE\033[0m") + log.Println("\tWe saved your changes, but this repository does not exist.") + log.Println("\tClick here to create it:") + log.Println() + log.Printf("\t%s/create?name=%s", origin, context.Repo.Name) + log.Println() + log.Println("\tYour changes will be discarded in 20 minutes.") + log.Println() +} + +type DbInfo struct { + RepoId int + RepoName string + Visibility string + OwnerUsername string + OwnerToken string + AsyncWebhooks []WebhookSubscription + SyncWebhooks []WebhookSubscription +} + +func fetchInfoForPush(db *sql.DB, repoId int) (DbInfo, error) { + var dbinfo DbInfo = DbInfo{RepoId: repoId} + + // With this query, we: + // 1. Fetch the owner's username and OAuth token + // 2. Fetch the repository's name and visibility + // 3. Update the repository's mtime + // 4. Determine how many webhooks this repo has: if there are zero sync + // webhooks then we can defer looking them up until after we've sent the + // user on their way. + query, err := db.Prepare(` + UPDATE repository repo + SET updated = NOW() AT TIME ZONE 'UTC' + FROM ( + SELECT "user".username, "user".oauth_token + FROM "user" + JOIN repository r ON r.owner_id = "user".id + WHERE r.id = $1 + ) AS owner, ( + SELECT + COUNT(*) FILTER(WHERE rws.sync = true) sync_count, + COUNT(*) FILTER(WHERE rws.sync = false) async_count + FROM repo_webhook_subscription rws + WHERE rws.repo_id = $1 AND rws.events LIKE '%repo:post-update%' + ) AS webhooks + WHERE repo.id = $1 + RETURNING + repo.name, + repo.visibility, + owner.username, + owner.oauth_token, + webhooks.sync_count, + webhooks.async_count; + `) + if err != nil { + return dbinfo, err + } + defer query.Close() + + var nasync, nsync int + if err = query.QueryRow(repoId).Scan(&dbinfo.RepoName, &dbinfo.Visibility, + &dbinfo.OwnerUsername, &dbinfo.OwnerToken, + &nsync, &nasync); err != nil { + + return dbinfo, err + } + + dbinfo.AsyncWebhooks = make([]WebhookSubscription, nasync) + dbinfo.SyncWebhooks = make([]WebhookSubscription, nsync) + if nsync == 0 { + // Don't fetch webhooks, we don't need to waste the user's time + return dbinfo, nil + } + + var rows *sql.Rows + if rows, err = db.Query(` + SELECT id, url, events + FROM repo_webhook_subscription rws + WHERE rws.repo_id = $1 + AND rws.events LIKE '%repo:post-update%' + AND rws.sync = true + `, repoId); err != nil { + + return dbinfo, err + } + defer rows.Close() + + for i := 0; rows.Next(); i++ { + var whs WebhookSubscription + if err = rows.Scan(&whs.Id, &whs.Url, &whs.Events); err != nil { + return dbinfo, err + } + dbinfo.SyncWebhooks[i] = whs + } + + return dbinfo, nil +} + +func postUpdate() { + var context PushContext + refs := os.Args[1:] + + contextJson, ctxOk := os.LookupEnv("SRHT_PUSH_CTX") + pushUuid, pushOk := os.LookupEnv("SRHT_PUSH") + if !ctxOk || !pushOk { + logger.Fatal("Missing required variables in environment, " + + "configuration error?") + } + + logger.Printf("Running post-update for push %s", pushUuid) + + if err := json.Unmarshal([]byte(contextJson), &context); err != nil { + logger.Fatalf("unmarshal SRHT_PUSH_CTX: %v", err) + } + + if context.Repo.Visibility == "autocreated" { + printAutocreateInfo(context) + } + + payload := WebhookPayload{ + Push: pushUuid, + Pusher: context.User, + Refs: make([]UpdatedRef, len(refs)), + } + + oids := make(map[string]interface{}) + repo, err := git.PlainOpen(context.Repo.Path) + if err != nil { + logger.Fatalf("git.PlainOpen: %v", err) + } + + db, err := sql.Open("postgres", pgcs) + if err != nil { + logger.Fatalf("Failed to open a database connection: %v", err) + } + + dbinfo, err := fetchInfoForPush(db, context.Repo.Id) + if err != nil { + logger.Fatalf("Failed to fetch info from database: %v", err) + } + + redis := goredis.NewClient(&goredis.Options{Addr: "localhost:6379"}) + for i, refname := range refs { + var oldref, newref string + var oldobj, newobj object.Object + updateKey := fmt.Sprintf("update.%s.%s", pushUuid, refname) + update, err := redis.Get(updateKey).Result() + if update == "" || err != nil { + logger.Println("redis.Get: missing key") + continue + } else { + parts := strings.Split(update, ":") + oldref = parts[0] + newref = parts[1] + } + oldobj, err = repo.Object(plumbing.AnyObject, plumbing.NewHash(oldref)) + if err == plumbing.ErrObjectNotFound { + logger.Printf("old object %s not found", oldref) + continue + } + newobj, err = repo.Object(plumbing.AnyObject, plumbing.NewHash(newref)) + if err == plumbing.ErrObjectNotFound { + logger.Printf("new object %s not found", newref) + continue + } + + var atag *AnnotatedTag = nil + if tag, ok := newobj.(*object.Tag); ok { + atag = &AnnotatedTag{ + Name: tag.Name, + Message: tag.Message, + } + newobj, err = repo.CommitObject(tag.Target) + if err != nil { + logger.Println("unresolvable annotated tag") + continue + } + } + + oldcommit, ok := oldobj.(*object.Commit) + if !ok { + logger.Println("Skipping non-commit old ref") + continue + } + commit, ok := newobj.(*object.Commit) + if !ok { + logger.Println("Skipping non-commit new ref") + continue + } + + payload.Refs[i] = UpdatedRef{ + Tag: atag, + Name: refname, + Old: GitCommitToWebhookCommit(oldcommit), + New: GitCommitToWebhookCommit(commit), + } + + if _, ok := oids[commit.Hash.String()]; ok { + continue + } + oids[commit.Hash.String()] = nil + + if buildOrigin != "" { + submitter := GitBuildSubmitter{ + BuildOrigin: buildOrigin, + Commit: commit, + GitOrigin: origin, + OwnerName: dbinfo.OwnerUsername, + OwnerToken: dbinfo.OwnerToken, + RepoName: dbinfo.RepoName, + Repository: repo, + Visibility: dbinfo.Visibility, + } + results, err := SubmitBuild(submitter) + if err != nil { + log.Fatalf("Error submitting build job: %v", err) + } + if len(results) == 0 { + continue + } else if len(results) == 1 { + log.Println("\033[1mBuild started:\033[0m") + } else { + log.Println("\033[1mBuilds started:\033[0m") + } + logger.Printf("Submitted %d builds for %s", + len(results), refname) + for _, result := range results { + log.Printf("\033[94m%s\033[0m [%s]", result.Url, result.Name) + } + } + } + + payloadBytes, err := json.Marshal(&payload) + if err != nil { + logger.Fatalf("Failed to marshal webhook payload: %v", err) + } + + deliveries := deliverWebhooks(dbinfo.SyncWebhooks, payloadBytes) + deliveriesJson, err := json.Marshal(deliveries) + if err != nil { + logger.Fatalf("Failed to marshal webhook deliveries: %v", err) + } + + hook, ok := config.Get("git.sr.ht", "post-update-script") + if !ok { + logger.Fatal("No post-update script configured, cannot run stage 3") + } + + // Run stage 3 asyncronously - the last few tasks can be done without + // blocking the pusher's terminal. + stage3 := exec.Command(hook, string(deliveriesJson), string(payloadBytes)) + stage3.Args[0] = "stage-3" + stage3.Start() +} diff --git a/gitsrht-update-hook/stage-3.go b/gitsrht-update-hook/stage-3.go new file mode 100644 index 0000000..cbabd9d --- /dev/null +++ b/gitsrht-update-hook/stage-3.go @@ -0,0 +1,83 @@ +package main + +import ( + "database/sql" + "encoding/json" + "os" + + _ "github.com/lib/pq" +) + +func stage3() { + var context PushContext + contextJson, ctxOk := os.LookupEnv("SRHT_PUSH_CTX") + pushUuid, pushOk := os.LookupEnv("SRHT_PUSH") + if !ctxOk || !pushOk { + logger.Fatal("Missing required variables in environment, " + + "configuration error?") + } + + logger.Printf("Running stage 3 for push %s", pushUuid) + + if err := json.Unmarshal([]byte(contextJson), &context); err != nil { + logger.Fatalf("unmarshal SRHT_PUSH_CTX: %v", err) + } + + db, err := sql.Open("postgres", pgcs) + if err != nil { + logger.Fatalf("Failed to open a database connection: %v", err) + } + + var subscriptions []WebhookSubscription + var deliveries []WebhookDelivery + if err := json.Unmarshal([]byte(os.Args[1]), &deliveries); err != nil { + logger.Fatalf("Unable to unmarhsal delivery array: %v", err) + } + payload := []byte(os.Args[2]) + + var rows *sql.Rows + if rows, err = db.Query(` + SELECT id, url, events + FROM repo_webhook_subscription rws + WHERE rws.repo_id = $1 + AND rws.events LIKE '%repo:post-update%' + AND rws.sync = false`, context.Repo.Id); err != nil { + logger.Fatalf("Error fetching webhooks: %v", err) + } + defer rows.Close() + + for i := 0; rows.Next(); i++ { + var whs WebhookSubscription + if err = rows.Scan(&whs.Id, &whs.Url, &whs.Events); err != nil { + logger.Fatalf("Scanning webhook rows: %v", err) + } + subscriptions = append(subscriptions, whs) + } + + deliveries = append(deliveries, deliverWebhooks(subscriptions, payload)...) + for _, delivery := range deliveries { + if _, err := db.Exec(` + INSERT INTO repo_webhook_delivery ( + uuid, + created, + event, + url, + payload, + payload_headers, + response, + response_status, + response_headers, + subscription_id + ) VALUES ( + $1, NOW() AT TIME ZONE 'UTC', 'repo:post-update', + $2, $3, $4, $5, $6, $7 + ); + `, delivery.UUID, delivery.Url, + delivery.Payload, delivery.Headers, + delivery.Response, delivery.ResponseStatus, delivery.ResponseHeaders, + delivery.SubscriptionId); err != nil { + + logger.Fatalf("Error inserting webhook delivery: %v", err) + } + } +} diff --git a/gitsrht-update-hook/submitter.go b/gitsrht-update-hook/submitter.go new file mode 100644 index 0000000..5feb9fd --- /dev/null +++ b/gitsrht-update-hook/submitter.go @@ -0,0 +1,265 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "path" + "strings" + "unicode/utf8" + + "github.com/microcosm-cc/bluemonday" + "github.com/pkg/errors" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing/object" +) + +type BuildSubmitter interface { + // Return a list of build manifests and their names + FindManifests() (map[string]string, error) + // Get builds.sr.ht origin + GetBuildsOrigin() string + // Get builds.sr.ht OAuth token + GetOauthToken() string + // Get a checkout-able string to append to matching source URLs + GetCommitId() string + // Get the build note which corresponds to this commit + GetCommitNote() string + // Get the clone URL for this repository + GetCloneUrl() string + // Get the name of the repository + GetRepoName() string + // Get the name of the repository owner + GetOwnerName() string +} + +// SQL notes +// +// We need: +// - The repo ID +// - The repo name & visibility +// - The owner's username & canonical name +// - The owner's OAuth token & scopes +// - A list of affected webhooks +type GitBuildSubmitter struct { + BuildOrigin string + Commit *object.Commit + GitOrigin string + OwnerName string + OwnerToken string + RepoName string + Repository *git.Repository + Visibility string +} + +func (submitter GitBuildSubmitter) FindManifests() (map[string]string, error) { + tree, err := submitter.Repository.TreeObject(submitter.Commit.TreeHash) + if err != nil { + return nil, errors.Wrap(err, "lookup tree failed") + } + + var files []*object.File + file, err := tree.File(".build.yml") + if err == nil { + files = append(files, file) + } else { + subtree, err := tree.Tree(".builds") + if err != nil { + return nil, nil + } + entries := subtree.Files() + for { + file, err = entries.Next() + if file == nil || err != nil { + break; + } + if strings.HasSuffix(file.Name, ".yml") { + files = append(files, file) + } + } + if err != io.EOF { + return nil, errors.Wrap(err, "EOF finding build manifest") + } + } + + manifests := make(map[string]string) + for _, file := range files { + var ( + reader io.Reader + content []byte + ) + if reader, err = file.Reader(); err != nil { + return nil, errors.Wrapf(err, "creating reader for %s", file.Name) + } + if content, err = ioutil.ReadAll(reader); err != nil { + return nil, errors.Wrap(err, "reading build manifest") + } + if !utf8.Valid(content) { + return nil, errors.Wrap(err, "manifest is not valid UTF-8 file") + } + manifests[file.Name] = string(content) + } + return manifests, nil +} + +func (submitter GitBuildSubmitter) GetBuildsOrigin() string { + return submitter.BuildOrigin +} + +func (submitter GitBuildSubmitter) GetOauthToken() string { + return submitter.OwnerToken +} + +func (submitter GitBuildSubmitter) GetCommitId() string { + return submitter.Commit.Hash.String() +} + +func firstLine(text string) string { + buf := bytes.NewBufferString(text) + scanner := bufio.NewScanner(buf) + if !scanner.Scan() { + return "" + } + return scanner.Text() +} + +func (submitter GitBuildSubmitter) GetCommitNote() string { + policy := bluemonday.StrictPolicy() + commitUrl := fmt.Sprintf("%s/%s/%s/commit/%s", submitter.GitOrigin, + submitter.OwnerName, submitter.RepoName, + submitter.GetCommitId()) + return fmt.Sprintf(`[%s](%s) — [%s](mailto:%s)\n\n
%s
`, + submitter.GetCommitId()[:7], commitUrl, + submitter.Commit.Author.Name, submitter.Commit.Author.Email, + policy.Sanitize(firstLine(submitter.Commit.Message))) +} + +func (submitter GitBuildSubmitter) GetCloneUrl() string { + if submitter.Visibility == "private" { + origin := strings.ReplaceAll(submitter.GitOrigin, "http://", "") + origin = strings.ReplaceAll(origin, "https://", "") + // Use SSH URL + return fmt.Sprintf("git+ssh://git@%s/~%s/%s", origin, + submitter.OwnerName, submitter.RepoName) + } else { + // Use HTTP(s) URL + return fmt.Sprintf("%s/~%s/%s", submitter.GitOrigin, + submitter.OwnerName, submitter.RepoName) + } +} + +func (submitter GitBuildSubmitter) GetRepoName() string { + return submitter.RepoName +} + +func (submitter GitBuildSubmitter) GetOwnerName() string { + return submitter.OwnerName +} + +type BuildSubmission struct { + // TODO: Move errors into this struct and set up per-submission error + // tracking + Name string + Url string +} + +// TODO: Move this to scm.sr.ht +func SubmitBuild(submitter BuildSubmitter) ([]BuildSubmission, error) { + manifests, err := submitter.FindManifests() + if err != nil || manifests == nil { + return nil, err + } + + var results []BuildSubmission + for name, contents := range manifests { + manifest, err := ManifestFromYAML(contents) + if err != nil { + return nil, errors.Wrap(err, name) + } + autoSetupManifest(submitter, &manifest) + + yaml, err := manifest.ToYAML() + if err != nil { + return nil, errors.Wrap(err, name) + } + + client := &http.Client{} + + submission := struct { + Manifest string `json:"manifest"` + Tags []string `json:"tags"` + }{ + Manifest: yaml, + Tags: []string{submitter.GetRepoName(), name}, + } + bodyBytes, err := json.Marshal(&submission) + if err != nil { + return nil, errors.Wrap(err, "preparing job") + } + body := bytes.NewBuffer(bodyBytes) + + req, err := http.NewRequest("POST", fmt.Sprintf("%s/api/jobs", + submitter.GetBuildsOrigin()), body) + req.Header.Add("Authorization", fmt.Sprintf("token %s", + submitter.GetOauthToken())) + req.Header.Add("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + return nil, errors.Wrap(err, "job submission") + } + + if resp.StatusCode == 403 { + return nil, errors.New("builds.sr.ht returned 403\n" + + "Log out and back into the website to authorize " + + "builds integration.") + } + + defer resp.Body.Close() + respBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "read response") + } + + if resp.StatusCode == 400 { + return nil, errors.New(fmt.Sprintf( + "builds.sr.ht returned %d\n", resp.StatusCode) + + string(respBytes)) + } + + var job struct { + Id int `json:"id"` + } + err = json.Unmarshal(respBytes, &job) + if err != nil { + return nil, errors.Wrap(err, "interpret response") + } + + results = append(results, BuildSubmission{ + Name: name, + Url: fmt.Sprintf("%s/~%s/job/%d", + submitter.GetBuildsOrigin(), + submitter.GetOwnerName(), + job.Id), + }) + } + + return results, nil +} + +func autoSetupManifest(submitter BuildSubmitter, manifest *Manifest) { + var hasSelf bool + cloneUrl := submitter.GetCloneUrl() + "#" + submitter.GetCommitId() + for i, src := range manifest.Sources { + if path.Base(src) == submitter.GetRepoName() { + manifest.Sources[i] = cloneUrl + hasSelf = true + } + } + if !hasSelf { + manifest.Sources = append(manifest.Sources, cloneUrl) + } +} diff --git a/gitsrht-update-hook/types.go b/gitsrht-update-hook/types.go new file mode 100644 index 0000000..8bf0616 --- /dev/null +++ b/gitsrht-update-hook/types.go @@ -0,0 +1,106 @@ +package main + +import ( + "encoding/base64" + "io/ioutil" + + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" +) + +type RepoContext struct { + Id int `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + Visibility string `json:"visibility"` +} + +type UserContext struct { + CanonicalName string `json:"canonical_name"` + Name string `json:"name"` +} + +type PushContext struct { + Repo RepoContext `json:"repo"` + User UserContext `json:"user"` +} + +type AnnotatedTag struct { + Name string `json:"name"` + Message string `json:"message"` +} + +type CommitSignature struct { + Data string `json:"data"` + Signature string `json:"signature"` +} + +type CommitAuthorship struct { + Email string `json:"email"` + Name string `json:"name"` +} + +// See gitsrht/blueprints/api.py +type Commit struct { + Id string `json:"id"` + Message string `json:"message"` + Parents []string `json:"parents"` + ShortId string `json:"short_id"` + Timestamp string `json:"timestamp"` + Tree string `json:"tree"` + + Author CommitAuthorship `json:"author"` + Committer CommitAuthorship `json:"committer"` + Signature *CommitSignature `json:"signature"` +} + +type UpdatedRef struct { + Tag *AnnotatedTag `json:"annotated_tag",omitempty` + Name string `json:"name"` + Old *Commit `json:"old"` + New *Commit `json:"commit"` +} + +type WebhookPayload struct { + Push string `json:"push"` + Pusher UserContext `json:"pusher"` + Refs []UpdatedRef `json:"refs"` +} + +func GitCommitToWebhookCommit(c *object.Commit) *Commit { + parents := make([]string, len(c.ParentHashes)) + for i, p := range c.ParentHashes { + parents[i] = p.String() + } + + var signature *CommitSignature = nil + if c.PGPSignature != "" { + encoded := &plumbing.MemoryObject{} + c.EncodeWithoutSignature(encoded) + reader, _ := encoded.Reader() + data, _ := ioutil.ReadAll(ioutil.NopCloser(reader)) + signature = &CommitSignature{ + Data: base64.StdEncoding.EncodeToString(data), + Signature: base64.StdEncoding.EncodeToString([]byte(c.PGPSignature)), + } + } + + return &Commit{ + Id: c.Hash.String(), + Message: c.Message, + Parents: parents, + ShortId: c.Hash.String()[:7], + Timestamp: c.Author.When.Format("2006-01-02T15:04:05-07:00"), + Tree: c.TreeHash.String(), + Author: CommitAuthorship{ + // TODO: Add timestamp + Name: c.Author.Name, + Email: c.Author.Email, + }, + Committer: CommitAuthorship{ + Name: c.Committer.Name, + Email: c.Committer.Email, + }, + Signature: signature, + } +} diff --git a/gitsrht-update-hook/update.go b/gitsrht-update-hook/update.go new file mode 100644 index 0000000..7972a59 --- /dev/null +++ b/gitsrht-update-hook/update.go @@ -0,0 +1,28 @@ +package main + +import ( + "fmt" + "os" + "time" + + goredis "github.com/go-redis/redis" +) + +// XXX: This is run once for every single ref that's pushed. If someone pushes +// lots of refs, it might be expensive. Needs to be tested. +func update() { + var ( + refname string = os.Args[1] + oldref string = os.Args[2] + newref string = os.Args[3] + ) + pushUuid, ok := os.LookupEnv("SRHT_PUSH") + if !ok { + logger.Fatal("Missing SRHT_PUSH in environment, configuration error?") + } + logger.Printf("Running update for push %s", pushUuid) + + redis := goredis.NewClient(&goredis.Options{Addr: "localhost:6379"}) + redis.Set(fmt.Sprintf("update.%s.%s", pushUuid, refname), + fmt.Sprintf("%s:%s", oldref, newref), 10*time.Minute) +} diff --git a/gitsrht-update-hook/webhooks.go b/gitsrht-update-hook/webhooks.go new file mode 100644 index 0000000..ea013af --- /dev/null +++ b/gitsrht-update-hook/webhooks.go @@ -0,0 +1,137 @@ +package main + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "encoding/hex" + "fmt" + "io/ioutil" + "log" + "net/http" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" + "github.com/mattn/go-runewidth" + "golang.org/x/crypto/ed25519" +) + +var ( + privkey ed25519.PrivateKey +) + +type WebhookSubscription struct { + Id int + Url string + Events string +} + +// Note: unlike normal sr.ht services, we don't add webhook deliveries to the +// database until after the HTTP request has been completed, to reduce time +// spent blocking the user's terminal. +type WebhookDelivery struct { + Headers string + Payload string + Response string + ResponseHeaders string + ResponseStatus int + SubscriptionId int + UUID string + Url string +} + +func initWebhookKey() { + b64key, ok := config.Get("webhooks", "private-key") + if !ok { + logger.Fatalf("No webhook key configured") + } + seed, err := base64.StdEncoding.DecodeString(b64key) + if err != nil { + logger.Fatalf("base64 decode webhooks private key: %v", err) + } + privkey = ed25519.NewKeyFromSeed(seed) +} + +func deliverWebhooks(subs []WebhookSubscription, + payload []byte) []WebhookDelivery { + + var deliveries []WebhookDelivery + initWebhookKey() + client := &http.Client{Timeout: 5 * time.Second} + + for _, sub := range subs { + var ( + nonceSeed []byte + nonceHex []byte + ) + _, err := rand.Read(nonceSeed) + if err != nil { + logger.Fatalf("generate nonce: %v", err) + } + hex.Encode(nonceHex, nonceSeed) + signature := ed25519.Sign(privkey, append(payload, nonceHex...)) + + deliveryUuid := uuid.New().String() + body := bytes.NewBuffer(payload) + req, err := http.NewRequest("POST", sub.Url, body) + req.Header.Add("Content-Type", "application/json") + req.Header.Add("X-Webhook-Event", "repo:post-update") + req.Header.Add("X-Webhook-Delivery", deliveryUuid) + req.Header.Add("X-Payload-Nonce", string(nonceHex)) + req.Header.Add("X-Payload-Signature", + base64.StdEncoding.EncodeToString(signature)) + + var requestHeaders bytes.Buffer + for name, values := range req.Header { + requestHeaders.WriteString(fmt.Sprintf("%s: %s\n", + name, strings.Join(values, ", "))) + } + + delivery := WebhookDelivery{ + Headers: requestHeaders.String(), + Payload: string(payload), + ResponseStatus: -1, + SubscriptionId: sub.Id, + UUID: deliveryUuid, + Url: sub.Url, + } + + resp, err := client.Do(req) + if err != nil { + delivery.Response = fmt.Sprintf("Error sending webhook: %v") + log.Printf(delivery.Response) + deliveries = append(deliveries, delivery) + continue + } + defer resp.Body.Close() + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + delivery.Response = fmt.Sprintf("Error reading webhook "+ + "response: %v", err) + log.Printf(delivery.Response) + deliveries = append(deliveries, delivery) + continue + } + if !utf8.Valid(respBody) { + delivery.Response = "Webhook response is not valid UTF-8" + log.Printf(delivery.Response) + deliveries = append(deliveries, delivery) + continue + } + log.Println(runewidth.Truncate(string(respBody), 1024, "...")) + + var responseHeaders bytes.Buffer + for name, values := range resp.Header { + responseHeaders.WriteString(fmt.Sprintf("%s: %s\n", + name, strings.Join(values, ", "))) + } + + delivery.ResponseHeaders = responseHeaders.String() + delivery.Response = string(respBody)[:65535] + deliveries = append(deliveries, delivery) + } + + return deliveries +} diff --git a/gitsrht/blueprints/api.py b/gitsrht/blueprints/api.py index 88bb506..f4c126a 100644 --- a/gitsrht/blueprints/api.py +++ b/gitsrht/blueprints/api.py @@ -17,6 +17,7 @@ from srht.validation import Validation data = Blueprint("api.data", __name__) +# See also gitsrht-update-hook/types.go def commit_to_dict(c): return { "id": str(c.id), diff --git a/gitsrht/submit.py b/gitsrht/submit.py deleted file mode 100644 index 8460c1e..0000000 --- a/gitsrht/submit.py +++ /dev/null @@ -1,178 +0,0 @@ -import html -import os -import re -from pygit2 import Repository as GitRepository, Commit, Tag -from gitsrht.blueprints.api import commit_to_dict -from gitsrht.types import User, Repository -from scmsrht.redis import redis -from scmsrht.repos import RepoVisibility -from scmsrht.submit import BuildSubmitterBase -from gitsrht.webhooks import RepoWebhook -from srht.config import cfg, get_origin -from srht.database import db -from urllib.parse import urlparse - -builds_sr_ht = cfg("builds.sr.ht", "origin", None) -git_sr_ht = get_origin("git.sr.ht", external=True) - -def first_line(text): - try: - i = text.index("\n") - except ValueError: - return text + "\n" - else: - return text[:i + 1] - -class GitBuildSubmitter(BuildSubmitterBase): - def __init__(self, repo, git_repo): - super().__init__(git_sr_ht, 'git', repo) - self.git_repo = git_repo - - def find_manifests(self, commit): - manifest_blobs = dict() - if ".build.yml" in commit.tree: - build_yml = commit.tree[".build.yml"] - if build_yml.type == 'blob': - manifest_blobs[".build.yml"] = build_yml - elif ".builds" in commit.tree: - build_dir = commit.tree[".builds"] - if build_dir.type == 'tree': - manifest_blobs.update( - { - blob.name: blob - for blob in self.git_repo.get(build_dir.id) - if blob.type == 'blob' and ( - blob.name.endswith('.yml') - or blob.name.endswith('.yaml') - ) - } - ) - - manifests = {} - for name, blob in manifest_blobs.items(): - m = self.git_repo.get(blob.id).data.decode() - manifests[name] = m - return manifests - - def get_commit_id(self, commit): - return str(commit.id) - - def get_commit_note(self, commit): - return "[{}]({}) — [{}](mailto:{})\n\n{}".format( - str(commit.id)[:7], - "{}/{}/{}/commit/{}".format( - git_sr_ht, - "~" + self.repo.owner.username, - self.repo.name, - str(commit.id)), - commit.author.name, - commit.author.email, - "
" + html.escape(first_line(commit.message)) + "
", - ) - - def get_clone_url(self): - origin = get_origin("git.sr.ht", external=True) - owner_name = self.repo.owner.canonical_name - repo_name = self.repo.name - if self.repo.visibility == RepoVisibility.private: - # Use SSH URL - origin = origin.replace("http://", "").replace("https://", "") - return f"git+ssh://git@{origin}/{owner_name}/{repo_name}" - else: - # Use http(s) URL - return f"{origin}/{owner_name}/{repo_name}" - -# https://stackoverflow.com/a/14693789 -ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') - -def do_post_update(context, refs): - global db - # TODO: we shouldn't need this once we move most of this shit to the - # internal API - if not hasattr(db, "session"): - import gitsrht.types - from srht.database import DbSession - db = DbSession(cfg("git.sr.ht", "connection-string")) - db.init() - - uid = os.environ.get("SRHT_UID") - push = os.environ.get("SRHT_PUSH") - user = context["user"] - repo = context["repo"] - - payload = { - "push": push, - "pusher": user, - "refs": list(), - } - - git_repo = GitRepository(repo["path"]) - oids = set() - for ref in refs: - update = redis.get(f"update.{push}.{ref}") - if update: - old, new = update.decode().split(":") - old = git_repo.get(old) - new = git_repo.get(new) - update = dict() - if isinstance(new, Tag): - update.update({ - "annotated_tag": { - "name": new.name, - "message": new.message, - }, - }) - new = git_repo.get(new.target) - update.update({ - "name": ref, - "old": commit_to_dict(old) if old else None, - "new": commit_to_dict(new) if new else None, - }) - payload["refs"].append(update) - - try: - if re.match(r"^[0-9a-z]{40}$", ref): # commit - commit = git_repo.get(ref) - elif ref.startswith("refs/"): # ref - target_id = git_repo.lookup_reference(ref).target - commit = git_repo.get(target_id) - if isinstance(commit, Tag): - commit = git_repo.get(commit.target) - else: - continue - if not isinstance(commit, Commit): - continue - if commit.id in oids: - continue - oids.add(commit.id) - except: - continue - - if builds_sr_ht: - # TODO: move this to internal API - r = Repository.query.get(repo["id"]) - s = GitBuildSubmitter(r, git_repo) - res = s.submit(commit) - if res.status != 'skipped': - res.printmsgs() - - # TODO: get these from internal API - # sync webhooks - for resp in RepoWebhook.deliver(RepoWebhook.Events.repo_post_update, payload, - RepoWebhook.Subscription.repo_id == repo["id"], - RepoWebhook.Subscription.sync, - delay=False): - if resp == None: - # TODO: Add details? - print("Error submitting webhook") - continue - if resp.status_code != 200: - print(f"Webhook returned status {resp.status_code}") - try: - print(ansi_escape.sub('', resp.text)) - except: - print("Unable to decode webhook response") - # async webhooks - RepoWebhook.deliver(RepoWebhook.Events.repo_post_update, payload, - RepoWebhook.Subscription.repo_id == repo["id"], - RepoWebhook.Subscription.sync == False) diff --git a/scripts/symlink-update-hook.py b/scripts/symlink-update-hook.py index 346dc5d..f90924e 100755 --- a/scripts/symlink-update-hook.py +++ b/scripts/symlink-update-hook.py @@ -12,9 +12,14 @@ def migrate(path, link): if not os.path.exists(path) \ or not os.path.islink(path) \ or os.readlink(path) != link: - if os.path.exists(path): + try: os.remove(path) - os.symlink(link, path) + except: + pass + try: + os.symlink(link, path) + except: + pass return True return False diff --git a/setup.py b/setup.py index fee7611..ef3fd67 100755 --- a/setup.py +++ b/setup.py @@ -63,6 +63,5 @@ setup( scripts = [ 'gitsrht-migrate', 'gitsrht-periodic', - 'gitsrht-update-hook', ] ) -- 2.38.4