M gitsrht-keys/main.go => gitsrht-keys/main.go +2 -2
@@ 211,7 211,7 @@ func main() {
push := uuid.New()
shellCommand := fmt.Sprintf("%s '%d' '%s' '%s'",
shell, userId, username, b64key)
- fmt.Printf(`restrict,command="%s",environment="SRHT_UID=%d",`+
+ fmt.Printf(`restrict,command="%s",`+
`environment="SRHT_PUSH=%s" %s %s %s`+"\n",
- shellCommand, userId, push.String(), keyType, b64key, username)
+ shellCommand, push.String(), keyType, b64key, username)
}
D gitsrht-update-hook => gitsrht-update-hook +0 -45
@@ 1,45 0,0 @@
-#!/usr/bin/env python3
-import json
-import os
-import sys
-from srht.config import cfg
-from configparser import ConfigParser
-from datetime import datetime, timedelta
-from gitsrht.submit import do_post_update
-from scmsrht.redis import redis
-
-op = sys.argv[0]
-origin = cfg("git.sr.ht", "origin")
-
-if op == "hooks/update":
- # Stash updated refs for later processing
- refname = sys.argv[1]
- old = sys.argv[2]
- new = sys.argv[3]
-
- push_uuid = os.environ.get("SRHT_PUSH")
- if not push_uuid:
- sys.exit(0)
- redis.setex(f"update.{push_uuid}.{refname}",
- timedelta(minutes=10), f"{old}:{new}")
-
-if op == "hooks/post-update":
- refs = sys.argv[1:]
-
- config = ConfigParser()
- with open("config") as f:
- config.read_file(f)
-
- context = json.loads(os.environ.get("SRHT_PUSH_CTX"))
- repo = context["repo"]
-
- if repo["visibility"] == "autocreated":
- print("\n\t\033[93mNOTICE\033[0m")
- print("\tWe saved your changes, but this repository does not exist.")
- print("\tClick here to create it:")
- print()
- print("\t{}/create?name={}".format(origin, repo["name"]))
- print()
- print("\tYour changes will be discarded in 20 minutes.\n")
-
- do_post_update(context, refs)
A gitsrht-update-hook/.gitignore => gitsrht-update-hook/.gitignore +1 -0
@@ 0,0 1,1 @@
+gitsrht-update-hook
A gitsrht-update-hook/go.mod => gitsrht-update-hook/go.mod +16 -0
@@ 0,0 1,16 @@
+module git.sr.ht/~sircmpwn/git.sr.ht/gitsrht-update-hook
+
+go 1.13
+
+require (
+ github.com/go-redis/redis v6.15.6+incompatible
+ github.com/google/uuid v1.1.1
+ github.com/lib/pq v1.2.0
+ github.com/mattn/go-runewidth v0.0.6
+ github.com/microcosm-cc/bluemonday v1.0.2
+ github.com/pkg/errors v0.8.1
+ github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec
+ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
+ gopkg.in/src-d/go-git.v4 v4.13.1
+ gopkg.in/yaml.v2 v2.2.7
+)
A gitsrht-update-hook/go.sum => gitsrht-update-hook/go.sum +77 -0
@@ 0,0 1,77 @@
+github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-redis/redis v6.15.6+incompatible h1:H9evprGPLI8+ci7fxQx6WNZHJSb7be8FqJQRhdQZ5Sg=
+github.com/go-redis/redis v6.15.6+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
+github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-runewidth v0.0.6 h1:V2iyH+aX9C5fsYCpK60U8BYIvmhqxuOL3JZcqc1NB7k=
+github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
+github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
+github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks=
+github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw=
+github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
+github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
+golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
+gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
+gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
+gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
+gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
A gitsrht-update-hook/main.go => gitsrht-update-hook/main.go +67 -0
@@ 0,0 1,67 @@
+package main
+
+import (
+ "log"
+ "os"
+
+ "github.com/vaughan0/go-ini"
+)
+
+var (
+ buildOrigin string
+ config ini.File
+ logger *log.Logger
+ origin string
+ pgcs string
+)
+
+func main() {
+ log.SetFlags(0)
+ // The update hook is run on the update and post-update git hooks, and also
+ // runs a third stage directly. The first two stages are performance
+ // critical and take place while the user is blocked at their terminal. The
+ // third stage is done in the background.
+ if os.Args[0] == "hooks/update" {
+ update()
+ } else if os.Args[0] == "hooks/post-update" {
+ postUpdate()
+ } else if os.Args[0] == "stage-3" {
+ stage3()
+ } else {
+ log.Fatalf("Unknown git hook %s", os.Args[0])
+ }
+}
+
+func init() {
+ logf, err := os.OpenFile("/var/log/gitsrht-update-hook",
+ os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
+ if err != nil {
+ log.Printf("Warning: unable to open log file: %v "+
+ "(using stderr instead)", err)
+ logger = log.New(os.Stderr, os.Args[0]+" ", log.LstdFlags)
+ } else {
+ logger = log.New(logf, os.Args[0]+" ", log.LstdFlags)
+ }
+
+ for _, path := range []string{"../config.ini", "/etc/sr.ht/config.ini"} {
+ config, err = ini.LoadFile(path)
+ if err == nil {
+ break
+ }
+ }
+ if err != nil {
+ logger.Fatalf("Failed to load config file: %v", err)
+ }
+
+ var ok bool
+ origin, ok = config.Get("git.sr.ht", "origin")
+ if !ok {
+ logger.Fatalf("No origin configured for git.sr.ht")
+ }
+ pgcs, ok = config.Get("git.sr.ht", "connection-string")
+ if !ok {
+ logger.Fatalf("No connection string configured for git.sr.ht: %v", err)
+ }
+
+ buildOrigin, _ = config.Get("builds.sr.ht", "origin") // Optional
+}
A gitsrht-update-hook/manifest.go => gitsrht-update-hook/manifest.go +38 -0
@@ 0,0 1,38 @@
+package main
+
+// TODO: Move this into builds.sr.ht
+
+import (
+ "gopkg.in/yaml.v2"
+)
+
+type Manifest struct {
+ Arch *string `yaml:"arch",omitempty`
+ Environment map[string]interface{} `yaml:"environment",omitempty`
+ Image string `yaml:"image"`
+ Packages []string `yaml:"packages",omitempty`
+ Repositories map[string]string `yaml:"repositories",omitempty`
+ Secrets []string `yaml:"secrets",omitempty`
+ Shell bool `yaml:"shell",omitempty`
+ Sources []string `yaml:"sources",omitempty`
+ Tasks []map[string]string `yaml:"tasks"`
+ Triggers []map[string]interface{} `yaml:"triggers",omitempty`
+}
+
+func ManifestFromYAML(src string) (Manifest, error) {
+ var m Manifest
+ if err := yaml.Unmarshal([]byte(src), &m); err != nil {
+ return m, err
+ }
+ // XXX: We could do validation here, but builds.sr.ht will also catch it
+ // for us later so it's not especially important to
+ return m, nil
+}
+
+func (manifest Manifest) ToYAML() (string, error) {
+ bytes, err := yaml.Marshal(&manifest)
+ if err != nil {
+ return "", err
+ }
+ return string(bytes), nil
+}
A gitsrht-update-hook/post-update.go => gitsrht-update-hook/post-update.go +273 -0
@@ 0,0 1,273 @@
+package main
+
+import (
+ "database/sql"
+ "encoding/json"
+ "fmt"
+ "log"
+ "os"
+ "os/exec"
+ "strings"
+
+ goredis "github.com/go-redis/redis"
+ _ "github.com/lib/pq"
+ "gopkg.in/src-d/go-git.v4"
+ "gopkg.in/src-d/go-git.v4/plumbing"
+ "gopkg.in/src-d/go-git.v4/plumbing/object"
+)
+
+func printAutocreateInfo(context PushContext) {
+ log.Println("\n\t\033[93mNOTICE\033[0m")
+ log.Println("\tWe saved your changes, but this repository does not exist.")
+ log.Println("\tClick here to create it:")
+ log.Println()
+ log.Printf("\t%s/create?name=%s", origin, context.Repo.Name)
+ log.Println()
+ log.Println("\tYour changes will be discarded in 20 minutes.")
+ log.Println()
+}
+
+type DbInfo struct {
+ RepoId int
+ RepoName string
+ Visibility string
+ OwnerUsername string
+ OwnerToken string
+ AsyncWebhooks []WebhookSubscription
+ SyncWebhooks []WebhookSubscription
+}
+
+func fetchInfoForPush(db *sql.DB, repoId int) (DbInfo, error) {
+ var dbinfo DbInfo = DbInfo{RepoId: repoId}
+
+ // With this query, we:
+ // 1. Fetch the owner's username and OAuth token
+ // 2. Fetch the repository's name and visibility
+ // 3. Update the repository's mtime
+ // 4. Determine how many webhooks this repo has: if there are zero sync
+ // webhooks then we can defer looking them up until after we've sent the
+ // user on their way.
+ query, err := db.Prepare(`
+ UPDATE repository repo
+ SET updated = NOW() AT TIME ZONE 'UTC'
+ FROM (
+ SELECT "user".username, "user".oauth_token
+ FROM "user"
+ JOIN repository r ON r.owner_id = "user".id
+ WHERE r.id = $1
+ ) AS owner, (
+ SELECT
+ COUNT(*) FILTER(WHERE rws.sync = true) sync_count,
+ COUNT(*) FILTER(WHERE rws.sync = false) async_count
+ FROM repo_webhook_subscription rws
+ WHERE rws.repo_id = $1 AND rws.events LIKE '%repo:post-update%'
+ ) AS webhooks
+ WHERE repo.id = $1
+ RETURNING
+ repo.name,
+ repo.visibility,
+ owner.username,
+ owner.oauth_token,
+ webhooks.sync_count,
+ webhooks.async_count;
+ `)
+ if err != nil {
+ return dbinfo, err
+ }
+ defer query.Close()
+
+ var nasync, nsync int
+ if err = query.QueryRow(repoId).Scan(&dbinfo.RepoName, &dbinfo.Visibility,
+ &dbinfo.OwnerUsername, &dbinfo.OwnerToken,
+ &nsync, &nasync); err != nil {
+
+ return dbinfo, err
+ }
+
+ dbinfo.AsyncWebhooks = make([]WebhookSubscription, nasync)
+ dbinfo.SyncWebhooks = make([]WebhookSubscription, nsync)
+ if nsync == 0 {
+ // Don't fetch webhooks, we don't need to waste the user's time
+ return dbinfo, nil
+ }
+
+ var rows *sql.Rows
+ if rows, err = db.Query(`
+ SELECT id, url, events
+ FROM repo_webhook_subscription rws
+ WHERE rws.repo_id = $1
+ AND rws.events LIKE '%repo:post-update%'
+ AND rws.sync = true
+ `, repoId); err != nil {
+
+ return dbinfo, err
+ }
+ defer rows.Close()
+
+ for i := 0; rows.Next(); i++ {
+ var whs WebhookSubscription
+ if err = rows.Scan(&whs.Id, &whs.Url, &whs.Events); err != nil {
+ return dbinfo, err
+ }
+ dbinfo.SyncWebhooks[i] = whs
+ }
+
+ return dbinfo, nil
+}
+
+func postUpdate() {
+ var context PushContext
+ refs := os.Args[1:]
+
+ contextJson, ctxOk := os.LookupEnv("SRHT_PUSH_CTX")
+ pushUuid, pushOk := os.LookupEnv("SRHT_PUSH")
+ if !ctxOk || !pushOk {
+ logger.Fatal("Missing required variables in environment, " +
+ "configuration error?")
+ }
+
+ logger.Printf("Running post-update for push %s", pushUuid)
+
+ if err := json.Unmarshal([]byte(contextJson), &context); err != nil {
+ logger.Fatalf("unmarshal SRHT_PUSH_CTX: %v", err)
+ }
+
+ if context.Repo.Visibility == "autocreated" {
+ printAutocreateInfo(context)
+ }
+
+ payload := WebhookPayload{
+ Push: pushUuid,
+ Pusher: context.User,
+ Refs: make([]UpdatedRef, len(refs)),
+ }
+
+ oids := make(map[string]interface{})
+ repo, err := git.PlainOpen(context.Repo.Path)
+ if err != nil {
+ logger.Fatalf("git.PlainOpen: %v", err)
+ }
+
+ db, err := sql.Open("postgres", pgcs)
+ if err != nil {
+ logger.Fatalf("Failed to open a database connection: %v", err)
+ }
+
+ dbinfo, err := fetchInfoForPush(db, context.Repo.Id)
+ if err != nil {
+ logger.Fatalf("Failed to fetch info from database: %v", err)
+ }
+
+ redis := goredis.NewClient(&goredis.Options{Addr: "localhost:6379"})
+ for i, refname := range refs {
+ var oldref, newref string
+ var oldobj, newobj object.Object
+ updateKey := fmt.Sprintf("update.%s.%s", pushUuid, refname)
+ update, err := redis.Get(updateKey).Result()
+ if update == "" || err != nil {
+ logger.Println("redis.Get: missing key")
+ continue
+ } else {
+ parts := strings.Split(update, ":")
+ oldref = parts[0]
+ newref = parts[1]
+ }
+ oldobj, err = repo.Object(plumbing.AnyObject, plumbing.NewHash(oldref))
+ if err == plumbing.ErrObjectNotFound {
+ logger.Printf("old object %s not found", oldref)
+ continue
+ }
+ newobj, err = repo.Object(plumbing.AnyObject, plumbing.NewHash(newref))
+ if err == plumbing.ErrObjectNotFound {
+ logger.Printf("new object %s not found", newref)
+ continue
+ }
+
+ var atag *AnnotatedTag = nil
+ if tag, ok := newobj.(*object.Tag); ok {
+ atag = &AnnotatedTag{
+ Name: tag.Name,
+ Message: tag.Message,
+ }
+ newobj, err = repo.CommitObject(tag.Target)
+ if err != nil {
+ logger.Println("unresolvable annotated tag")
+ continue
+ }
+ }
+
+ oldcommit, ok := oldobj.(*object.Commit)
+ if !ok {
+ logger.Println("Skipping non-commit old ref")
+ continue
+ }
+ commit, ok := newobj.(*object.Commit)
+ if !ok {
+ logger.Println("Skipping non-commit new ref")
+ continue
+ }
+
+ payload.Refs[i] = UpdatedRef{
+ Tag: atag,
+ Name: refname,
+ Old: GitCommitToWebhookCommit(oldcommit),
+ New: GitCommitToWebhookCommit(commit),
+ }
+
+ if _, ok := oids[commit.Hash.String()]; ok {
+ continue
+ }
+ oids[commit.Hash.String()] = nil
+
+ if buildOrigin != "" {
+ submitter := GitBuildSubmitter{
+ BuildOrigin: buildOrigin,
+ Commit: commit,
+ GitOrigin: origin,
+ OwnerName: dbinfo.OwnerUsername,
+ OwnerToken: dbinfo.OwnerToken,
+ RepoName: dbinfo.RepoName,
+ Repository: repo,
+ Visibility: dbinfo.Visibility,
+ }
+ results, err := SubmitBuild(submitter)
+ if err != nil {
+ log.Fatalf("Error submitting build job: %v", err)
+ }
+ if len(results) == 0 {
+ continue
+ } else if len(results) == 1 {
+ log.Println("\033[1mBuild started:\033[0m")
+ } else {
+ log.Println("\033[1mBuilds started:\033[0m")
+ }
+ logger.Printf("Submitted %d builds for %s",
+ len(results), refname)
+ for _, result := range results {
+ log.Printf("\033[94m%s\033[0m [%s]", result.Url, result.Name)
+ }
+ }
+ }
+
+ payloadBytes, err := json.Marshal(&payload)
+ if err != nil {
+ logger.Fatalf("Failed to marshal webhook payload: %v", err)
+ }
+
+ deliveries := deliverWebhooks(dbinfo.SyncWebhooks, payloadBytes)
+ deliveriesJson, err := json.Marshal(deliveries)
+ if err != nil {
+ logger.Fatalf("Failed to marshal webhook deliveries: %v", err)
+ }
+
+ hook, ok := config.Get("git.sr.ht", "post-update-script")
+ if !ok {
+ logger.Fatal("No post-update script configured, cannot run stage 3")
+ }
+
+ // Run stage 3 asyncronously - the last few tasks can be done without
+ // blocking the pusher's terminal.
+ stage3 := exec.Command(hook, string(deliveriesJson), string(payloadBytes))
+ stage3.Args[0] = "stage-3"
+ stage3.Start()
+}
A gitsrht-update-hook/stage-3.go => gitsrht-update-hook/stage-3.go +83 -0
@@ 0,0 1,83 @@
+package main
+
+import (
+ "database/sql"
+ "encoding/json"
+ "os"
+
+ _ "github.com/lib/pq"
+)
+
+func stage3() {
+ var context PushContext
+ contextJson, ctxOk := os.LookupEnv("SRHT_PUSH_CTX")
+ pushUuid, pushOk := os.LookupEnv("SRHT_PUSH")
+ if !ctxOk || !pushOk {
+ logger.Fatal("Missing required variables in environment, " +
+ "configuration error?")
+ }
+
+ logger.Printf("Running stage 3 for push %s", pushUuid)
+
+ if err := json.Unmarshal([]byte(contextJson), &context); err != nil {
+ logger.Fatalf("unmarshal SRHT_PUSH_CTX: %v", err)
+ }
+
+ db, err := sql.Open("postgres", pgcs)
+ if err != nil {
+ logger.Fatalf("Failed to open a database connection: %v", err)
+ }
+
+ var subscriptions []WebhookSubscription
+ var deliveries []WebhookDelivery
+ if err := json.Unmarshal([]byte(os.Args[1]), &deliveries); err != nil {
+ logger.Fatalf("Unable to unmarhsal delivery array: %v", err)
+ }
+ payload := []byte(os.Args[2])
+
+ var rows *sql.Rows
+ if rows, err = db.Query(`
+ SELECT id, url, events
+ FROM repo_webhook_subscription rws
+ WHERE rws.repo_id = $1
+ AND rws.events LIKE '%repo:post-update%'
+ AND rws.sync = false`, context.Repo.Id); err != nil {
+ logger.Fatalf("Error fetching webhooks: %v", err)
+ }
+ defer rows.Close()
+
+ for i := 0; rows.Next(); i++ {
+ var whs WebhookSubscription
+ if err = rows.Scan(&whs.Id, &whs.Url, &whs.Events); err != nil {
+ logger.Fatalf("Scanning webhook rows: %v", err)
+ }
+ subscriptions = append(subscriptions, whs)
+ }
+
+ deliveries = append(deliveries, deliverWebhooks(subscriptions, payload)...)
+ for _, delivery := range deliveries {
+ if _, err := db.Exec(`
+ INSERT INTO repo_webhook_delivery (
+ uuid,
+ created,
+ event,
+ url,
+ payload,
+ payload_headers,
+ response,
+ response_status,
+ response_headers,
+ subscription_id
+ ) VALUES (
+ $1, NOW() AT TIME ZONE 'UTC', 'repo:post-update',
+ $2, $3, $4, $5, $6, $7
+ );
+ `, delivery.UUID, delivery.Url,
+ delivery.Payload, delivery.Headers,
+ delivery.Response, delivery.ResponseStatus, delivery.ResponseHeaders,
+ delivery.SubscriptionId); err != nil {
+
+ logger.Fatalf("Error inserting webhook delivery: %v", err)
+ }
+ }
+}
A gitsrht-update-hook/submitter.go => gitsrht-update-hook/submitter.go +265 -0
@@ 0,0 1,265 @@
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "path"
+ "strings"
+ "unicode/utf8"
+
+ "github.com/microcosm-cc/bluemonday"
+ "github.com/pkg/errors"
+ "gopkg.in/src-d/go-git.v4"
+ "gopkg.in/src-d/go-git.v4/plumbing/object"
+)
+
+type BuildSubmitter interface {
+ // Return a list of build manifests and their names
+ FindManifests() (map[string]string, error)
+ // Get builds.sr.ht origin
+ GetBuildsOrigin() string
+ // Get builds.sr.ht OAuth token
+ GetOauthToken() string
+ // Get a checkout-able string to append to matching source URLs
+ GetCommitId() string
+ // Get the build note which corresponds to this commit
+ GetCommitNote() string
+ // Get the clone URL for this repository
+ GetCloneUrl() string
+ // Get the name of the repository
+ GetRepoName() string
+ // Get the name of the repository owner
+ GetOwnerName() string
+}
+
+// SQL notes
+//
+// We need:
+// - The repo ID
+// - The repo name & visibility
+// - The owner's username & canonical name
+// - The owner's OAuth token & scopes
+// - A list of affected webhooks
+type GitBuildSubmitter struct {
+ BuildOrigin string
+ Commit *object.Commit
+ GitOrigin string
+ OwnerName string
+ OwnerToken string
+ RepoName string
+ Repository *git.Repository
+ Visibility string
+}
+
+func (submitter GitBuildSubmitter) FindManifests() (map[string]string, error) {
+ tree, err := submitter.Repository.TreeObject(submitter.Commit.TreeHash)
+ if err != nil {
+ return nil, errors.Wrap(err, "lookup tree failed")
+ }
+
+ var files []*object.File
+ file, err := tree.File(".build.yml")
+ if err == nil {
+ files = append(files, file)
+ } else {
+ subtree, err := tree.Tree(".builds")
+ if err != nil {
+ return nil, nil
+ }
+ entries := subtree.Files()
+ for {
+ file, err = entries.Next()
+ if file == nil || err != nil {
+ break;
+ }
+ if strings.HasSuffix(file.Name, ".yml") {
+ files = append(files, file)
+ }
+ }
+ if err != io.EOF {
+ return nil, errors.Wrap(err, "EOF finding build manifest")
+ }
+ }
+
+ manifests := make(map[string]string)
+ for _, file := range files {
+ var (
+ reader io.Reader
+ content []byte
+ )
+ if reader, err = file.Reader(); err != nil {
+ return nil, errors.Wrapf(err, "creating reader for %s", file.Name)
+ }
+ if content, err = ioutil.ReadAll(reader); err != nil {
+ return nil, errors.Wrap(err, "reading build manifest")
+ }
+ if !utf8.Valid(content) {
+ return nil, errors.Wrap(err, "manifest is not valid UTF-8 file")
+ }
+ manifests[file.Name] = string(content)
+ }
+ return manifests, nil
+}
+
+func (submitter GitBuildSubmitter) GetBuildsOrigin() string {
+ return submitter.BuildOrigin
+}
+
+func (submitter GitBuildSubmitter) GetOauthToken() string {
+ return submitter.OwnerToken
+}
+
+func (submitter GitBuildSubmitter) GetCommitId() string {
+ return submitter.Commit.Hash.String()
+}
+
+func firstLine(text string) string {
+ buf := bytes.NewBufferString(text)
+ scanner := bufio.NewScanner(buf)
+ if !scanner.Scan() {
+ return ""
+ }
+ return scanner.Text()
+}
+
+func (submitter GitBuildSubmitter) GetCommitNote() string {
+ policy := bluemonday.StrictPolicy()
+ commitUrl := fmt.Sprintf("%s/%s/%s/commit/%s", submitter.GitOrigin,
+ submitter.OwnerName, submitter.RepoName,
+ submitter.GetCommitId())
+ return fmt.Sprintf(`[%s](%s) — [%s](mailto:%s)\n\n<pre>%s</pre>`,
+ submitter.GetCommitId()[:7], commitUrl,
+ submitter.Commit.Author.Name, submitter.Commit.Author.Email,
+ policy.Sanitize(firstLine(submitter.Commit.Message)))
+}
+
+func (submitter GitBuildSubmitter) GetCloneUrl() string {
+ if submitter.Visibility == "private" {
+ origin := strings.ReplaceAll(submitter.GitOrigin, "http://", "")
+ origin = strings.ReplaceAll(origin, "https://", "")
+ // Use SSH URL
+ return fmt.Sprintf("git+ssh://git@%s/~%s/%s", origin,
+ submitter.OwnerName, submitter.RepoName)
+ } else {
+ // Use HTTP(s) URL
+ return fmt.Sprintf("%s/~%s/%s", submitter.GitOrigin,
+ submitter.OwnerName, submitter.RepoName)
+ }
+}
+
+func (submitter GitBuildSubmitter) GetRepoName() string {
+ return submitter.RepoName
+}
+
+func (submitter GitBuildSubmitter) GetOwnerName() string {
+ return submitter.OwnerName
+}
+
+type BuildSubmission struct {
+ // TODO: Move errors into this struct and set up per-submission error
+ // tracking
+ Name string
+ Url string
+}
+
+// TODO: Move this to scm.sr.ht
+func SubmitBuild(submitter BuildSubmitter) ([]BuildSubmission, error) {
+ manifests, err := submitter.FindManifests()
+ if err != nil || manifests == nil {
+ return nil, err
+ }
+
+ var results []BuildSubmission
+ for name, contents := range manifests {
+ manifest, err := ManifestFromYAML(contents)
+ if err != nil {
+ return nil, errors.Wrap(err, name)
+ }
+ autoSetupManifest(submitter, &manifest)
+
+ yaml, err := manifest.ToYAML()
+ if err != nil {
+ return nil, errors.Wrap(err, name)
+ }
+
+ client := &http.Client{}
+
+ submission := struct {
+ Manifest string `json:"manifest"`
+ Tags []string `json:"tags"`
+ }{
+ Manifest: yaml,
+ Tags: []string{submitter.GetRepoName(), name},
+ }
+ bodyBytes, err := json.Marshal(&submission)
+ if err != nil {
+ return nil, errors.Wrap(err, "preparing job")
+ }
+ body := bytes.NewBuffer(bodyBytes)
+
+ req, err := http.NewRequest("POST", fmt.Sprintf("%s/api/jobs",
+ submitter.GetBuildsOrigin()), body)
+ req.Header.Add("Authorization", fmt.Sprintf("token %s",
+ submitter.GetOauthToken()))
+ req.Header.Add("Content-Type", "application/json")
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, errors.Wrap(err, "job submission")
+ }
+
+ if resp.StatusCode == 403 {
+ return nil, errors.New("builds.sr.ht returned 403\n" +
+ "Log out and back into the website to authorize " +
+ "builds integration.")
+ }
+
+ defer resp.Body.Close()
+ respBytes, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, errors.Wrap(err, "read response")
+ }
+
+ if resp.StatusCode == 400 {
+ return nil, errors.New(fmt.Sprintf(
+ "builds.sr.ht returned %d\n", resp.StatusCode) +
+ string(respBytes))
+ }
+
+ var job struct {
+ Id int `json:"id"`
+ }
+ err = json.Unmarshal(respBytes, &job)
+ if err != nil {
+ return nil, errors.Wrap(err, "interpret response")
+ }
+
+ results = append(results, BuildSubmission{
+ Name: name,
+ Url: fmt.Sprintf("%s/~%s/job/%d",
+ submitter.GetBuildsOrigin(),
+ submitter.GetOwnerName(),
+ job.Id),
+ })
+ }
+
+ return results, nil
+}
+
+func autoSetupManifest(submitter BuildSubmitter, manifest *Manifest) {
+ var hasSelf bool
+ cloneUrl := submitter.GetCloneUrl() + "#" + submitter.GetCommitId()
+ for i, src := range manifest.Sources {
+ if path.Base(src) == submitter.GetRepoName() {
+ manifest.Sources[i] = cloneUrl
+ hasSelf = true
+ }
+ }
+ if !hasSelf {
+ manifest.Sources = append(manifest.Sources, cloneUrl)
+ }
+}
A gitsrht-update-hook/types.go => gitsrht-update-hook/types.go +106 -0
@@ 0,0 1,106 @@
+package main
+
+import (
+ "encoding/base64"
+ "io/ioutil"
+
+ "gopkg.in/src-d/go-git.v4/plumbing"
+ "gopkg.in/src-d/go-git.v4/plumbing/object"
+)
+
+type RepoContext struct {
+ Id int `json:"id"`
+ Name string `json:"name"`
+ Path string `json:"path"`
+ Visibility string `json:"visibility"`
+}
+
+type UserContext struct {
+ CanonicalName string `json:"canonical_name"`
+ Name string `json:"name"`
+}
+
+type PushContext struct {
+ Repo RepoContext `json:"repo"`
+ User UserContext `json:"user"`
+}
+
+type AnnotatedTag struct {
+ Name string `json:"name"`
+ Message string `json:"message"`
+}
+
+type CommitSignature struct {
+ Data string `json:"data"`
+ Signature string `json:"signature"`
+}
+
+type CommitAuthorship struct {
+ Email string `json:"email"`
+ Name string `json:"name"`
+}
+
+// See gitsrht/blueprints/api.py
+type Commit struct {
+ Id string `json:"id"`
+ Message string `json:"message"`
+ Parents []string `json:"parents"`
+ ShortId string `json:"short_id"`
+ Timestamp string `json:"timestamp"`
+ Tree string `json:"tree"`
+
+ Author CommitAuthorship `json:"author"`
+ Committer CommitAuthorship `json:"committer"`
+ Signature *CommitSignature `json:"signature"`
+}
+
+type UpdatedRef struct {
+ Tag *AnnotatedTag `json:"annotated_tag",omitempty`
+ Name string `json:"name"`
+ Old *Commit `json:"old"`
+ New *Commit `json:"commit"`
+}
+
+type WebhookPayload struct {
+ Push string `json:"push"`
+ Pusher UserContext `json:"pusher"`
+ Refs []UpdatedRef `json:"refs"`
+}
+
+func GitCommitToWebhookCommit(c *object.Commit) *Commit {
+ parents := make([]string, len(c.ParentHashes))
+ for i, p := range c.ParentHashes {
+ parents[i] = p.String()
+ }
+
+ var signature *CommitSignature = nil
+ if c.PGPSignature != "" {
+ encoded := &plumbing.MemoryObject{}
+ c.EncodeWithoutSignature(encoded)
+ reader, _ := encoded.Reader()
+ data, _ := ioutil.ReadAll(ioutil.NopCloser(reader))
+ signature = &CommitSignature{
+ Data: base64.StdEncoding.EncodeToString(data),
+ Signature: base64.StdEncoding.EncodeToString([]byte(c.PGPSignature)),
+ }
+ }
+
+ return &Commit{
+ Id: c.Hash.String(),
+ Message: c.Message,
+ Parents: parents,
+ ShortId: c.Hash.String()[:7],
+ Timestamp: c.Author.When.Format("2006-01-02T15:04:05-07:00"),
+ Tree: c.TreeHash.String(),
+ Author: CommitAuthorship{
+ // TODO: Add timestamp
+ Name: c.Author.Name,
+ Email: c.Author.Email,
+ },
+ Committer: CommitAuthorship{
+ Name: c.Committer.Name,
+ Email: c.Committer.Email,
+ },
+ Signature: signature,
+ }
+}
A gitsrht-update-hook/update.go => gitsrht-update-hook/update.go +28 -0
@@ 0,0 1,28 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "time"
+
+ goredis "github.com/go-redis/redis"
+)
+
+// XXX: This is run once for every single ref that's pushed. If someone pushes
+// lots of refs, it might be expensive. Needs to be tested.
+func update() {
+ var (
+ refname string = os.Args[1]
+ oldref string = os.Args[2]
+ newref string = os.Args[3]
+ )
+ pushUuid, ok := os.LookupEnv("SRHT_PUSH")
+ if !ok {
+ logger.Fatal("Missing SRHT_PUSH in environment, configuration error?")
+ }
+ logger.Printf("Running update for push %s", pushUuid)
+
+ redis := goredis.NewClient(&goredis.Options{Addr: "localhost:6379"})
+ redis.Set(fmt.Sprintf("update.%s.%s", pushUuid, refname),
+ fmt.Sprintf("%s:%s", oldref, newref), 10*time.Minute)
+}
A gitsrht-update-hook/webhooks.go => gitsrht-update-hook/webhooks.go +137 -0
@@ 0,0 1,137 @@
+package main
+
+import (
+ "bytes"
+ "crypto/rand"
+ "encoding/base64"
+ "encoding/hex"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "github.com/google/uuid"
+ "github.com/mattn/go-runewidth"
+ "golang.org/x/crypto/ed25519"
+)
+
+var (
+ privkey ed25519.PrivateKey
+)
+
+type WebhookSubscription struct {
+ Id int
+ Url string
+ Events string
+}
+
+// Note: unlike normal sr.ht services, we don't add webhook deliveries to the
+// database until after the HTTP request has been completed, to reduce time
+// spent blocking the user's terminal.
+type WebhookDelivery struct {
+ Headers string
+ Payload string
+ Response string
+ ResponseHeaders string
+ ResponseStatus int
+ SubscriptionId int
+ UUID string
+ Url string
+}
+
+func initWebhookKey() {
+ b64key, ok := config.Get("webhooks", "private-key")
+ if !ok {
+ logger.Fatalf("No webhook key configured")
+ }
+ seed, err := base64.StdEncoding.DecodeString(b64key)
+ if err != nil {
+ logger.Fatalf("base64 decode webhooks private key: %v", err)
+ }
+ privkey = ed25519.NewKeyFromSeed(seed)
+}
+
+func deliverWebhooks(subs []WebhookSubscription,
+ payload []byte) []WebhookDelivery {
+
+ var deliveries []WebhookDelivery
+ initWebhookKey()
+ client := &http.Client{Timeout: 5 * time.Second}
+
+ for _, sub := range subs {
+ var (
+ nonceSeed []byte
+ nonceHex []byte
+ )
+ _, err := rand.Read(nonceSeed)
+ if err != nil {
+ logger.Fatalf("generate nonce: %v", err)
+ }
+ hex.Encode(nonceHex, nonceSeed)
+ signature := ed25519.Sign(privkey, append(payload, nonceHex...))
+
+ deliveryUuid := uuid.New().String()
+ body := bytes.NewBuffer(payload)
+ req, err := http.NewRequest("POST", sub.Url, body)
+ req.Header.Add("Content-Type", "application/json")
+ req.Header.Add("X-Webhook-Event", "repo:post-update")
+ req.Header.Add("X-Webhook-Delivery", deliveryUuid)
+ req.Header.Add("X-Payload-Nonce", string(nonceHex))
+ req.Header.Add("X-Payload-Signature",
+ base64.StdEncoding.EncodeToString(signature))
+
+ var requestHeaders bytes.Buffer
+ for name, values := range req.Header {
+ requestHeaders.WriteString(fmt.Sprintf("%s: %s\n",
+ name, strings.Join(values, ", ")))
+ }
+
+ delivery := WebhookDelivery{
+ Headers: requestHeaders.String(),
+ Payload: string(payload),
+ ResponseStatus: -1,
+ SubscriptionId: sub.Id,
+ UUID: deliveryUuid,
+ Url: sub.Url,
+ }
+
+ resp, err := client.Do(req)
+ if err != nil {
+ delivery.Response = fmt.Sprintf("Error sending webhook: %v")
+ log.Printf(delivery.Response)
+ deliveries = append(deliveries, delivery)
+ continue
+ }
+ defer resp.Body.Close()
+ respBody, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ delivery.Response = fmt.Sprintf("Error reading webhook "+
+ "response: %v", err)
+ log.Printf(delivery.Response)
+ deliveries = append(deliveries, delivery)
+ continue
+ }
+ if !utf8.Valid(respBody) {
+ delivery.Response = "Webhook response is not valid UTF-8"
+ log.Printf(delivery.Response)
+ deliveries = append(deliveries, delivery)
+ continue
+ }
+ log.Println(runewidth.Truncate(string(respBody), 1024, "..."))
+
+ var responseHeaders bytes.Buffer
+ for name, values := range resp.Header {
+ responseHeaders.WriteString(fmt.Sprintf("%s: %s\n",
+ name, strings.Join(values, ", ")))
+ }
+
+ delivery.ResponseHeaders = responseHeaders.String()
+ delivery.Response = string(respBody)[:65535]
+ deliveries = append(deliveries, delivery)
+ }
+
+ return deliveries
+}
M gitsrht/blueprints/api.py => gitsrht/blueprints/api.py +1 -0
@@ 17,6 17,7 @@ from srht.validation import Validation
data = Blueprint("api.data", __name__)
+# See also gitsrht-update-hook/types.go
def commit_to_dict(c):
return {
"id": str(c.id),
D gitsrht/submit.py => gitsrht/submit.py +0 -178
@@ 1,178 0,0 @@
-import html
-import os
-import re
-from pygit2 import Repository as GitRepository, Commit, Tag
-from gitsrht.blueprints.api import commit_to_dict
-from gitsrht.types import User, Repository
-from scmsrht.redis import redis
-from scmsrht.repos import RepoVisibility
-from scmsrht.submit import BuildSubmitterBase
-from gitsrht.webhooks import RepoWebhook
-from srht.config import cfg, get_origin
-from srht.database import db
-from urllib.parse import urlparse
-
-builds_sr_ht = cfg("builds.sr.ht", "origin", None)
-git_sr_ht = get_origin("git.sr.ht", external=True)
-
-def first_line(text):
- try:
- i = text.index("\n")
- except ValueError:
- return text + "\n"
- else:
- return text[:i + 1]
-
-class GitBuildSubmitter(BuildSubmitterBase):
- def __init__(self, repo, git_repo):
- super().__init__(git_sr_ht, 'git', repo)
- self.git_repo = git_repo
-
- def find_manifests(self, commit):
- manifest_blobs = dict()
- if ".build.yml" in commit.tree:
- build_yml = commit.tree[".build.yml"]
- if build_yml.type == 'blob':
- manifest_blobs[".build.yml"] = build_yml
- elif ".builds" in commit.tree:
- build_dir = commit.tree[".builds"]
- if build_dir.type == 'tree':
- manifest_blobs.update(
- {
- blob.name: blob
- for blob in self.git_repo.get(build_dir.id)
- if blob.type == 'blob' and (
- blob.name.endswith('.yml')
- or blob.name.endswith('.yaml')
- )
- }
- )
-
- manifests = {}
- for name, blob in manifest_blobs.items():
- m = self.git_repo.get(blob.id).data.decode()
- manifests[name] = m
- return manifests
-
- def get_commit_id(self, commit):
- return str(commit.id)
-
- def get_commit_note(self, commit):
- return "[{}]({}) — [{}](mailto:{})\n\n{}".format(
- str(commit.id)[:7],
- "{}/{}/{}/commit/{}".format(
- git_sr_ht,
- "~" + self.repo.owner.username,
- self.repo.name,
- str(commit.id)),
- commit.author.name,
- commit.author.email,
- "<pre>" + html.escape(first_line(commit.message)) + "</pre>",
- )
-
- def get_clone_url(self):
- origin = get_origin("git.sr.ht", external=True)
- owner_name = self.repo.owner.canonical_name
- repo_name = self.repo.name
- if self.repo.visibility == RepoVisibility.private:
- # Use SSH URL
- origin = origin.replace("http://", "").replace("https://", "")
- return f"git+ssh://git@{origin}/{owner_name}/{repo_name}"
- else:
- # Use http(s) URL
- return f"{origin}/{owner_name}/{repo_name}"
-
-# https://stackoverflow.com/a/14693789
-ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
-
-def do_post_update(context, refs):
- global db
- # TODO: we shouldn't need this once we move most of this shit to the
- # internal API
- if not hasattr(db, "session"):
- import gitsrht.types
- from srht.database import DbSession
- db = DbSession(cfg("git.sr.ht", "connection-string"))
- db.init()
-
- uid = os.environ.get("SRHT_UID")
- push = os.environ.get("SRHT_PUSH")
- user = context["user"]
- repo = context["repo"]
-
- payload = {
- "push": push,
- "pusher": user,
- "refs": list(),
- }
-
- git_repo = GitRepository(repo["path"])
- oids = set()
- for ref in refs:
- update = redis.get(f"update.{push}.{ref}")
- if update:
- old, new = update.decode().split(":")
- old = git_repo.get(old)
- new = git_repo.get(new)
- update = dict()
- if isinstance(new, Tag):
- update.update({
- "annotated_tag": {
- "name": new.name,
- "message": new.message,
- },
- })
- new = git_repo.get(new.target)
- update.update({
- "name": ref,
- "old": commit_to_dict(old) if old else None,
- "new": commit_to_dict(new) if new else None,
- })
- payload["refs"].append(update)
-
- try:
- if re.match(r"^[0-9a-z]{40}$", ref): # commit
- commit = git_repo.get(ref)
- elif ref.startswith("refs/"): # ref
- target_id = git_repo.lookup_reference(ref).target
- commit = git_repo.get(target_id)
- if isinstance(commit, Tag):
- commit = git_repo.get(commit.target)
- else:
- continue
- if not isinstance(commit, Commit):
- continue
- if commit.id in oids:
- continue
- oids.add(commit.id)
- except:
- continue
-
- if builds_sr_ht:
- # TODO: move this to internal API
- r = Repository.query.get(repo["id"])
- s = GitBuildSubmitter(r, git_repo)
- res = s.submit(commit)
- if res.status != 'skipped':
- res.printmsgs()
-
- # TODO: get these from internal API
- # sync webhooks
- for resp in RepoWebhook.deliver(RepoWebhook.Events.repo_post_update, payload,
- RepoWebhook.Subscription.repo_id == repo["id"],
- RepoWebhook.Subscription.sync,
- delay=False):
- if resp == None:
- # TODO: Add details?
- print("Error submitting webhook")
- continue
- if resp.status_code != 200:
- print(f"Webhook returned status {resp.status_code}")
- try:
- print(ansi_escape.sub('', resp.text))
- except:
- print("Unable to decode webhook response")
- # async webhooks
- RepoWebhook.deliver(RepoWebhook.Events.repo_post_update, payload,
- RepoWebhook.Subscription.repo_id == repo["id"],
- RepoWebhook.Subscription.sync == False)
M scripts/symlink-update-hook.py => scripts/symlink-update-hook.py +7 -2
@@ 12,9 12,14 @@ def migrate(path, link):
if not os.path.exists(path) \
or not os.path.islink(path) \
or os.readlink(path) != link:
- if os.path.exists(path):
+ try:
os.remove(path)
- os.symlink(link, path)
+ except:
+ pass
+ try:
+ os.symlink(link, path)
+ except:
+ pass
return True
return False
M setup.py => setup.py +0 -1
@@ 63,6 63,5 @@ setup(
scripts = [
'gitsrht-migrate',
'gitsrht-periodic',
- 'gitsrht-update-hook',
]
)