From 8e47e31c592dac9d02329f168cc6819684547835 Mon Sep 17 00:00:00 2001 From: Drew DeVault Date: Thu, 14 Nov 2019 14:01:23 -0500 Subject: [PATCH] Move gitsrht-shell entirely into Golang binary This should improve push/pull performance considerably. --- gitsrht-shell/go.mod | 1 + gitsrht-shell/go.sum | 2 + gitsrht-shell/main.go | 345 +++++++++++++++++++++++---------- gitsrht-update-hook | 10 +- gitsrht/app.py | 2 - gitsrht/blueprints/internal.py | 87 --------- gitsrht/repos.py | 1 + 7 files changed, 251 insertions(+), 197 deletions(-) delete mode 100644 gitsrht/blueprints/internal.py diff --git a/gitsrht-shell/go.mod b/gitsrht-shell/go.mod index e3c7e00..c9370a8 100644 --- a/gitsrht-shell/go.mod +++ b/gitsrht-shell/go.mod @@ -2,6 +2,7 @@ module git.sr.ht/~sircmpwn/git.sr.ht/gitsrht-shell require ( github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf + github.com/lib/pq v1.2.0 github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 ) diff --git a/gitsrht-shell/go.sum b/gitsrht-shell/go.sum index 984f16c..b660f21 100644 --- a/gitsrht-shell/go.sum +++ b/gitsrht-shell/go.sum @@ -1,5 +1,7 @@ github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks= github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/gitsrht-shell/main.go b/gitsrht-shell/main.go index a398ffa..4caedc7 100644 --- a/gitsrht-shell/main.go +++ b/gitsrht-shell/main.go @@ -1,16 +1,10 @@ package main import ( - "bytes" - "crypto/rand" - "encoding/base64" - "encoding/hex" "encoding/json" - "fmt" - "io/ioutil" + "database/sql" "log" - "net/http" - "net/url" + "fmt" "os" "os/exec" gopath "path" @@ -19,9 +13,16 @@ import ( "strings" "syscall" + _ "github.com/lib/pq" "github.com/google/shlex" "github.com/vaughan0/go-ini" - "golang.org/x/crypto/ed25519" +) + +const ( + ACCESS_NONE = 0 + ACCESS_READ = 1 + ACCESS_WRITE = 2 + ACCESS_MANAGE = 4 ) func main() { @@ -30,17 +31,20 @@ func main() { err error logger *log.Logger - userId int - username string + pusherId int + pusherName string - origin string - repos string - privkey ed25519.PrivateKey + origin string + repos string + siteOwnerName string + siteOwnerEmail string + postUpdate string cmdstr string cmd []string ) + log.SetFlags(0) logf, err := os.OpenFile("/var/log/gitsrht-shell", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) if err != nil { @@ -56,10 +60,10 @@ func main() { } logger.Printf("os.Args: %v", os.Args) - if userId, err = strconv.Atoi(os.Args[1]); err != nil { + if pusherId, err = strconv.Atoi(os.Args[1]); err != nil { logger.Fatalf("Couldn't interpret user ID: %v", err) } - username = os.Args[2] + pusherName = os.Args[2] for _, path := range []string{"../config.ini", "/etc/sr.ht/config.ini"} { config, err = ini.LoadFile(path) @@ -71,11 +75,8 @@ func main() { logger.Fatalf("Failed to load config file: %v", err) } - origin, ok := config.Get("git.sr.ht", "internal-origin") + origin, ok := config.Get("git.sr.ht", "origin") if !ok { - origin, ok = config.Get("git.sr.ht", "origin") - } - if !ok || origin == "" { logger.Fatalf("No origin configured for git.sr.ht") } @@ -84,15 +85,13 @@ func main() { logger.Fatalf("No repo path configured for git.sr.ht") } - b64key, ok := config.Get("webhooks", "private-key") + postUpdate, ok = config.Get("git.sr.ht", "post-update-script") if !ok { - logger.Fatalf("No webhook key configured") - } - seed, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - logger.Fatalf("base64 decode webhooks private key: %v", err) + logger.Fatalf("No post-update script configured for git.sr.ht") } - privkey = ed25519.NewKeyFromSeed(seed) + + siteOwnerName, _ = config.Get("sr.ht", "owner-name") + siteOwnerEmail, _ = config.Get("sr.ht", "owner-email") cmdstr, ok = os.LookupEnv("SSH_ORIGINAL_COMMAND") if !ok { @@ -118,8 +117,8 @@ func main() { if !valid { logger.Printf("Not permitting unacceptable command: %v", cmd) - fmt.Printf("Hi %s! You've successfully authenticated, " + - "but I do not provide an interactive shell. Bye!\n", username) + log.Printf("Hi %s! You've successfully authenticated, " + + "but I do not provide an interactive shell. Bye!", pusherName) os.Exit(128) } @@ -135,93 +134,239 @@ func main() { } cmd[len(cmd)-1] = path - access := 1 + needsAccess := ACCESS_READ if cmd[0] == "git-receive-pack" { - access = 2 + needsAccess = ACCESS_WRITE } - payload, err := json.Marshal(struct { - Access int `json:"access"` - Path string `json:"path"` - UserId int `json:"user_id"` - }{ - Access: access, - Path: path, - UserId: userId, - }) + pgcs, ok := config.Get("git.sr.ht", "connection-string") + if !ok { + logger.Fatalf("No connection string configured for git.sr.ht: %v", err) + } + db, err := sql.Open("postgres", pgcs) if err != nil { - logger.Fatalf("json.Marshal: %v", err) + logger.Fatalf("Failed to open a database connection: %v", err) + } + if err := db.Ping(); err != nil { + logger.Fatalf("Failed to open a database connection: %v", err) } - logger.Println(string(payload)) + // Note: when updating push access logic, also update scm.sr.ht/access.py var ( - nonceSeed []byte - nonceHex []byte + repoId int + repoName string + repoOwnerId int + repoOwnerName string + repoVisibility string + pusherType string + pusherSuspendNotice string + accessGrant *string ) - _, err = rand.Read(nonceSeed) - if err != nil { - logger.Fatalf("generate nonce: %v", err) + row := db.QueryRow(` + SELECT + repo.id, + repo.name, + repo.owner_id, + repo.visibility, + owner.username, + pusher.user_type, + pusher.suspension_notice, + access.mode + FROM repository repo + JOIN "user" owner ON owner.id = repo.owner_id + JOIN "user" pusher ON pusher.id = $1 + LEFT JOIN access + ON (access.repo_id = repo.id AND access.user_id = $1) + WHERE + repo.path = $2; + `, pusherId, path) + if err := row.Scan(&repoId, &repoName, &repoOwnerId, &repoOwnerName, + &repoVisibility, &pusherType, &pusherSuspendNotice, &accessGrant); err != nil { + + row = db.QueryRow(` + SELECT + repo.id, + repo.name, + repo.owner_id, + repo.visibility, + owner.username, + pusher.user_type, + pusher.suspension_notice, + access.mode + FROM repository repo + JOIN "user" owner ON owner.id = repo.owner_id + JOIN "user" pusher ON pusher.id = $1 + JOIN redirect ON redirect.new_repo_id = repo.id + LEFT JOIN access + ON (access.repo_id = repo.id AND access.user_id = $1) + WHERE + redirect.path = $2; + `, pusherId, path) + + if err := row.Scan(&repoId, &repoName, &repoOwnerId, &repoOwnerName, + &repoVisibility, &pusherType, &pusherSuspendNotice, + &accessGrant); err != nil { + + repoName = gopath.Base(path) + repoOwnerName = gopath.Base(gopath.Dir(path)) + if repoOwnerName != "" { + repoOwnerName = repoOwnerName[1:] + } + + notFound := func(ctx string, err error) { + if err != nil { + logger.Printf("Error autocreating repo: %s: %v", ctx, err) + } + log.Println("Repository not found.") + log.Println() + os.Exit(128) + } + + if needsAccess == ACCESS_READ || repoOwnerName != pusherName { + notFound("access", nil) + } + + if needsAccess == ACCESS_WRITE { + repoOwnerId = pusherId + repoOwnerName = pusherName + repoVisibility = "autocreated" + + createQuery, err := db.Prepare(` + INSERT INTO repository ( + created, + updated, + name, + owner_id, + path, + visibility + ) VALUES ( + NOW() at time zone 'utc', + NOW() at time zone 'utc', + $1, $2, $3, 'autocreated' + ) RETURNING id; + `) + if err != nil { + notFound("create query prepare", err) + } + defer createQuery.Close() + + if createQuery.QueryRow(repoName, repoOwnerId, path). + Scan(&repoId); err != nil { + + notFound("insert", err) + } + + // Note: update gitsrht/repos.py when changing this + if err = exec.Command("mkdir", "-p", path).Run(); err != nil { + notFound("mkdir", err) + } + if err = exec.Command("git", "init", + "--bare", path).Run(); err != nil { + + notFound("git init", err) + } + if err = exec.Command("ln", "-s", postUpdate, + gopath.Join(path, "hooks", "update")).Run(); err != nil { + + notFound("ln update", err) + } + if err = exec.Command("ln", "-s", postUpdate, + gopath.Join(path, "hooks", "post-update")).Run(); err != nil { + + notFound("ln post-update", err) + } + + logger.Printf("Autocreated repo %s", path) + } + } else { + log.Printf("\033[93mNOTICE\033[0m: This repository has moved.") + log.Printf("Please update your remote to:") + log.Println() + log.Printf("\t%s/~%s/%s", origin, repoOwnerName, repoName) + log.Println() + } } - hex.Encode(nonceHex, nonceSeed) - signature := ed25519.Sign(privkey, append(payload, nonceHex...)) - headers := make(http.Header) - headers.Add("Content-Type", "application/json") - headers.Add("X-Payload-Nonce", string(nonceHex)) - headers.Add("X-Payload-Signature", - base64.StdEncoding.EncodeToString(signature)) + hasAccess := ACCESS_NONE + if pusherId == repoOwnerId { + hasAccess = ACCESS_READ | ACCESS_WRITE | ACCESS_MANAGE + } else { + if accessGrant == nil { + switch repoVisibility { + case "public": + fallthrough + case "unlisted": + hasAccess = ACCESS_READ + case "private": + hasAccess = ACCESS_NONE + default: + hasAccess = ACCESS_NONE + } + } else { + switch *accessGrant { + case "r": + hasAccess = ACCESS_READ + case "rw": + hasAccess = ACCESS_WRITE + default: + hasAccess = ACCESS_NONE + } + } + } - check, err := url.Parse(fmt.Sprintf("%s/internal/push-check", origin)) - if err != nil { - logger.Fatalf("url.Parse: %v", err) + if needsAccess & hasAccess != needsAccess { + log.Println("Access denied.") + log.Println() + os.Exit(128) } - req := http.Request{ - Body: ioutil.NopCloser(bytes.NewBuffer(payload)), - ContentLength: int64(len(payload)), - Header: headers, - Method: "POST", - URL: check, + + if pusherType == "suspended" { + log.Println("Your account has been suspended for the following reason:") + log.Println() + log.Println("\t" + pusherSuspendNotice) + log.Println() + log.Printf("Please contact support: %s <%s>", + siteOwnerName, siteOwnerEmail) + log.Println() + os.Exit(128) } - resp, err := http.DefaultClient.Do(&req) - if err != nil { - logger.Fatalf("http.Client.Do: %v", err) + + type RepoContext struct { + Id int `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + Visibility string `json:"visibility"` } - defer resp.Body.Close() - results, err := ioutil.ReadAll(resp.Body) - if err != nil { - logger.Fatal("ReadAll(resp.Body): %v", err) + + type UserContext struct { + CanonicalName string `json:"canonical_name"` + Name string `json:"name"` } - logger.Println(string(results)) - switch resp.StatusCode { - case 302: - var redirect struct { - Redirect string `json:"redirect"` - } - json.Unmarshal(results, &redirect) + pushContext, _ := json.Marshal(struct { + Repo RepoContext `json:"repo"` + User UserContext `json:"user"` + }{ + Repo: RepoContext{ + Id: repoId, + Name: repoName, + Path: path, + Visibility: repoVisibility, + }, + User: UserContext{ + CanonicalName: "~" + pusherName, + Name: pusherName, + }, + }) - fmt.Printf("\n\t\033[93mNOTICE\033[0m\n\n") - fmt.Printf("\tThis repository has moved:\n\n") - fmt.Printf("\t%s\n\n", redirect.Redirect) - fmt.Printf("\tPlease update your remote.\n\n\n") - os.Exit(128) - case 200: - logger.Printf("Executing command: %v", cmd) - bin, err := exec.LookPath(cmd[0]) - if err != nil { - logger.Fatalf("exec.LookPath: %v", err) - } - if err := syscall.Exec(bin, cmd, - append(os.Environ(), fmt.Sprintf( - "SRHT_PUSH_CTX=%s", string(results)))); err != nil { - logger.Fatalf("syscall.Exec: %v", err) - } - default: - var why struct { - Why string `json:"why"` - } - json.Unmarshal(results, &why) - fmt.Println(why.Why) - os.Exit(128) + logger.Printf("Executing command: %v", cmd) + bin, err := exec.LookPath(cmd[0]) + if err != nil { + logger.Fatalf("exec.LookPath: %v", err) + } + if err := syscall.Exec(bin, cmd, + append(os.Environ(), fmt.Sprintf( + "SRHT_PUSH_CTX=%s", string(pushContext)))); err != nil { + logger.Fatalf("syscall.Exec: %v", err) } } diff --git a/gitsrht-update-hook b/gitsrht-update-hook index 8af77cc..989f1a9 100755 --- a/gitsrht-update-hook +++ b/gitsrht-update-hook @@ -2,9 +2,6 @@ import json import os import sys -# TEMP -sys.path.append('/home/sircmpwn/sources/git.sr.ht') -sys.path.append('/home/sircmpwn/sources/scm.sr.ht') from srht.config import cfg from configparser import ConfigParser from datetime import datetime, timedelta @@ -33,11 +30,6 @@ if op == "hooks/post-update": with open("config") as f: config.read_file(f) - repo_id = config.get("srht", "repo-id") - if not repo_id: - sys.exit(0) - repo_id = int(repo_id) - context = json.loads(os.environ.get("SRHT_PUSH_CTX")) repo = context["repo"] @@ -45,7 +37,9 @@ if op == "hooks/post-update": print("\n\t\033[93mNOTICE\033[0m") print("\tWe saved your changes, but this repository does not exist.") print("\tClick here to create it:") + print() print("\t{}/create?name={}".format(origin, repo["name"])) + print() print("\tYour changes will be discarded in 20 minutes.\n") do_post_update(context, refs) diff --git a/gitsrht/app.py b/gitsrht/app.py index e9deb37..923c4e1 100644 --- a/gitsrht/app.py +++ b/gitsrht/app.py @@ -24,13 +24,11 @@ class GitApp(ScmSrhtFlask): from gitsrht.blueprints.api import data from gitsrht.blueprints.email import mail - from gitsrht.blueprints.internal import internal from gitsrht.blueprints.repo import repo from gitsrht.blueprints.stats import stats self.register_blueprint(data) self.register_blueprint(mail) - self.register_blueprint(internal) self.register_blueprint(repo) self.register_blueprint(stats) self.register_blueprint(webhooks_notify) diff --git a/gitsrht/blueprints/internal.py b/gitsrht/blueprints/internal.py deleted file mode 100644 index 2c0bfab..0000000 --- a/gitsrht/blueprints/internal.py +++ /dev/null @@ -1,87 +0,0 @@ -""" -This blueprint is used internally by gitsrht-shell to speed up git pushes, by -taking advantage of the database connection already established by the web app. -""" - -from datetime import datetime -from flask import Blueprint, request -from gitsrht.repos import GitRepoApi -from gitsrht.types import User, Repository, RepoVisibility, Redirect -from scmsrht.access import has_access, UserAccess -from scmsrht.urls import get_clone_urls -from srht.config import cfg, get_origin -from srht.crypto import verify_request_signature -from srht.database import db -from srht.flask import csrf_bypass -from srht.oauth import UserType -from srht.validation import Validation -import base64 -import os - -internal = Blueprint("internal", __name__) - -@csrf_bypass -@internal.route("/internal/push-check", methods=["POST"]) -def push_check(): - verify_request_signature(request) - valid = Validation(request) - path = valid.require("path") - user_id = valid.require("user_id", cls=int) - access = valid.require("access", cls=int) - if not valid.ok: - return valid.response - access = UserAccess(access) - user = User.query.filter(User.id == user_id).one() - - def push_context(user, repo): - if access == UserAccess.write: - repo.updated = datetime.utcnow() - db.session.commit() - return { - "user": user.to_dict(), - "repo": { - "path": repo.path, - **repo.to_dict(), - }, - } - - repo = Repository.query.filter(Repository.path == path).first() - if not repo: - redir = Redirect.query.filter(Redirect.path == path).first() - if redir: - origin = get_origin("git.sr.ht", external=True) - repo = redir.new_repo - # TODO: orgs - return { - "redirect": 'git@{origin}:{repo.owner.username}/{repo.name}' - }, 302 - - if access == UserAccess.write: - # Autocreate this repo - _path, repo_name = os.path.split(path) - owner = os.path.basename(_path) - if "~" + user.username != owner: - return { }, 401 - - valid = Validation({ "name": repo_name }) - repo_api = GitRepoApi() - repo = repo_api.create_repo(valid, user) - if not valid.ok: - return valid.response - repo.visibility = RepoVisibility.autocreated - db.session.commit() - return push_context(user, repo), 200 - else: - return { }, 404 - - if not has_access(repo, access, user): - return { }, 401 - - if access == UserAccess.write and user.user_type == UserType.suspended: - return { - "why": "Your account has been suspended with the following notice:\n" + - user.suspension_notice + "\n" + - "Please contact support: " + cfg("sr.ht", "owner-email"), - }, 401 - - return push_context(user, repo), 200 diff --git a/gitsrht/repos.py b/gitsrht/repos.py index 054bf3a..c8894f8 100644 --- a/gitsrht/repos.py +++ b/gitsrht/repos.py @@ -14,6 +14,7 @@ class GitRepoApi(SimpleRepoApi): repository_class=Repository) def do_init_repo(self, owner, repo): + # Note: update gitsrht-shell when changing this subprocess.run(["mkdir", "-p", repo.path], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) subprocess.run(["git", "init", "--bare"], cwd=repo.path, check=True, -- 2.38.4