From 579047169f6cf065509ad8d2ec6e19cf7c196bec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Thu, 13 Aug 2020 17:18:31 +0200 Subject: [PATCH] Use pygit2/go-git for creating new repos. Also link pre-receive in autocreated repos --- gitsrht-shell/go.mod | 6 ++-- gitsrht-shell/go.sum | 69 +++++++++++++++++++++++++++++++++++++++---- gitsrht-shell/main.go | 41 +++++++++++++------------ gitsrht/repos.py | 45 +++++++++++----------------- 4 files changed, 103 insertions(+), 58 deletions(-) diff --git a/gitsrht-shell/go.mod b/gitsrht-shell/go.mod index c9370a8..6140a10 100644 --- a/gitsrht-shell/go.mod +++ b/gitsrht-shell/go.mod @@ -1,10 +1,10 @@ module git.sr.ht/~sircmpwn/git.sr.ht/gitsrht-shell require ( - github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf - github.com/lib/pq v1.2.0 + github.com/go-git/go-git/v5 v5.1.0 + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 + github.com/lib/pq v1.8.0 github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec - golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 ) go 1.13 diff --git a/gitsrht-shell/go.sum b/gitsrht-shell/go.sum index b660f21..fe32c9e 100644 --- a/gitsrht-shell/go.sum +++ b/gitsrht-shell/go.sum @@ -1,13 +1,70 @@ -github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg= -github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= -github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM= +github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git v1.0.0 h1:YcN9iDGDoXuIw0vHls6rINwV416HYa0EB2X+RBsyYp4= +github.com/go-git/go-git v4.7.0+incompatible h1:+W9rgGY4DOKKdX2x6HxSR7HNeTxqiKrOvKnuittYVdA= +github.com/go-git/go-git-fixtures/v4 v4.0.1/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw= +github.com/go-git/go-git/v5 v5.1.0 h1:HxJn9g/E7eYvKW3Fm7Jt4ee8LXfPOm/H1cdDu8vEssk= +github.com/go-git/go-git/v5 v5.1.0/go.mod h1:ZKfuPUoY1ZqIG4QG9BDBh3G4gLM5zvPuSJAozQrZuyM= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= +github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks= github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw= +github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= +github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/gitsrht-shell/main.go b/gitsrht-shell/main.go index b98f1d4..810a5d3 100644 --- a/gitsrht-shell/main.go +++ b/gitsrht-shell/main.go @@ -17,6 +17,7 @@ import ( "github.com/google/shlex" _ "github.com/lib/pq" "github.com/vaughan0/go-ini" + "github.com/go-git/go-git/v5" ) const ( @@ -299,33 +300,31 @@ func main() { } // Note: update gitsrht/repos.py when changing this - if err = exec.Command("mkdir", "-p", path).Run(); err != nil { - notFound("mkdir", err) - } - if err = exec.Command("git", "init", - "--bare", path).Run(); err != nil { - + repo, err := git.PlainInit(path, true) + if err != nil { notFound("git init", err) } - if err = exec.Command("git", "-C", path, "config", - "srht.repo-id", strconv.Itoa(repoId)).Run(); err != nil { - - notFound("git config srht.repo-id", err) + config, err := repo.Config() + if err != nil { + notFound("git config load", err) } - if err = exec.Command("git", "-C", path, "config", - "receive.denyDeleteCurrent", "ignore").Run(); err != nil { - - notFound("git config receive.denyDeleteCurrent", err) + // These two are set by default by git(1) and libgit2 + config.Raw.SetOption("core", "", "repositoryformatversion", "0") + config.Raw.SetOption("core", "", "filemode", "true") + config.Raw.SetOption("srht", "", "repo-id", strconv.Itoa(repoId)) + config.Raw.SetOption("receive", "", "denyDeleteCurrent", "ignore") + if err = repo.Storer.SetConfig(config); err != nil { + notFound("git config save", err) } - if err = exec.Command("ln", "-s", postUpdate, - gopath.Join(path, "hooks", "update")).Run(); err != nil { - notFound("ln update", err) + hookdir := gopath.Join(path, "hooks") + if err = os.Mkdir(hookdir, os.ModePerm); err != nil { + notFound("git hook directory", err) } - if err = exec.Command("ln", "-s", postUpdate, - gopath.Join(path, "hooks", "post-update")).Run(); err != nil { - - notFound("ln post-update", err) + for _, hook := range []string{"pre-receive", "update", "post-update"} { + if err = os.Symlink(postUpdate, gopath.Join(hookdir, hook)); err != nil { + notFound(fmt.Sprintf("linking git hook %v"), err) + } } logger.Printf("Autocreated repo %s", path) diff --git a/gitsrht/repos.py b/gitsrht/repos.py index f0ad5fa..03fd7f4 100644 --- a/gitsrht/repos.py +++ b/gitsrht/repos.py @@ -1,5 +1,6 @@ import hashlib import os.path +import pygit2 import subprocess from gitsrht.types import Artifact, Repository, Redirect from minio import Minio @@ -85,38 +86,28 @@ class GitRepoApi(SimpleRepoApi): def do_init_repo(self, owner, repo): # Note: update gitsrht-shell when changing this, # do_clone_repo(), or _repo_config_init() - subprocess.run(["mkdir", "-p", repo.path], check=True, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - subprocess.run(["git", "init", "--bare"], cwd=repo.path, check=True, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - self._repo_config_init(repo) + git_repo = pygit2.init_repository(repo.path, bare=True, + flags=pygit2.GIT_REPOSITORY_INIT_BARE | + pygit2.GIT_REPOSITORY_INIT_MKPATH) + self._repo_config_init(repo, git_repo) - def _repo_config_init(self, repo): - subprocess.run(["git", "config", "srht.repo-id", str(repo.id)], check=True, - cwd=repo.path, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + def _repo_config_init(self, repo, git_repo): + git_repo.config["srht.repo-id"] = repo.id # We handle this ourselves in the post-update hook, and git's # default behaviour is to print a large notice and reject the push entirely - subprocess.run(["git", "config", "receive.denyDeleteCurrent", "ignore"], - check=True, cwd=repo.path, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - subprocess.run(["ln", "-s", - post_update, - os.path.join(repo.path, "hooks", "pre-receive") - ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - subprocess.run(["ln", "-s", - post_update, - os.path.join(repo.path, "hooks", "update") - ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - subprocess.run(["ln", "-s", - post_update, - os.path.join(repo.path, "hooks", "post-update") - ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + git_repo.config["receive.denyDeleteCurrent"] = "ignore" + os.unlink(os.path.join(repo.path, "info", "exclude")) + os.unlink(os.path.join(repo.path, "hooks", "README.sample")) + os.unlink(os.path.join(repo.path, "description")) + os.symlink(post_update, os.path.join(repo.path, "hooks", "pre-receive")) + os.symlink(post_update, os.path.join(repo.path, "hooks", "update")) + os.symlink(post_update, os.path.join(repo.path, "hooks", "post-update")) def do_delete_repo(self, repo): from gitsrht.webhooks import RepoWebhook RepoWebhook.Subscription.query.filter( RepoWebhook.Subscription.repo_id == repo.id).delete() - # TODO: Should we delete these asyncronously? + # TODO: Should we delete these asynchronously? for artifact in (Artifact.query .filter(Artifact.user_id == repo.owner_id) .filter(Artifact.repo_id == repo.id)): @@ -124,7 +115,5 @@ class GitRepoApi(SimpleRepoApi): super().do_delete_repo(repo) def do_clone_repo(self, source, repo): - subprocess.run(["mkdir", "-p", repo.path], check=True, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - subprocess.run(["git", "clone", "--bare", source, repo.path]) - self._repo_config_init(repo) + git_repo = pygit2.clone_repository(source, repo.path, bare=True) + self._repo_config_init(repo, git_repo) -- 2.38.4