From 188abe70faa0d81408d25409cd24bbc1f8d215c6 Mon Sep 17 00:00:00 2001 From: Adnan Maolood Date: Sat, 5 Feb 2022 23:04:12 -0500 Subject: [PATCH] api/graph: Do clones in the background --- api/clones/middleware.go | 55 ++++++++++++++++++++++++++ api/go.mod | 1 + api/graph/schema.resolvers.go | 73 ++++++++++++++++------------------- api/server.go | 5 ++- 4 files changed, 93 insertions(+), 41 deletions(-) create mode 100644 api/clones/middleware.go diff --git a/api/clones/middleware.go b/api/clones/middleware.go new file mode 100644 index 0000000..911fc58 --- /dev/null +++ b/api/clones/middleware.go @@ -0,0 +1,55 @@ +package clones + +import ( + "context" + "log" + "net/http" + + work "git.sr.ht/~sircmpwn/dowork" + "github.com/go-git/go-git/v5" +) + +type ClonesQueue struct { + Queue *work.Queue +} + +func NewQueue() *ClonesQueue { + return &ClonesQueue{work.NewQueue("clones")} +} + +type contextKey struct { + name string +} + +var clonesCtxKey = &contextKey{"clones"} + +func Middleware(queue *ClonesQueue) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := context.WithValue(r.Context(), clonesCtxKey, queue) + r = r.WithContext(ctx) + next.ServeHTTP(w, r) + }) + } +} + +// Schedules a clone. +func Schedule(ctx context.Context, repo *git.Repository, cloneURL string) { + queue, ok := ctx.Value(clonesCtxKey).(*ClonesQueue) + if !ok { + panic("No clones worker for this context") + } + task := work.NewTask(func(ctx context.Context) error { + err := repo.Clone(ctx, &git.CloneOptions{ + URL: cloneURL, + RecurseSubmodules: git.NoRecurseSubmodules, + }) + if err != nil { + // TODO: Set repo to error state. Email error to user. + panic(err) + } + return nil + }) + queue.Queue.Enqueue(task) + log.Printf("Enqueued clone of %s", cloneURL) +} diff --git a/api/go.mod b/api/go.mod index 34bd30d..5f10fdc 100644 --- a/api/go.mod +++ b/api/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( git.sr.ht/~sircmpwn/core-go v0.0.0-20220203103213-b2c8e81ee698 + git.sr.ht/~sircmpwn/dowork v0.0.0-20210820133136-d3970e97def3 github.com/99designs/gqlgen v0.14.0 github.com/Masterminds/squirrel v1.4.0 github.com/go-git/go-git/v5 v5.0.0 diff --git a/api/graph/schema.resolvers.go b/api/graph/schema.resolvers.go index 7c05b22..f9af2e7 100644 --- a/api/graph/schema.resolvers.go +++ b/api/graph/schema.resolvers.go @@ -28,6 +28,7 @@ import ( "git.sr.ht/~sircmpwn/core-go/server" "git.sr.ht/~sircmpwn/core-go/valid" corewebhooks "git.sr.ht/~sircmpwn/core-go/webhooks" + "git.sr.ht/~sircmpwn/git.sr.ht/api/clones" "git.sr.ht/~sircmpwn/git.sr.ht/api/graph/api" "git.sr.ht/~sircmpwn/git.sr.ht/api/graph/model" "git.sr.ht/~sircmpwn/git.sr.ht/api/loaders" @@ -147,11 +148,39 @@ func (r *mutationResolver) CreateRepository(ctx context.Context, name string, vi return err } - var gitrepo *git.Repository + gitrepo, err := git.PlainInit(repoPath, true) + if err != nil { + return err + } + repoCreated = true + + gitconfig, err := gitrepo.Config() + if err != nil { + return err + } + gitconfig.Raw.SetOption("core", "", "repositoryformatversion", "0") + gitconfig.Raw.SetOption("core", "", "filemode", "true") + gitconfig.Raw.SetOption("srht", "", "repo-id", strconv.Itoa(repo.ID)) + gitconfig.Raw.SetOption("receive", "", "denyDeleteCurrent", "ignore") + gitconfig.Raw.SetOption("receive", "", "advertisePushOptions", "true") + if err := gitrepo.Storer.SetConfig(gitconfig); err != nil { + return err + } + + hookdir := path.Join(repoPath, "hooks") + if err := os.Mkdir(hookdir, os.ModePerm); err != nil { + return err + } + for _, hook := range []string{"pre-receive", "update", "post-update"} { + if err := os.Symlink(postUpdate, path.Join(hookdir, hook)); err != nil { + return err + } + } + if cloneURL != nil { u, err := url.Parse(*cloneURL) if err != nil { - return err + return valid.Errorf(ctx, "cloneUrl", "Invalid clone URL: %s", err) } else if u.Host == "" { return valid.Errorf(ctx, "cloneUrl", "Cannot use URL without host") } else if _, ok := allowedCloneSchemes[u.Scheme]; !ok { @@ -181,50 +210,14 @@ func (r *mutationResolver) CreateRepository(ctx context.Context, name string, vi repo, err := loaders.ForContext(ctx). RepositoriesByOwnerRepoName.Load([2]string{entity, repoName}) if err != nil { - return err + panic(err) } else if repo == nil { return valid.Errorf(ctx, "cloneUrl", "Repository not found") } cloneURL = &repo.Path } - gitrepo, err = git.PlainClone(repoPath, true, &git.CloneOptions{ - URL: *cloneURL, - RecurseSubmodules: git.NoRecurseSubmodules, - }) - if err != nil { - return valid.Errorf(ctx, "cloneUrl", "%s", err) - } - } else { - var err error - gitrepo, err = git.PlainInit(repoPath, true) - if err != nil { - return err - } - } - - repoCreated = true - config, err := gitrepo.Config() - if err != nil { - return err - } - config.Raw.SetOption("core", "", "repositoryformatversion", "0") - config.Raw.SetOption("core", "", "filemode", "true") - config.Raw.SetOption("srht", "", "repo-id", strconv.Itoa(repo.ID)) - config.Raw.SetOption("receive", "", "denyDeleteCurrent", "ignore") - config.Raw.SetOption("receive", "", "advertisePushOptions", "true") - if err = gitrepo.Storer.SetConfig(config); err != nil { - return err - } - - hookdir := path.Join(repoPath, "hooks") - if err = os.Mkdir(hookdir, os.ModePerm); err != nil { - return err - } - for _, hook := range []string{"pre-receive", "update", "post-update"} { - if err = os.Symlink(postUpdate, path.Join(hookdir, hook)); err != nil { - return err - } + clones.Schedule(ctx, gitrepo, *cloneURL) } webhooks.DeliverRepoEvent(ctx, model.WebhookEventRepoCreated, &repo) diff --git a/api/server.go b/api/server.go index 64c42c3..a99a05a 100644 --- a/api/server.go +++ b/api/server.go @@ -7,6 +7,7 @@ import ( "git.sr.ht/~sircmpwn/core-go/server" "github.com/99designs/gqlgen/graphql" + "git.sr.ht/~sircmpwn/git.sr.ht/api/clones" "git.sr.ht/~sircmpwn/git.sr.ht/api/graph" "git.sr.ht/~sircmpwn/git.sr.ht/api/graph/api" "git.sr.ht/~sircmpwn/git.sr.ht/api/graph/model" @@ -30,6 +31,7 @@ func main() { scopes[i] = s.String() } + clonesQueue := clones.NewQueue() webhookQueue := webhooks.NewQueue(schema) legacyWebhooks := webhooks.NewLegacyQueue() @@ -37,10 +39,11 @@ func main() { WithDefaultMiddleware(). WithMiddleware( loaders.Middleware, + clones.Middleware(clonesQueue), webhooks.Middleware(webhookQueue), webhooks.LegacyMiddleware(legacyWebhooks), ). WithSchema(schema, scopes). - WithQueues(webhookQueue.Queue, legacyWebhooks.Queue). + WithQueues(clonesQueue.Queue, webhookQueue.Queue, legacyWebhooks.Queue). Run() } -- 2.38.4