~edwargix/git.sr.ht

3b3a7d801748f61474f2302201823f2519b8edc9 — Drew DeVault 6 years ago 4760455
Lay groundwork for push performance improvements
M gitsrht-shell => gitsrht-shell +1 -0
@@ 71,6 71,7 @@ if r.status_code == 302:
    print("\tPlease update your remote.\n\n")
    sys.exit(128)
elif r.status_code == 200:
    os.environ["SRHT_PUSH_CTX"] = r.text
    log("Executing {}", " ".join(cmd))
    sys.stderr.close()
    os.execvp(cmd[0], cmd)

M gitsrht-update-hook => gitsrht-update-hook +11 -15
@@ 1,15 1,15 @@
#!/usr/bin/env python3
import json
import os
import sys
# TEMP
sys.path.append('/home/sircmpwn/sources/git.sr.ht')
sys.path.append('/home/sircmpwn/sources/scm.sr.ht')
from srht.config import cfg
from srht.database import DbSession
db = DbSession(cfg("git.sr.ht", "connection-string"))
from gitsrht.types import Repository, RepoVisibility
db.init()
from configparser import ConfigParser
from datetime import datetime, timedelta
from gitsrht.submit import do_post_update
from scmsrht.redis import redis
import os
import sys

op = sys.argv[0]
origin = cfg("git.sr.ht", "origin")


@@ 38,18 38,14 @@ if op == "hooks/post-update":
        sys.exit(0)
    repo_id = int(repo_id)

    repo = Repository.query.get(repo_id)
    if not repo:
        sys.exit(0)
    context = json.loads(os.environ.get("SRHT_PUSH_CTX"))
    repo = context["repo"]

    if repo.visibility == RepoVisibility.autocreated:
    if repo["visibility"] == "autocreated":
        print("\n\t\033[93mNOTICE\033[0m")
        print("\tWe saved your changes, but this repository does not exist.")
        print("\tClick here to create it:")
        print("\t{}/create?name={}".format(origin, repo.name))
        print("\t{}/create?name={}".format(origin, repo["name"]))
        print("\tYour changes will be discarded in 20 minutes.\n")

    repo.updated = datetime.utcnow()
    db.session.commit()

    do_post_update(repo, refs)
    do_post_update(context, refs)

M gitsrht/blueprints/internal.py => gitsrht/blueprints/internal.py +18 -5
@@ 3,12 3,13 @@ This blueprint is used internally by gitsrht-shell to speed up git pushes, by
taking advantage of the database connection already established by the web app.
"""

from datetime import datetime
from flask import Blueprint, request
from srht.config import get_origin
from scmsrht.access import has_access, UserAccess
from scmsrht.urls import get_clone_urls
from gitsrht.repos import GitRepoApi
from gitsrht.types import User, Repository, RepoVisibility, Redirect
from scmsrht.access import has_access, UserAccess
from scmsrht.urls import get_clone_urls
from srht.config import get_origin
from srht.crypto import verify_request_signature
from srht.database import db
from srht.flask import csrf_bypass


@@ 31,6 32,18 @@ def push_check():
    access = UserAccess(access)
    user = User.query.filter(User.id == user_id).one()

    def push_context(user, repo):
        if access == UserAccess.write:
            repo.updated = datetime.utcnow()
            db.session.commit()
        return {
            "user": user.to_dict(),
            "repo": {
                "path": repo.path,
                **repo.to_dict(),
            },
        }

    repo = Repository.query.filter(Repository.path == path).first()
    if not repo:
        redir = Redirect.query.filter(Redirect.path == path).first()


@@ 56,11 69,11 @@ def push_check():
                return valid.response
            repo.visibility = RepoVisibility.autocreated
            db.session.commit()
            return { }, 200
            return push_context(user, repo), 200
        else:
            return { }, 404

    if not has_access(repo, access, user):
        return { }, 401

    return { }, 200
    return push_context(user, repo), 200

M gitsrht/submit.py => gitsrht/submit.py +21 -20
@@ 1,30 1,18 @@
import html
import os
import os.path
import re
import requests
import yaml
from pygit2 import Repository as GitRepository, Commit, Tag
from gitsrht.blueprints.api import commit_to_dict
from gitsrht.types import User
from gitsrht.types import User, Repository
from scmsrht.redis import redis
from scmsrht.repos import RepoVisibility
from scmsrht.submit import BuildSubmitterBase
from scmsrht.urls import get_clone_urls
from gitsrht.webhooks import RepoWebhook
from srht.config import cfg, get_origin
from srht.database import db
from srht.oauth import OAuthScope
from urllib.parse import urlparse

if not hasattr(db, "session"):
    import gitsrht.types
    from srht.database import DbSession
    db = DbSession(cfg("git.sr.ht", "connection-string"))
    db.init()

builds_sr_ht = cfg("builds.sr.ht", "origin", None)
builds_client_id = cfg("builds.sr.ht", "oauth-client-id", None)
git_sr_ht = get_origin("git.sr.ht", external=True)

def first_line(text):


@@ 97,18 85,28 @@ class GitBuildSubmitter(BuildSubmitterBase):
# https://stackoverflow.com/a/14693789
ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')

def do_post_update(repo, refs):
def do_post_update(context, refs):
    global db
    # TODO: we shouldn't need this once we move most of this shit to the
    # internal API
    if not hasattr(db, "session"):
        import gitsrht.types
        from srht.database import DbSession
        db = DbSession(cfg("git.sr.ht", "connection-string"))
        db.init()

    uid = os.environ.get("SRHT_UID")
    push = os.environ.get("SRHT_PUSH")
    user = User.query.get(int(uid))
    user = context["user"]
    repo = context["repo"]

    payload = {
        "push": push,
        "pusher": user.to_dict(),
        "pusher": user,
        "refs": list(),
    }

    git_repo = GitRepository(repo.path)
    git_repo = GitRepository(repo["path"])
    oids = set()
    for ref in refs:
        update = redis.get(f"update.{push}.{ref}")


@@ 151,12 149,15 @@ def do_post_update(repo, refs):
            continue

        if builds_sr_ht:
            s = GitBuildSubmitter(repo, git_repo)
            # TODO: move this to internal API
            r = Repository.query.get(repo["id"])
            s = GitBuildSubmitter(r, git_repo)
            s.submit(commit)

    # TODO: get these from internal API
    # sync webhooks
    for resp in RepoWebhook.deliver(RepoWebhook.Events.repo_post_update, payload,
            RepoWebhook.Subscription.repo_id == repo.id,
            RepoWebhook.Subscription.repo_id == repo["id"],
            RepoWebhook.Subscription.sync,
            delay=False):
        if resp == None:


@@ 171,5 172,5 @@ def do_post_update(repo, refs):
            print("Unable to decode webhook response")
    # async webhooks
    RepoWebhook.deliver(RepoWebhook.Events.repo_post_update, payload,
            RepoWebhook.Subscription.repo_id == repo.id,
            RepoWebhook.Subscription.repo_id == repo["id"],
            RepoWebhook.Subscription.sync == False)

M scripts/symlink-update-hook.py => scripts/symlink-update-hook.py +1 -1
@@ 20,5 20,5 @@ def migrate(path, link):

for repo in Repository.query.all():
    if migrate(os.path.join(repo.path, "hooks", "update"), post_update) \
        or migrate(os.path.join(repo.path, "hooks", "post-update"), post_update):
        and migrate(os.path.join(repo.path, "hooks", "post-update"), post_update):
        print("Migrated {}".format(repo.name))