7 files changed, 157 insertions(+), 15 deletions(-)

M gitsrht-keys
M gitsrht-update-hook
A gitsrht/alembic/versions/61b01f874da8_fix_user_webhook_add_repo_webhook.py
M gitsrht/app.py
M gitsrht/blueprints/api.py
M gitsrht/submit.py
M gitsrht/webhooks.py

@@ 8,6 8,7 @@ from srht.database import DbSession
 db = DbSession(cfg("git.sr.ht", "connection-string"))
 from gitsrht.types import User, SSHKey
 db.init()
+from uuid import uuid4
 
 sys.stderr.write(str(sys.argv) + "\n")
 key_type = sys.argv[3]


@@ 37,9 38,11 @@ if not user:
 
 default_shell = os.path.join(os.path.dirname(sys.argv[0]), "gitsrht-shell")
 shell = cfg("git.sr.ht", "shell", default=default_shell)
-keys = "command=\"{} '{}' '{}'\",".format(shell, user.id, b64key) + \
-    "no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty " + \
-    "{} {} {}".format(key_type, b64key, user.username) + "\n"
+keys = ("command=\"{} '{}' '{}'\",".format(shell, user.id, b64key) +
+    "no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty," +
+    "environment=\"SRHT_UID={}\",environment=\"SRHT_PUSH={}\"".format(
+        user.id, str(uuid4())) +
+    " {} {} {}".format(key_type, b64key, user.username) + "\n")
 print(keys)
 sys.stderr.write(keys)
 sys.exit(0)

@@ 5,13 5,27 @@ db = DbSession(cfg("git.sr.ht", "connection-string"))
 from gitsrht.types import Repository, RepoVisibility
 db.init()
 from configparser import ConfigParser
-from datetime import datetime
+from datetime import datetime, timedelta
 from gitsrht.submit import do_post_update
+from scmsrht.redis import redis
+import os
 import sys
 
 op = sys.argv[0]
 origin = cfg("git.sr.ht", "origin")
 
+if op == "hooks/update":
+    # Stash updated refs for later processing
+    refname = sys.argv[1]
+    old = sys.argv[2]
+    new = sys.argv[3]
+
+    push_uuid = os.environ.get("SRHT_PUSH")
+    if not push_uuid:
+        sys.exit(0)
+    redis.setex(f"update.{push_uuid}.{refname}",
+            timedelta(minutes=10), f"{old}:{new}")
+
 if op == "hooks/post-update":
     refs = sys.argv[1:]
 

@@ 0,0 1,48 @@
+"""Fix user webhook, add repo webhook
+
+Revision ID: 61b01f874da8
+Revises: 778f04602534
+Create Date: 2019-04-23 13:30:33.457525
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '61b01f874da8'
+down_revision = '778f04602534'
+
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils as sau
+
+
+def upgrade():
+    op.create_table('user_webhook_subscription',
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("created", sa.DateTime, nullable=False),
+        sa.Column("url", sa.Unicode(2048), nullable=False),
+        sa.Column("events", sa.Unicode, nullable=False),
+        sa.Column("user_id", sa.Integer, sa.ForeignKey("user.id")),
+        sa.Column("token_id", sa.Integer, sa.ForeignKey("oauthtoken.id")),
+    )
+    op.create_table('user_webhook_delivery',
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("uuid", sau.UUIDType, nullable=False),
+        sa.Column("created", sa.DateTime, nullable=False),
+        sa.Column("event", sa.Unicode(256), nullable=False),
+        sa.Column("url", sa.Unicode(2048), nullable=False),
+        sa.Column("payload", sa.Unicode(65536), nullable=False),
+        sa.Column("payload_headers", sa.Unicode(16384), nullable=False),
+        sa.Column("response", sa.Unicode(65536)),
+        sa.Column("response_status", sa.Integer, nullable=False),
+        sa.Column("response_headers", sa.Unicode(16384)),
+        sa.Column("subscription_id", sa.Integer,
+            sa.ForeignKey('user_webhook_subscription.id'), nullable=False),
+    )
+    op.add_column("repo_webhook_subscription",
+            sa.Column("sync", sa.Boolean, nullable=False, server_default="f"))
+
+
+def downgrade():
+    op.drop_table("user_webhook_delivery")
+    op.drop_table("user_webhook_subscription")
+    op.drop_column("repo_webhook_subscription", "sync")

@@ 11,7 11,6 @@ from gitsrht.types import Access, Redirect, Repository, User
 from scmsrht.flask import ScmSrhtFlask
 from srht.config import cfg
 from srht.database import DbSession
-import gitsrht.webhooks # makes valid the global
 
 db = DbSession(cfg("git.sr.ht", "connection-string"))
 db.init()


@@ 21,8 20,7 @@ class GitApp(ScmSrhtFlask):
         super().__init__("git.sr.ht", __name__,
                 access_class=Access, redirect_class=Redirect,
                 repository_class=Repository, user_class=User,
-                repo_api=GitRepoApi(),
-                oauth_service=oauth_service)
+                repo_api=GitRepoApi(), oauth_service=oauth_service)
 
         from gitsrht.blueprints.api import data
         from gitsrht.blueprints.repo import repo

@@ 3,6 3,7 @@ import pygit2
 from flask import Blueprint, current_app, request, send_file, abort
 from gitsrht.blueprints.repo import lookup_ref, get_log, collect_refs
 from gitsrht.git import Repository as GitRepository, commit_time, annotate_tree
+from gitsrht.webhooks import RepoWebhook
 from io import BytesIO
 from scmsrht.blueprints.api import get_user, get_repo
 from srht.api import paginated_response


@@ 10,7 11,7 @@ from srht.oauth import current_token, oauth
 
 data = Blueprint("api.data", __name__)
 
-def _commit_to_dict(c):
+def commit_to_dict(c):
     return {
         "id": str(c.id),
         "short_id": c.short_id,


@@ 32,7 33,7 @@ def _commit_to_dict(c):
         } if c.gpg_signature[0] else None
     }
 
-def _tree_to_dict(t):
+def tree_to_dict(t):
     return {
         "id": str(t.id),
         "short_id": t.short_id,


@@ 100,7 101,7 @@ def repo_commits_GET(username, reponame, ref, path):
             next_id = str(commits[-1].id)
         return {
             "next": next_id,
-            "results": [_commit_to_dict(c) for c in commits],
+            "results": [commit_to_dict(c) for c in commits],
             # TODO: Track total commits per repo per branch
             "total": -1,
             "results_per_page": commits_per_page


@@ 130,7 131,7 @@ def repo_tree_GET(username, reponame, ref, path):
             tree = commit
         else:
             abort(404)
-        return _tree_to_dict(tree)
+        return tree_to_dict(tree)
 
 @data.route("/api/repos/<reponame>/blob/<path:ref>",
         defaults={"username": None, "path": ""})


@@ 175,3 176,19 @@ def repo_blob_GET(username, reponame, ref, path):
                 as_attachment=blob.is_binary,
                 attachment_filename=entry.name if entry else None,
                 mimetype="text/plain" if not blob.is_binary else None)
+
+
+def _webhook_filters(query, username, reponame):
+    user = get_user(username)
+    repo = get_repo(user, reponame)
+    return query.filter(RepoWebhook.Subscription.repo_id == repo.id)
+
+def _webhook_create(sub, valid, username, reponame):
+    user = get_user(username)
+    repo = get_repo(user, reponame)
+    sub.repo_id = repo.id
+    sub.sync = valid.optional("sync", cls=bool, default=False)
+    return sub
+
+RepoWebhook.api_routes(data, "/api/<username>/repos/<reponame>",
+        filters=_webhook_filters, create=_webhook_create)

@@ 6,9 6,13 @@ import requests
 import yaml
 from buildsrht.manifest import Manifest
 from pygit2 import Repository as GitRepository, Commit, Tag
+from gitsrht.blueprints.api import commit_to_dict
+from gitsrht.types import User
+from scmsrht.redis import redis
 from scmsrht.repos import RepoVisibility
 from scmsrht.submit import BuildSubmitterBase
 from scmsrht.urls import get_clone_urls
+from gitsrht.webhooks import RepoWebhook
 from srht.config import cfg, get_origin
 from srht.database import db
 from srht.oauth import OAuthScope


@@ 91,13 95,34 @@ class GitBuildSubmitter(BuildSubmitterBase):
             # Use http(s) URL
             return f"{origin}/{owner_name}/{repo_name}"
 
+# https://stackoverflow.com/a/14693789
+ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
+
 def do_post_update(repo, refs):
-    if not builds_sr_ht:
-        return False
+    uid = os.environ.get("SRHT_UID")
+    push = os.environ.get("SRHT_PUSH")
+    user = User.query.get(int(uid))
+
+    payload = {
+        "push": push,
+        "pusher": user.to_dict(),
+        "refs": list(),
+    }
 
     git_repo = GitRepository(repo.path)
     oids = set()
     for ref in refs:
+        update = redis.get(f"update.{push}.{ref}")
+        if update:
+            old, new = update.decode().split(":")
+            old = git_repo.get(old)
+            new = git_repo.get(new)
+            payload["refs"].append({
+                "name": ref,
+                "old": commit_to_dict(old) if old else None,
+                "new": commit_to_dict(new),
+            })
+
         try:
             if re.match(r"^[0-9a-z]{40}$", ref): # commit
                 commit = git_repo.get(ref)


@@ 119,3 144,23 @@ def do_post_update(repo, refs):
         if builds_sr_ht:
             s = GitBuildSubmitter(repo, git_repo)
             s.submit(commit)
+
+    # sync webhooks
+    for resp in RepoWebhook.deliver(RepoWebhook.Events.repo_post_update, payload,
+            RepoWebhook.Subscription.repo_id == repo.id,
+            RepoWebhook.Subscription.sync,
+            delay=False):
+        if resp == None:
+            # TODO: Add details?
+            print("Error submitting webhook")
+            continue
+        if resp.status_code != 200:
+            print(f"Webhook returned status {resp.status_code}")
+        try:
+            print(ansi_escape.sub('', resp.text))
+        except:
+            print("Unable to decode webhook response")
+    # async webhooks
+    RepoWebhook.deliver(RepoWebhook.Events.repo_post_update, payload,
+            RepoWebhook.Subscription.repo_id == repo.id,
+            RepoWebhook.Subscription.sync == False)

@@ 5,7 5,24 @@ if not hasattr(db, "session"):
     db = DbSession(cfg("git.sr.ht", "connection-string"))
     import gitsrht.types
     db.init()
-from srht.webhook.celery import make_worker
-from scmsrht.webhooks import RepoWebhook
+from srht.webhook import Event
+from srht.webhook.celery import CeleryWebhook, make_worker
+from scmsrht.webhooks import UserWebhook
+import sqlalchemy as sa
 
 worker = make_worker(broker=cfg("git.sr.ht", "webhooks"))
+
+class RepoWebhook(CeleryWebhook):
+    events = [
+        Event("repo:post-update", "data:read"),
+    ]
+
+    sync = sa.Column(sa.Boolean, nullable=False, server_default="f")
+    """
+    If true, this webhook will be run syncronously during a git push, and
+    the response text printed to the console of the pushing user.
+    """
+
+    repo_id = sa.Column(sa.Integer,
+            sa.ForeignKey('repository.id'), nullable=False)
+    repo = sa.orm.relationship('Repository')