From a3fe57b6844709eac64fdca60370e5c227979fd3 Mon Sep 17 00:00:00 2001 From: Drew DeVault Date: Mon, 30 Mar 2020 16:10:15 -0400 Subject: [PATCH] Add plumbing API and pygit2 backend implementation This new API exposes the raw odb and refdb data and the provided pygit2 backend implementation allows users to create pygit2 repositories which are backed by the git.sr.ht API over the network, rather than by local storage. --- gitsrht/app.py | 5 +- gitsrht/blueprints/api/__init__.py | 2 + gitsrht/blueprints/api/plumbing.py | 70 ++++++++ .../blueprints/{api.py => api/porcelain.py} | 52 +++--- gitsrht/pygit2_backend.py | 165 ++++++++++++++++++ 5 files changed, 266 insertions(+), 28 deletions(-) create mode 100644 gitsrht/blueprints/api/__init__.py create mode 100644 gitsrht/blueprints/api/plumbing.py rename gitsrht/blueprints/{api.py => api/porcelain.py} (84%) create mode 100644 gitsrht/pygit2_backend.py diff --git a/gitsrht/app.py b/gitsrht/app.py index 451a539..a37f453 100644 --- a/gitsrht/app.py +++ b/gitsrht/app.py @@ -22,13 +22,14 @@ class GitApp(ScmSrhtFlask): repository_class=Repository, user_class=User, repo_api=GitRepoApi(), oauth_service=oauth_service) - from gitsrht.blueprints.api import data + from gitsrht.blueprints.api import plumbing, porcelain from gitsrht.blueprints.artifacts import artifacts from gitsrht.blueprints.email import mail from gitsrht.blueprints.repo import repo from gitsrht.blueprints.stats import stats - self.register_blueprint(data) + self.register_blueprint(plumbing) + self.register_blueprint(porcelain) self.register_blueprint(mail) self.register_blueprint(repo) self.register_blueprint(stats) diff --git a/gitsrht/blueprints/api/__init__.py b/gitsrht/blueprints/api/__init__.py new file mode 100644 index 0000000..9c514de --- /dev/null +++ b/gitsrht/blueprints/api/__init__.py @@ -0,0 +1,2 @@ +from gitsrht.blueprints.api.plumbing import plumbing +from gitsrht.blueprints.api.porcelain import porcelain diff --git a/gitsrht/blueprints/api/plumbing.py b/gitsrht/blueprints/api/plumbing.py new file mode 100644 index 0000000..d65a5fe --- /dev/null +++ b/gitsrht/blueprints/api/plumbing.py @@ -0,0 +1,70 @@ +import base64 +import binascii +import pygit2 +from flask import Blueprint, Response, abort, request +from gitsrht.git import Repository as GitRepository +from scmsrht.blueprints.api import get_user, get_repo +from srht.oauth import oauth + +plumbing = Blueprint("api.plumbing", __name__) + +def libgit2_object_type_to_str(otype): + return { + pygit2.GIT_OBJ_COMMIT: "commit", + pygit2.GIT_OBJ_TREE: "tree", + pygit2.GIT_OBJ_BLOB: "blob", + pygit2.GIT_OBJ_TAG: "tag", + }[otype] + +@plumbing.route("/api/repos//odb/", defaults={"username": None}) +@plumbing.route("/api//repos//odb/") +@oauth("data:read") +def repo_get_object(username, reponame, oid): + user = get_user(username) + repo = get_repo(user, reponame) + with GitRepository(repo.path) as git_repo: + try: + otype, odata = git_repo.odb.read(oid) + except KeyError: + return "object not found", 404 + return Response(odata, headers={ + "X-Git-Object-Type": libgit2_object_type_to_str(otype), + }, content_type="application/octet-stream") + +@plumbing.route("/api/repos//lookup/", + defaults={"username": None}) +@plumbing.route("/api//repos//lookup/") +@oauth("data:read") +def repo_lookup_prefix(username, reponame, oid_prefix): + user = get_user(username) + repo = get_repo(user, reponame) + with GitRepository(repo.path) as git_repo: + # XXX: This will look up anything, not just a partially qualified Oid + try: + o = git_repo.revparse_single(oid_prefix) + except KeyError: + return "object not found", 404 + except ValueError: + return "ambiguous oid", 409 + return o.oid.hex + +@plumbing.route("/api/repos//refdb/", + defaults={"username": None}) +@plumbing.route("/api//repos//refdb/") +@oauth("data:read") +def repo_get_ref(username, reponame, refname): + user = get_user(username) + repo = get_repo(user, reponame) + with GitRepository(repo.path) as git_repo: + try: + ref = git_repo.lookup_reference(refname) + except pygit2.InvalidSpecError: + return "invalid reference", 400 + except KeyError: + return "unknown reference", 404 + if isinstance(ref.target, pygit2.Oid): + # direct reference + return f"{ref.target.hex} {ref.peel().oid.hex}" + else: + # symbolic reference + return str(ref.target) diff --git a/gitsrht/blueprints/api.py b/gitsrht/blueprints/api/porcelain.py similarity index 84% rename from gitsrht/blueprints/api.py rename to gitsrht/blueprints/api/porcelain.py index 9eea653..df9758c 100644 --- a/gitsrht/blueprints/api.py +++ b/gitsrht/blueprints/api/porcelain.py @@ -19,7 +19,7 @@ from srht.oauth import current_token, oauth from srht.redis import redis from srht.validation import Validation -data = Blueprint("api.data", __name__) +porcelain = Blueprint("api.porcelain", __name__) # See also gitsrht-update-hook/types.go def commit_to_dict(c): @@ -66,8 +66,8 @@ def ref_to_dict(artifacts, ref): "artifacts": [a.to_dict() for a in artifacts.get(target, [])], } -@data.route("/api/repos//refs", defaults={"username": None}) -@data.route("/api//repos//refs") +@porcelain.route("/api/repos//refs", defaults={"username": None}) +@porcelain.route("/api//repos//refs") @oauth("data:read") def repo_refs_GET(username, reponame): user = get_user(username) @@ -94,8 +94,8 @@ def repo_refs_GET(username, reponame): "results_per_page": len(refs), } -@data.route("/api/repos//artifacts/", defaults={"username": None}, methods=["POST"]) -@data.route("/api//repos//artifacts/", methods=["POST"]) +@porcelain.route("/api/repos//artifacts/", defaults={"username": None}, methods=["POST"]) +@porcelain.route("/api//repos//artifacts/", methods=["POST"]) @oauth("data:write") def repo_refs_by_name_POST(username, reponame, refname): user = get_user(username) @@ -124,17 +124,17 @@ def repo_refs_by_name_POST(username, reponame, refname): return artifact.to_dict() # dear god, this routing -@data.route("/api/repos//log", +@porcelain.route("/api/repos//log", defaults={"username": None, "ref": None, "path": ""}) -@data.route("/api/repos//log/", +@porcelain.route("/api/repos//log/", defaults={"username": None, "path": ""}) -@data.route("/api/repos//log//", +@porcelain.route("/api/repos//log//", defaults={"username": None}) -@data.route("/api//repos//log", +@porcelain.route("/api//repos//log", defaults={"ref": None, "path": ""}) -@data.route("/api//repos//log/", +@porcelain.route("/api//repos//log/", defaults={"path": ""}) -@data.route("/api//repos//log//") +@porcelain.route("/api//repos//log//") @oauth("data:read") def repo_commits_GET(username, reponame, ref, path): user = get_user(username) @@ -161,17 +161,17 @@ def repo_commits_GET(username, reponame, ref, path): "results_per_page": commits_per_page } -@data.route("/api/repos//tree", +@porcelain.route("/api/repos//tree", defaults={"username": None, "ref": None, "path": ""}) -@data.route("/api/repos//tree/", +@porcelain.route("/api/repos//tree/", defaults={"username": None, "path": ""}) -@data.route("/api/repos//tree//", +@porcelain.route("/api/repos//tree//", defaults={"username": None}) -@data.route("/api//repos//tree", +@porcelain.route("/api//repos//tree", defaults={"ref": None, "path": ""}) -@data.route("/api//repos//tree/", +@porcelain.route("/api//repos//tree/", defaults={"path": ""}) -@data.route("/api//repos//tree//") +@porcelain.route("/api//repos//tree//") @oauth("data:read") def repo_tree_GET(username, reponame, ref, path): user = get_user(username) @@ -201,13 +201,13 @@ def repo_tree_GET(username, reponame, ref, path): return tree_to_dict(tree) # TODO: remove fallback routes -@data.route("/api/repos//annotate", methods=["PUT"], +@porcelain.route("/api/repos//annotate", methods=["PUT"], defaults={"username": None, "commit": "master"}) -@data.route("/api//repos//annotate", methods=["PUT"], +@porcelain.route("/api//repos//annotate", methods=["PUT"], defaults={"commit": "master"}) -@data.route("/api/repos///annotate", methods=["PUT"], +@porcelain.route("/api/repos///annotate", methods=["PUT"], defaults={"username": None}) -@data.route("/api//repos///annotate", methods=["PUT"]) +@porcelain.route("/api//repos///annotate", methods=["PUT"]) @oauth("repo:write") def repo_annotate_PUT(username, reponame, commit): user = get_user(username) @@ -245,13 +245,13 @@ def repo_annotate_PUT(username, reponame, commit): return { "updated": nblobs }, 200 -@data.route("/api/repos//blob/", +@porcelain.route("/api/repos//blob/", defaults={"username": None, "path": ""}) -@data.route("/api/repos//blob//", +@porcelain.route("/api/repos//blob//", defaults={"username": None}) -@data.route("/api//blob//blob/", +@porcelain.route("/api//blob//blob/", defaults={"path": ""}) -@data.route("/api//repos//blob//") +@porcelain.route("/api//repos//blob//") @oauth("data:read") def repo_blob_GET(username, reponame, ref, path): user = get_user(username) @@ -317,5 +317,5 @@ def _webhook_create(sub, valid, username, reponame): sub.sync = valid.optional("sync", cls=bool, default=False) return sub -RepoWebhook.api_routes(data, "/api//repos/", +RepoWebhook.api_routes(porcelain, "/api//repos/", filters=_webhook_filters, create=_webhook_create) diff --git a/gitsrht/pygit2_backend.py b/gitsrht/pygit2_backend.py new file mode 100644 index 0000000..994d1a6 --- /dev/null +++ b/gitsrht/pygit2_backend.py @@ -0,0 +1,165 @@ +import pygit2 +import requests +from srht.config import get_origin + +def str_to_libgit2_object_type(otype): + return { + "commit": pygit2.GIT_OBJ_COMMIT, + "tree": pygit2.GIT_OBJ_TREE, + "blob": pygit2.GIT_OBJ_BLOB, + "tag": pygit2.GIT_OBJ_TAG, + }[otype] + +_gitsrht = get_origin("git.sr.ht") + +class OdbBackend(pygit2.OdbBackend): + def __init__(self, authorization, owner_name, repo_name, + upstream=_gitsrht, session=None): + super().__init__() + self.base_url = f"{upstream}/api/{owner_name}/repos/{repo_name}" + self.authorization = authorization + if session == None: + self.session = requests.Session() + else: + self.session = session + + def _get(self, path, *args, **kwargs): + headers = kwargs.pop("headers", dict()) + return self.session.get(f"{self.base_url}{path}", + headers={**self.authorization, **headers}) + + def _head(self, path, *args, **kwargs): + headers = kwargs.pop("headers", dict()) + return self.session.head(f"{self.base_url}{path}", + headers={**self.authorization, **headers}) + + def exists(self, oid): + r = self._head(f"/lookup/{str(oid)}") + return r.status_code != 404 + + def exists_prefix(self, oid_prefix): + r = self._get(f"/lookup/{str(oid_prefix)}") + if r.status_code == 404: + raise KeyError(r.text) + elif r.status_code == 409: + raise ValueError(r.text) + return r.text + + def read(self, oid): + r = self._get(f"/odb/{str(oid)}") + if r.status_code == 404: + raise KeyError(r.text) + elif r.status_code == 409: + raise ValueError(r.text) + otype = r.headers["X-Git-Object-Type"] + otype = str_to_libgit2_object_type(otype) + return otype, r.content + + def read_header(self, oid): + r = self._head(f"/odb/{str(oid)}") + if r.status_code == 404: + raise KeyError(r.text) + elif r.status_code == 409: + raise ValueError(r.text) + otype = r.headers["X-Git-Object-Type"] + otype = str_to_libgit2_object_type(otype) + length = int(r.headers["Content-Length"]) + return otype, length + + def read_prefix(self, oid_prefix): + oid = self.exists_prefix(oid_prefix) + return (oid, *self.read(oid)) + + def __iter__(self): + raise NotImplementedError() + + def refresh(self): + pass # no-op + +class RefdbBackend(pygit2.RefdbBackend): + def __init__(self, authorization, owner_name, repo_name, + upstream=_gitsrht, session=None): + super().__init__() + self.base_url = f"{upstream}/api/{owner_name}/repos/{repo_name}" + self.authorization = authorization + if session == None: + self.session = requests.Session() + else: + self.session = session + + def _get(self, path, *args, **kwargs): + headers = kwargs.pop("headers", dict()) + return requests.get(f"{self.base_url}{path}", + headers={**self.authorization, **headers}) + + def _head(self, path, *args, **kwargs): + headers = kwargs.pop("headers", dict()) + return requests.head(f"{self.base_url}{path}", + headers={**self.authorization, **headers}) + + def exists(self, ref): + r = self._head(f"/refdb/{ref}") + if r.status_code == 404: + return False + elif r.status_code == 200: + return True + else: + raise Exception(r.text) + + def lookup(self, ref): + r = self._get(f"/refdb/{ref}") + if r.status_code == 404: + raise KeyError(r.text) + elif r.status_code != 200: + raise Exception(r.text) + if " " in r.text: + target, peel = r.text.split(" ", 1) + return pygit2.Reference(ref, target, peel) + else: + return pygit2.Reference(ref, r.text) + + def write(self, ref, force, who, message, old, old_target): + raise NotImplementedError() + + def rename(self, old_name, new_name, force, who, message): + raise NotImplementedError() + + def delete(self, ref_name, old_id, old_target): + raise NotImplementedError() + + def has_log(self, ref_name): + raise NotImplementedError() + + def ensure_log(self, ref_name): + raise NotImplementedError() + + def __iter__(self): + raise NotImplementedError() + + def __next__(self): + raise NotImplementedError() + +class GitSrhtRepository(pygit2.Repository): + """ + A pygit2.Repository which is backed by the git.sr.ht API rather than by + local storage. + """ + def __init__(self, authorization, owner_name, repo_name, upstream=_gitsrht): + """ + authorization: a dictionary of headers providing API authorization + (e.g. from srht.api.get_authorization) + + owner_name: the canonical name of the repository owner + """ + super().__init__() + self.session = requests.Session() + odb = pygit2.Odb() + odb_backend = OdbBackend(authorization, owner_name, repo_name, + upstream=upstream, session=self.session) + odb.add_backend(odb_backend, 1) + refdb = pygit2.Refdb.new(self) + refdb_backend = RefdbBackend(authorization, owner_name, repo_name, + upstream=upstream, session=self.session) + refdb.set_backend(refdb_backend) + self.set_odb(odb) + self.set_refdb(refdb) -- 2.38.4