From a13786d6cc185822f5940582efde2349ef729145 Mon Sep 17 00:00:00 2001 From: HampusM Date: Thu, 24 Jun 2021 22:50:38 +0200 Subject: Refactored the backend yet again --- packages/server/src/git/blob.ts | 13 ++++ packages/server/src/git/branch.ts | 24 ++++++++ packages/server/src/git/commit.ts | 69 +++++++++++++++++++++ packages/server/src/git/diff.ts | 63 +++++++++++++++++++ packages/server/src/git/http.ts | 84 +++++++++++++++++++++++++ packages/server/src/git/misc.ts | 36 +++++++++++ packages/server/src/git/patch.ts | 113 ++++++++++++++++++++++++++++++++++ packages/server/src/git/reference.ts | 18 ++++++ packages/server/src/git/repository.ts | 105 +++++++++++++++++++++++++++++++ packages/server/src/git/tag.ts | 80 ++++++++++++++++++++++++ packages/server/src/git/tree.ts | 39 ++++++++++++ packages/server/src/git/tree_entry.ts | 40 ++++++++++++ 12 files changed, 684 insertions(+) create mode 100644 packages/server/src/git/blob.ts create mode 100644 packages/server/src/git/branch.ts create mode 100644 packages/server/src/git/commit.ts create mode 100644 packages/server/src/git/diff.ts create mode 100644 packages/server/src/git/http.ts create mode 100644 packages/server/src/git/misc.ts create mode 100644 packages/server/src/git/patch.ts create mode 100644 packages/server/src/git/reference.ts create mode 100644 packages/server/src/git/repository.ts create mode 100644 packages/server/src/git/tag.ts create mode 100644 packages/server/src/git/tree.ts create mode 100644 packages/server/src/git/tree_entry.ts (limited to 'packages/server/src/git') diff --git a/packages/server/src/git/blob.ts b/packages/server/src/git/blob.ts new file mode 100644 index 0000000..aa3f9ad --- /dev/null +++ b/packages/server/src/git/blob.ts @@ -0,0 +1,13 @@ +import { TreeEntry as NodeGitTreeEntry } from "nodegit"; + +export class Blob { + private _ng_tree_entry: NodeGitTreeEntry; + + constructor(entry: NodeGitTreeEntry) { + this._ng_tree_entry = entry; + } + + async content(): Promise { + return this._ng_tree_entry.isBlob() ? (await this._ng_tree_entry.getBlob()).toString() : ""; + } +} \ No newline at end of file diff --git a/packages/server/src/git/branch.ts b/packages/server/src/git/branch.ts new file mode 100644 index 0000000..2142724 --- /dev/null +++ b/packages/server/src/git/branch.ts @@ -0,0 +1,24 @@ +import { CommitSummary } from "./commit"; +import { Reference } from "./reference"; +import { Repository } from "./repository"; + +export class Branch extends Reference { + async latestCommit(): Promise { + const latest_commit = this._owner.nodegitRepository.getBranchCommit(this._ng_reference); + return { + id: (await latest_commit).sha(), + message: (await latest_commit).message(), + date: (await latest_commit).time() + }; + } + + static async lookup(owner: Repository, branch: string): Promise { + const reference = await owner.nodegitRepository.getBranch(branch).catch(err => { + if(err.errno === -3) { + return null; + } + throw(err); + }); + return reference ? new Branch(owner, reference) : null; + } +} \ No newline at end of file diff --git a/packages/server/src/git/commit.ts b/packages/server/src/git/commit.ts new file mode 100644 index 0000000..64bae4d --- /dev/null +++ b/packages/server/src/git/commit.ts @@ -0,0 +1,69 @@ +import { Commit as NodeGitCommit, Oid as NodeGitOid } from "nodegit"; +import { Author } from "./misc"; +import { Diff } from "./diff"; +import { Repository } from "./repository"; +import { Tree } from "./tree"; + +export type CommitSummary = { + id: string | null, + message: string | null, + date: number | null +} + +type DiffStats = { + insertions: number, + deletions: number, + files_changed: number +} + +export class Commit { + private _ng_commit: NodeGitCommit; + private _owner: Repository; + + public id: string; + public author: Author; + public date: number; + public message: string; + + constructor(owner: Repository, commit: NodeGitCommit) { + this._ng_commit = commit; + this._owner = owner; + + this.id = commit.sha(); + this.author = { + name: commit.author().name(), + email: commit.author().email() + }; + this.date = commit.time(); + this.message = commit.message(); + } + + async diff(): Promise { + return Diff.get((await this._ng_commit.getDiff())[0]); + } + + async stats(): Promise { + const stats = await (await this._ng_commit.getDiff())[0].getStats(); + + return { + insertions: stats.insertions(), + deletions: stats.deletions(), + files_changed: stats.filesChanged() + }; + } + + async tree(): Promise { + return new Tree(this._owner, await this._ng_commit.getTree()); + } + + static async lookup(repository: Repository, id: string | NodeGitOid): Promise { + const commit = await NodeGitCommit.lookup(repository.nodegitRepository, id instanceof NodeGitOid ? id : NodeGitOid.fromString(id)); + return new Commit(repository, commit); + } + + static lookupExists(repository: Repository, id: string): Promise { + return NodeGitCommit.lookup(repository.nodegitRepository, NodeGitOid.fromString(id)) + .then(() => true) + .catch(() => false); + } +} \ No newline at end of file diff --git a/packages/server/src/git/diff.ts b/packages/server/src/git/diff.ts new file mode 100644 index 0000000..a3ea375 --- /dev/null +++ b/packages/server/src/git/diff.ts @@ -0,0 +1,63 @@ +import { Diff as NodeGitDiff } from "nodegit"; +import { Patch } from "./patch"; + +type PatchHeaderData = { + indexes: number[], + lengths: number[], + last: number | null +} + +type DiffConstructorData = { + patch_buf: string, + patch_header_buf: string +} + +export class Diff { + private _ng_diff: NodeGitDiff; + + public raw_patches: string; + public patch_header_indexes: number[]; + public patch_header_lengths: number[]; + + constructor(diff: NodeGitDiff, data: DiffConstructorData) { + this._ng_diff = diff; + this.raw_patches = data.patch_buf; + + const raw_patches_arr = this.raw_patches.split("\n"); + const patch_headers = data.patch_header_buf.split("\n"); + + const patch_header_data = patch_headers.reduce((result, line, index) => { + // The start of a patch header + if((/^diff --git/u).test(line)) { + result.indexes.push(raw_patches_arr.indexOf(line)); + + if(result.last !== null) { + result.lengths.push(patch_headers.slice(result.last, index).length); + } + result.last = index; + } + + // Include the last patch header when the end is reached + if(index === patch_headers.length - 1 && result.last !== null) { + result.lengths.push(patch_headers.slice(result.last, index).length); + } + + return result; + }, { indexes: [], lengths: [], last: null }); + + this.patch_header_indexes = patch_header_data.indexes; + this.patch_header_lengths = patch_header_data.lengths; + + } + + async getPatches(): Promise { + return (await this._ng_diff.patches()).map((patch, index) => new Patch(this, patch, index)); + } + + static async get(diff: NodeGitDiff): Promise { + return new Diff(diff, { + patch_buf: String((await diff.toBuf(1))), + patch_header_buf: String((await diff.toBuf(2))) + }); + } +} \ No newline at end of file diff --git a/packages/server/src/git/http.ts b/packages/server/src/git/http.ts new file mode 100644 index 0000000..2d707cb --- /dev/null +++ b/packages/server/src/git/http.ts @@ -0,0 +1,84 @@ +import { FastifyReply, FastifyRequest } from "fastify"; +import { IncomingMessage } from "http"; +import { Repository } from "./repository"; +import { Route } from "../fastify_types"; +import { join } from "path"; +import { spawn } from "child_process"; +import { verifyGitRequest } from "../api/util"; + +export type RequestInfo = { + repo: string, + url_path: string, + parsed_url: URL, + url_path_parts: string[], + is_discovery: boolean, + service: string | null, + content_type: string +} + +export interface Request extends FastifyRequest { + params: Route["Params"], +} + +function getRequestInfo(req: Request): RequestInfo { + const repo = req.params.repo + ".git"; + const url_path = req.url.replace(req.params.repo, repo); + + const parsed_url = new URL(`${req.protocol}://${req.hostname}${url_path}`); + const url_path_parts = parsed_url.pathname.split("/"); + + const is_discovery = (/\/info\/refs$/u).test(parsed_url.pathname); + + const service = is_discovery ? parsed_url.searchParams.get("service") : url_path_parts[url_path_parts.length - 1]; + + const content_type = `application/x-${service}-${is_discovery ? "advertisement" : "result"}`; + + return { + repo, + url_path, + parsed_url, + is_discovery, + url_path_parts, + service, + content_type + }; +} + +export function connect(repository: Repository, req: Request, reply: FastifyReply): void { + const request_info = getRequestInfo(req); + + const valid_request = verifyGitRequest(request_info); + if(valid_request.success === false && valid_request.code) { + reply.header("Content-Type", request_info.content_type); + reply.code(valid_request.code).send(valid_request.message); + return; + } + + reply.raw.writeHead(200, { "Content-Type": request_info.content_type }); + + const spawn_args = [ "--stateless-rpc", join(repository.base_dir, request_info.repo) ]; + if(request_info.is_discovery) { + spawn_args.push("--advertise-refs"); + } + + const git_pack = spawn(request_info.service, spawn_args); + + if(request_info.is_discovery) { + const s = "# service=" + request_info.service + "\n"; + const n = (4 + s.length).toString(16); + reply.raw.write(Buffer.from((Array(4 - n.length + 1).join("0") + n + s) + "0000")); + } + else { + const request_body: IncomingMessage = req.raw; + + request_body.on("data", data => git_pack.stdin.write(data)); + request_body.on("close", () => git_pack.stdin.end()); + } + + git_pack.on("error", err => console.log(err)); + + git_pack.stderr.on("data", (stderr: Buffer) => console.log(stderr.toString())); + git_pack.stdout.on("data", data => reply.raw.write(data)); + + git_pack.on("close", () => reply.raw.end()); +} \ No newline at end of file diff --git a/packages/server/src/git/misc.ts b/packages/server/src/git/misc.ts new file mode 100644 index 0000000..fcfaf29 --- /dev/null +++ b/packages/server/src/git/misc.ts @@ -0,0 +1,36 @@ +import { readFile, readdir } from "fs"; + +export async function findAsync(arr: T[], callback: (t: T) => Promise): Promise { + const results = await Promise.all(arr.map(callback)); + const index = results.findIndex(result => result); + return arr[index]; +} + +export type Author = { + name: string, + email: string +} + +export function getFile(base_dir: string, repository: string, file: string): Promise { + return new Promise(resolve => { + readFile(`${base_dir}/${repository}/${file}`, (err, content) => { + if(err) { + resolve(null); + return; + } + resolve(content.toString().replace(/\n/gu, "")); + }); + }); +} + +export function getDirectory(directory: string): Promise { + return new Promise(resolve => { + readdir(directory, (err, dir_content) => { + if(err) { + resolve([]); + } + + resolve(dir_content); + }); + }); +} \ No newline at end of file diff --git a/packages/server/src/git/patch.ts b/packages/server/src/git/patch.ts new file mode 100644 index 0000000..71d9193 --- /dev/null +++ b/packages/server/src/git/patch.ts @@ -0,0 +1,113 @@ +import { Diff } from "./diff"; +import { ConvenientPatch as NodeGitPatch } from "nodegit"; + +type Hunk = { + new_start: number, + new_lines_cnt: number, + old_start: number, + old_lines_cnt: number, + new_lines: number[], + deleted_lines: number[], + hunk: string +} + +type Hunks = { + prev: null | number, + hunks: Hunk[] +} + +function getHunkContent(hunk: string[]) { + interface Lines { + new_lines: number[], + deleted_lines: number[] + } + + const lines = hunk.reduce((result: Lines, line, index) => { + if(line.charAt(0) === "+") { + hunk[index] = line.slice(1); + result.new_lines.push(index); + } + else if(line.charAt(0) === "-") { + hunk[index] = line.slice(1); + result.deleted_lines.push(index); + } + return result; + }, { new_lines: [], deleted_lines: [] }); + + return { ...lines, hunk: hunk.join("\n") }; +} + +export class Patch { + private _ng_patch: NodeGitPatch; + + public from: string; + public to: string; + public additions: number; + public deletions: number; + public too_large = false; + public content: string | null = null; + + constructor(diff: Diff, patch: NodeGitPatch, index: number) { + this._ng_patch = patch; + + this.from = patch.oldFile().path(); + this.to = patch.newFile().path(); + this.additions = patch.lineStats()["total_additions"]; + this.deletions = patch.lineStats()["total_deletions"]; + + const raw_patches_arr = diff.raw_patches.split("\n"); + const start = diff.patch_header_indexes[index] + diff.patch_header_lengths[index]; + const end = (typeof diff.patch_header_indexes[index + 1] === "undefined") ? raw_patches_arr.length - 1 : diff.patch_header_indexes[index + 1]; + + const patch_content = raw_patches_arr.slice(start, end); + + if(patch_content.length !== 0) { + this.content = patch_content.join("\n"); + + const line_lengths = patch_content.map(line => line.length).reduce((result, length) => result + length); + + if(patch_content.length > 5000 || line_lengths > 5000) { + this.too_large = true; + } + } + } + + async getHunks(): Promise { + if(!this.content) { + return null; + } + + const content = this.content.split("\n"); + const hunks = await this._ng_patch.hunks(); + + const hunks_data = hunks.reduce((result: Hunks, hunk, hunk_index) => { + const hunk_header = hunk.header(); + const hunk_header_index = content.indexOf(hunk_header.replace(/\n/gu, "")); + + if(result.prev !== null) { + const prev_hunk = hunks[hunk_index - 1]; + result.hunks.push({ + new_start: prev_hunk.newStart(), + new_lines_cnt: prev_hunk.newLines(), + old_start: prev_hunk.oldStart(), + old_lines_cnt: prev_hunk.oldLines(), + ...getHunkContent(content.slice(result.prev, hunk_header_index)) + }); + } + + result.prev = hunk_header_index; + return result; + }, { prev: null, hunks: [] }); + + const prev_hunk = hunks[hunks.length - 1]; + hunks_data.hunks.push({ + new_start: prev_hunk.newStart(), + new_lines_cnt: prev_hunk.newLines(), + old_start: prev_hunk.oldStart(), + old_lines_cnt: prev_hunk.oldLines(), + ...getHunkContent(content.slice(hunks_data.prev)) + }); + + return hunks_data.hunks; + } +} \ No newline at end of file diff --git a/packages/server/src/git/reference.ts b/packages/server/src/git/reference.ts new file mode 100644 index 0000000..910fa7d --- /dev/null +++ b/packages/server/src/git/reference.ts @@ -0,0 +1,18 @@ +import { Reference as NodeGitReference } from "nodegit"; +import { Repository } from "./repository"; + +export abstract class Reference { + protected _ng_reference: NodeGitReference; + protected _owner: Repository; + + id: string; + name: string; + + constructor(owner: Repository, reference: NodeGitReference) { + this._ng_reference = reference; + this._owner = owner; + + this.id = reference.target().tostrS(); + this.name = reference.shorthand(); + } +} \ No newline at end of file diff --git a/packages/server/src/git/repository.ts b/packages/server/src/git/repository.ts new file mode 100644 index 0000000..ac0927a --- /dev/null +++ b/packages/server/src/git/repository.ts @@ -0,0 +1,105 @@ +import { Object as NodeGitObject, Oid as NodeGitOid, Repository as NodeGitRepository, Revwalk as NodeGitRevwalk } from "nodegit"; +import { Request, connect } from "./http"; +import { basename, dirname } from "path"; +import { getDirectory, getFile } from "./misc"; +import { Branch } from "./branch"; +import { Commit } from "./commit"; +import { FastifyReply } from "fastify"; +import { Tag } from "./tag"; +import { Tree } from "./tree"; + +function getFullRepositoryName(repo_name: string) { + return repo_name.endsWith(".git") ? repo_name : `${repo_name}.git`; +} + +type RepositoryName = { + short: string, + full: string, +} + +type RepositoryConstructorData = { + description: string | null, + owner: string | null +} + +export class Repository { + private _ng_repository: NodeGitRepository; + + public name: RepositoryName; + public base_dir: string; + public description: string | null; + public owner: string | null; + + constructor(repository: NodeGitRepository, data: RepositoryConstructorData) { + this._ng_repository = repository; + this.name = { + short: basename(repository.path()).slice(0, -4), + full: basename(repository.path()) + }; + this.base_dir = dirname(repository.path()); + this.description = data.description; + this.owner = data.owner; + } + + async commits(): Promise { + const walker = NodeGitRevwalk.create(this._ng_repository); + walker.pushHead(); + + return Promise.all((await walker.getCommitsUntil(() => true)).map(commit => new Commit(this, commit))); + } + + async tree(): Promise { + const master_commit = await this._ng_repository.getMasterCommit(); + const tree = await master_commit.getTree(); + return new Tree(this, tree); + } + + lookupExists(id: string): Promise { + return NodeGitObject.lookup(this._ng_repository, NodeGitOid.fromString(id), NodeGitObject.TYPE.ANY) + .then(() => true) + .catch(() => false); + } + + async branches(): Promise { + const references = await this._ng_repository.getReferences(); + return references.filter(ref => ref.isBranch()).map(branch => new Branch(this, branch)); + } + + async tags(): Promise { + const references = await this._ng_repository.getReferences(); + return references.filter(ref => ref.isTag()).map(tag => new Tag(this, tag)); + } + + async latestCommit(): Promise { + return new Commit(this, await this._ng_repository.getMasterCommit()); + } + + HTTPconnect(req: Request, reply: FastifyReply): void { + connect(this, req, reply); + } + + get nodegitRepository(): NodeGitRepository { + return this._ng_repository; + } + + static async open(base_dir: string, repository: string): Promise { + const ng_repository = await NodeGitRepository.openBare(`${base_dir}/${getFullRepositoryName(repository)}`); + + return new Repository(ng_repository, { + description: await getFile(base_dir, getFullRepositoryName(repository), "description"), + owner: await getFile(base_dir, getFullRepositoryName(repository), "owner") + }); + } + + static async openAll(base_dir: string): Promise { + const dir_content = await getDirectory(base_dir); + + if(dir_content.length === 0) { + return null; + } + + const repositories = dir_content.filter(dir_entry => dir_entry.endsWith(".git")); + + return Promise.all(repositories.map(repository => this.open(base_dir, repository))); + } +} \ No newline at end of file diff --git a/packages/server/src/git/tag.ts b/packages/server/src/git/tag.ts new file mode 100644 index 0000000..ffda9c4 --- /dev/null +++ b/packages/server/src/git/tag.ts @@ -0,0 +1,80 @@ +import { Object as NodeGitObject, Tag as NodeGitTag } from "nodegit"; +import { Pack, pack } from "tar-stream"; +import { Author } from "./misc"; +import { Blob } from "./blob"; +import { Commit } from "./commit"; +import { FastifyReply } from "fastify"; +import { Reference } from "./reference"; +import { Repository } from "./repository"; +import { Tree } from "./tree"; +import { TreeEntry } from "./tree_entry"; +import { createGzip } from "zlib"; +import { pipeline } from "stream"; + +async function addArchiveEntries(entries: TreeEntry[], repository: string, archive: Pack) { + for(const tree_entry of entries) { + const peeled = (await tree_entry.peel()); + + if(tree_entry.type === "blob") { + if(peeled instanceof Blob) { + archive.entry({ name: `${repository}/${tree_entry.path}` }, await peeled.content()); + } + } + else if(peeled instanceof Tree) { + addArchiveEntries(peeled.entries(), repository, archive); + } + } +} + +export class Tag extends Reference { + async author(): Promise { + const tagger = (await NodeGitTag.lookup(this._owner.nodegitRepository, this._ng_reference.target())).tagger(); + return { + name: tagger.name(), + email: tagger.email() + }; + } + + async date(): Promise { + return (await NodeGitTag.lookup(this._owner.nodegitRepository, this._ng_reference.target())).tagger().when() + .time(); + } + + async downloadTarball(reply: FastifyReply): Promise { + const commit = await Commit.lookup(this._owner, (await this._ng_reference.peel(NodeGitObject.TYPE.COMMIT)).id()); + const tree = await commit.tree(); + + const archive = pack(); + const gzip = createGzip(); + + reply.raw.writeHead(200, { + "Content-Encoding": "gzip", + "Content-Type": "application/gzip", + "Content-Disposition": `attachment; filename="${this._owner.name.short}-${this._owner.name.short}.tar.gz"` + }); + + pipeline(archive, gzip, reply.raw, () => reply.raw.end()); + + gzip.on("close", () => reply.raw.end()); + gzip.on("error", () => reply.raw.end()); + archive.on("error", () => reply.raw.end()); + + addArchiveEntries(await tree.entries(), this._owner.name.short, archive) + .then(() => archive.finalize()) + .catch(() => { + archive.finalize(); + reply.raw.end(); + }); + } + + static async lookup(owner: Repository, tag: string): Promise { + const reference = await owner.nodegitRepository.getReference(tag).catch(err => { + if(err.errno === -3) { + return null; + } + + throw(err); + }); + return reference ? new Tag(owner, reference) : null; + } +} \ No newline at end of file diff --git a/packages/server/src/git/tree.ts b/packages/server/src/git/tree.ts new file mode 100644 index 0000000..b9bd142 --- /dev/null +++ b/packages/server/src/git/tree.ts @@ -0,0 +1,39 @@ +import { Blob } from "./blob"; +import { Tree as NodeGitTree } from "nodegit"; +import { Repository } from "./repository"; +import { TreeEntry } from "./tree_entry"; + +export class Tree { + private _ng_tree: NodeGitTree; + private _owner: Repository; + + constructor(owner: Repository, tree: NodeGitTree) { + this._ng_tree = tree; + this._owner = owner; + } + + entries(): TreeEntry[] { + return this._ng_tree.entries().map(entry => new TreeEntry(this._owner, entry)); + } + + async find(path: string): Promise { + const entry = await this._ng_tree.getEntry(path).catch(err => { + if(err.errno === -3) { + return null; + } + throw(err); + }); + + if(!entry) { + return null; + } + + return entry.isBlob() ? new Blob(entry) : new Tree(this._owner, await entry.getTree()); + } + + findExists(path: string): Promise { + return this._ng_tree.getEntry(path) + .then(() => true) + .catch(() => false); + } +} \ No newline at end of file diff --git a/packages/server/src/git/tree_entry.ts b/packages/server/src/git/tree_entry.ts new file mode 100644 index 0000000..3bcf10e --- /dev/null +++ b/packages/server/src/git/tree_entry.ts @@ -0,0 +1,40 @@ +import { Blob } from "./blob"; +import { Commit } from "./commit"; +import { TreeEntry as NodeGitTreeEntry } from "nodegit"; +import { Repository } from "./repository"; +import { Tree } from "./tree"; +import { dirname } from "path"; +import { findAsync } from "./misc"; + +export class TreeEntry { + private _ng_tree_entry: NodeGitTreeEntry; + private _owner: Repository; + + public path: string; + public type: "blob" | "tree"; + + constructor(owner: Repository, entry: NodeGitTreeEntry) { + this._ng_tree_entry = entry; + this._owner = owner; + + this.path = entry.path(); + this.type = entry.isBlob() ? "blob" : "tree"; + } + + async latestCommit(): Promise { + const commits = await this._owner.commits(); + + return findAsync(commits, async commit => { + const diff = await commit.diff(); + const patches = await diff.getPatches(); + + return Boolean(this.type === "blob" + ? patches.find(patch => patch.to === this.path) + : patches.find(patch => dirname(patch.to).startsWith(this.path))); + }); + } + + async peel(): Promise { + return this.type === "blob" ? new Blob(this._ng_tree_entry) : new Tree(this._owner, await this._ng_tree_entry.getTree()); + } +} \ No newline at end of file -- cgit v1.2.3-18-g5258