From d1a1b7dc947063aef5f8375a6a1e03246b272c84 Mon Sep 17 00:00:00 2001 From: HampusM Date: Wed, 18 Aug 2021 17:29:55 +0200 Subject: Implemented caching for certain API endpoints, Added documentation & made backend-fixes --- packages/server/package.json | 2 + packages/server/src/app.ts | 18 +- packages/server/src/cache/index.ts | 64 +++++ packages/server/src/cache/sources.ts | 314 +++++++++++++++++++++ packages/server/src/git/branch.ts | 5 + packages/server/src/git/commit.ts | 17 +- packages/server/src/git/diff.ts | 6 +- packages/server/src/git/error/index.ts | 3 +- packages/server/src/git/error/types.ts | 12 + packages/server/src/git/patch.ts | 18 +- packages/server/src/git/repository.ts | 36 ++- packages/server/src/git/tree_entry.ts | 8 +- packages/server/src/routes/api/v1/data.ts | 119 ++++++++ packages/server/src/routes/api/v1/index.ts | 81 +++--- packages/server/src/routes/api/v1/repo/branches.ts | 37 +-- packages/server/src/routes/api/v1/repo/index.ts | 35 +-- packages/server/src/routes/api/v1/repo/log.ts | 56 +--- packages/server/src/routes/api/v1/repo/map.ts | 24 -- packages/server/src/routes/repo.ts | 10 +- packages/server/src/server.ts | 70 +++-- packages/server/src/types/fastify.d.ts | 11 +- packages/server/src/types/index.d.ts | 10 +- packages/server/tsconfig.json | 2 +- 23 files changed, 736 insertions(+), 222 deletions(-) create mode 100644 packages/server/src/cache/index.ts create mode 100644 packages/server/src/cache/sources.ts create mode 100644 packages/server/src/routes/api/v1/data.ts delete mode 100644 packages/server/src/routes/api/v1/repo/map.ts (limited to 'packages/server') diff --git a/packages/server/package.json b/packages/server/package.json index 6d213dd..b295966 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -9,6 +9,7 @@ "start": "ts-node src/server.ts" }, "dependencies": { + "cache-manager": "^3.4.4", "date-fns": "^2.22.1", "fastify": "^3.17.0", "fastify-static": "^4.2.2", @@ -18,6 +19,7 @@ "whatwg-url": "^9.0.0" }, "devDependencies": { + "@types/cache-manager": "^3.4.2", "@types/js-yaml": "^4.0.1", "@types/node": "^16.3.1", "@types/nodegit": "^0.27.2", diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 42d096a..33c5a5a 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -4,10 +4,11 @@ import fastifyStatic from "fastify-static"; import { Settings } from "./types"; import repo from "./routes/repo"; import { join } from "path"; -import { readdirSync } from "fs"; +import { readdir } from "fs/promises"; import { exit } from "process"; +import { ServerCache } from "./cache"; -export default function buildApp(settings: Settings): FastifyInstance { +export default async function buildApp(settings: Settings, cache: ServerCache | null): Promise { const fastify = fastifyFactory(); fastify.setErrorHandler((err, req, reply) => { @@ -20,20 +21,17 @@ export default function buildApp(settings: Settings): FastifyInstance { reply.code(500).send("Internal server error!"); }); - fastify.setNotFoundHandler({}, function(req, reply) { + fastify.setNotFoundHandler({}, (req, reply) => { reply.code(404).send("Page not found!"); }); if(!settings.dev) { const dist_dir = join(__dirname, "/../../client/dist"); - try { - readdirSync(dist_dir); - } - catch { + await readdir(dist_dir).catch(() => { console.error("Error: Client dist directory doesn't exist!"); exit(1); - } + }); fastify.register(fastifyStatic, { root: dist_dir }); @@ -49,8 +47,8 @@ export default function buildApp(settings: Settings): FastifyInstance { fastify.addContentTypeParser("application/x-git-upload-pack-request", (req, payload, done) => done(null, payload)); fastify.addContentTypeParser("application/x-git-receive-pack-request", (req, payload, done) => done(null, payload)); - fastify.register(api, { prefix: "/api/v1", config: { settings: settings } }); - fastify.register(repo, { prefix: "/:repo([a-zA-Z0-9\\.\\-_]+)", config: { settings: settings } }); + fastify.register(api, { prefix: "/api/v1", config: { settings: settings, cache: cache } }); + fastify.register(repo, { prefix: "/:repo([a-zA-Z0-9\\.\\-_]+)", config: { settings: settings, cache: cache } }); return fastify; } \ No newline at end of file diff --git a/packages/server/src/cache/index.ts b/packages/server/src/cache/index.ts new file mode 100644 index 0000000..9bb4abd --- /dev/null +++ b/packages/server/src/cache/index.ts @@ -0,0 +1,64 @@ +/** + * Utilities for managing server-side cache + * + * @module cache + */ + +import { caching, Cache } from "cache-manager"; +import { cacheAllSources, CacheSource } from "./sources"; +import { CacheConfig } from "../types"; + +export *as sources from "./sources"; + +export class ServerCache { + private _cache: Cache; + + public ready = false; + + /** + * @param [config] - Cache configuration from the settings + */ + constructor(config?: Omit) { + this._cache = caching({ + store: "memory", + max: config?.max || 5000000, + ttl: config?.ttl || 120, + refreshThreshold: config?.refreshThreshold || 80 + }); + } + + /** + * Returns the cache value specified in the source & caches it if need be + * + * @template T - The constructor of a cache source + * @param Source - Information about where to get the value from + * @param args - Source arguments + * @returns A value from the cache + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public async receive CacheSource>(Source: T, ...args: ConstructorParameters): Promise { + const source = new Source(...args); + + const result = await this._cache.wrap(source.key(), () => source.func()) as T; + + return source.post + ? source.post(result) as T + : result; + } + + /** + * Initialize the cache. + * This will cache all of the available sources. + * + * @param git_dir - A git directory + */ + public async init(git_dir: string): Promise { + if(this.ready === true) { + throw(new Error("Cache has already been initialized!")); + } + + await cacheAllSources(this, git_dir); + + this.ready = true; + } +} \ No newline at end of file diff --git a/packages/server/src/cache/sources.ts b/packages/server/src/cache/sources.ts new file mode 100644 index 0000000..9ee953d --- /dev/null +++ b/packages/server/src/cache/sources.ts @@ -0,0 +1,314 @@ +import { + LogCommit, + Commit as APICommit, + Tag as APITag, + RepositorySummary, + Repository as APIRepository, + BranchSummary, + Branch as APIBranch +} from "api"; +import { getBranch, getBranches, getCommit, getLogCommits, getRepositories, getRepository, getTags } from "../routes/api/v1/data"; +import { Branch, Commit, Repository, Tag } from "../git"; +import { ServerCache } from "."; + +/** + * Holds information about a cache key + * + * @abstract + */ +export abstract class CacheSource { + private _key?: string; + + /** + * @param [key] - The name of the cache key + */ + constructor(key?: string) { + this._key = key; + } + + /** + * Returns the full cache key name + */ + public key(): string { + return `repositories${this._key || ""}`; + } + + /** + * Returns fresh data + */ + public abstract func(): Promise | unknown; + + /** + * Returns the input but modified + * @param [input] - Variable to modify + */ + public abstract post?(input: unknown): unknown; +} + +/** + * Cache source for log commits + * + * @extends CacheSource + */ +export class LogCommitsSource extends CacheSource { + private _count: number; + private _repository: Repository; + + /** + * @param repository - An instance of a repository + * @param [count] - The number of commits to return + */ + constructor(repository: Repository, count = 20) { + super(`_${repository.name.short}_${repository.branch}_commits`); + + this._repository = repository; + this._count = count; + } + + /** + * @returns An array of log commits + */ + public readonly func = async(): Promise => getLogCommits(await this._repository.commits(true)); + + /** + * @param commits - An array of log commits + * @returns A modified array of log commits + */ + public readonly post = (commits: LogCommit[]): LogCommit[] => commits.slice(0, this._count); +} + +/** + * Cache source for a commit + * + * @extends CacheSource + */ +export class CommitSource extends CacheSource { + private _commit: Commit; + + /** + * @param repository - An instance of a repository + * @param commit - An instance of a commit + */ + constructor(repository: Repository, commit: Commit) { + super(`_${repository.name.short}_${repository.branch}_commits_${commit.id}`); + + this._commit = commit; + } + + /** + * @returns An array of API commits + */ + public readonly func = (): Promise => getCommit(this._commit); + + public readonly post = undefined; +} + +/** + * Cache source for tags + * + * @extends CacheSource + */ +export class TagsSource extends CacheSource { + private _tags: Tag[]; + + /** + * @param repository - An instance of a repository + * @param tags - An array of tag instances + */ + constructor(repository: Repository, tags: Tag[]) { + super(`_${repository.name.short}_tags`); + + this._tags = tags; + } + + /** + * @returns An array of API tags + */ + public readonly func = (): Promise => getTags(this._tags); + + public readonly post = undefined; +} + +/** + * Cache source for repositories + * + * @extends CacheSource + */ +export class RepositoriesSource extends CacheSource { + private _repositories: Repository[]; + + /** + * @param repositories An array of repository instances + */ + constructor(repositories: Repository[]) { + super(); + + this._repositories = repositories; + } + + /** + * @returns An array of repository summaries + */ + public readonly func = (): Promise => getRepositories(this._repositories); + + public readonly post = undefined; +} + +/** + * Cache source for a repository + * + * @extends CacheSource + */ +export class RepositorySource extends CacheSource { + private _repository: Repository; + + /** + * @param repository - An instance of a repository + */ + constructor(repository: Repository) { + super(`_${repository.name.short}`); + + this._repository = repository; + } + + /** + * @returns A API repository + */ + public readonly func = (): Promise => getRepository(this._repository); + + public readonly post = undefined; +} + +/** + * Cache source for branches + * + * @extends CacheSource + */ +export class BranchesSource extends CacheSource { + private _branches: Branch[]; + + /** + * @param repository - An instance of a repository + * @param branches - An array of branch instances + */ + constructor(repository: Repository, branches: Branch[]) { + super(`_${repository.name.short}_branches`); + + this._branches = branches; + } + + /** + * @returns An array of branch summaries + */ + public readonly func = (): BranchSummary[] => getBranches(this._branches); + + public readonly post = undefined; +} + +/** + * Cache source for a branch + * + * @extends CacheSource + */ +export class BranchSource extends CacheSource { + private _branch: Branch; + + /** + * @param repository - An instance of a repository + * @param branch - An instance of a branch + */ + constructor(repository: Repository, branch: Branch) { + super(`_${repository.name.short}_branches_${branch.name}`); + + this._branch = branch; + } + + /** + * @returns A API branch + */ + public readonly func = (): Promise => getBranch(this._branch); + + public readonly post = undefined; +} + +/** + * Caches all of the available cache sources + * + * @param cache - A server cache instance + * @param git_dir - A git directory + */ +export async function cacheAllSources(cache: ServerCache, git_dir: string): Promise { + console.log("Initializing cache... this may take a while\n"); + + const repositories = await Repository.openAll(git_dir); + + process.stdout.write("Caching repositories... "); + + await cache.receive(RepositoriesSource, repositories); + + process.stdout.write("done\n\n"); + + for(const repository of repositories) { + console.log(repository.name.short); + + process.stdout.write("-> Caching repository... "); + + await cache.receive(RepositorySource, repository); + + process.stdout.write("done\n"); + + process.stdout.write("-> Caching tags... "); + + const tags = await repository.tags(); + + await cache.receive(TagsSource, repository, tags); + + process.stdout.write("done\n"); + + process.stdout.write("-> Caching branches... "); + + const branches = await repository.branches(); + + await cache.receive(BranchesSource, repository, branches); + + process.stdout.write("done\n"); + + for(const branch of branches) { + const branch_repository = await branch.repository(); + + console.log(`\n-> ${branch.name}`); + + process.stdout.write(" -> Caching branch... "); + + await cache.receive(BranchSource, branch_repository, branch); + + process.stdout.write("done\n"); + + process.stdout.write(" -> Caching log commits... "); + + await cache.receive(LogCommitsSource, branch_repository, 0); + + process.stdout.write("done\n"); + + const message = " -> Caching commits... "; + process.stdout.write(message); + + const commits = await branch_repository.commits(true); + + const commits_cnt = commits.length; + for(const commit of commits) { + process.stdout.clearLine(1); + process.stdout.cursorTo(message.length); + process.stdout.write(`${Math.round(commits.indexOf(commit) / commits_cnt * 100)}%`); + + await cache.receive(CommitSource, branch_repository, commit); + } + + process.stdout.clearLine(1); + process.stdout.cursorTo(message.length); + process.stdout.write("done\n"); + } + + console.log(""); + } +} \ No newline at end of file diff --git a/packages/server/src/git/branch.ts b/packages/server/src/git/branch.ts index dacabda..85f83be 100644 --- a/packages/server/src/git/branch.ts +++ b/packages/server/src/git/branch.ts @@ -10,6 +10,10 @@ import { createError, ErrorWhere, FailedError, NotFoundError, UnknownError } fro * @extends Reference */ export class Branch extends Reference { + public async repository(): Promise { + return this._owner.withBranch(this.name); + } + /** * Returns the branch's latest commit * @@ -41,6 +45,7 @@ export class Branch extends Reference { } throw(createError(ErrorWhere.Branch, UnknownError)); }); + return new Branch(owner, reference); } diff --git a/packages/server/src/git/commit.ts b/packages/server/src/git/commit.ts index 6ef02ef..7062304 100644 --- a/packages/server/src/git/commit.ts +++ b/packages/server/src/git/commit.ts @@ -207,21 +207,28 @@ export class Commit { * @returns An instance of a commit */ public static async branchCommit(owner: Repository): Promise { - return new Commit(owner, await owner.ng_repository.getBranchCommit(owner.branch_name)); + return new Commit(owner, await owner.ng_repository.getBranchCommit(owner.branch)); } /** * Returns a number of commits in a repository * * @param owner - The repository which the commits are in - * @param [count=20] - The number of commits to get + * @param [amount=20] - The number of commits to get or whether or not to get all commits * @returns An array of commit instances */ - public static async getMultiple(owner: Repository, count = 20): Promise { + public static async getMultiple(owner: Repository, amount: number | boolean = 20): Promise { const walker = NodeGitRevwalk.create(owner.ng_repository); - walker.pushRef(`refs/heads/${owner.branch_name}`); + walker.pushRef(`refs/heads/${owner.branch}`); - return Promise.all((await walker.getCommits(count)).map(commit => new Commit(owner, commit))); + if(typeof amount === "boolean") { + return Promise.all((await (amount + ? walker.getCommitsUntil(() => true) + : walker.getCommits(20) + )).map(commit => new Commit(owner, commit))); + } + + return Promise.all((await walker.getCommits(amount)).map(commit => new Commit(owner, commit))); } } \ No newline at end of file diff --git a/packages/server/src/git/diff.ts b/packages/server/src/git/diff.ts index d084e5d..a2c8829 100644 --- a/packages/server/src/git/diff.ts +++ b/packages/server/src/git/diff.ts @@ -1,5 +1,5 @@ import { Diff as NodeGitDiff } from "nodegit"; -import { createError, ErrorWhere, NotFoundError } from "./error"; +import { createError, DiffTooLargeError, ErrorWhere, NotFoundError } from "./error"; import { Patch } from "./patch"; type PatchHeaderData = { @@ -63,6 +63,10 @@ export class Diff { * @returns An array of patch instances */ public async patches(): Promise { + if((await this.rawPatches()).split("\n").length > 50000) { + throw(createError(ErrorWhere.Diff, DiffTooLargeError)); + } + return (await this.ng_diff.patches()).map((patch, index) => new Patch(this, patch, index)); } diff --git a/packages/server/src/git/error/index.ts b/packages/server/src/git/error/index.ts index b8994d3..55a3aef 100644 --- a/packages/server/src/git/error/index.ts +++ b/packages/server/src/git/error/index.ts @@ -21,7 +21,8 @@ export enum ErrorWhere { Commit = "commit", Diff = "diff", Misc = "misc", - Blob = "blob" + Blob = "blob", + Patch = "patch" } /** diff --git a/packages/server/src/git/error/types.ts b/packages/server/src/git/error/types.ts index b8c860b..19ad710 100644 --- a/packages/server/src/git/error/types.ts +++ b/packages/server/src/git/error/types.ts @@ -42,4 +42,16 @@ export class NotInKeyringError extends ErrorType { constructor(email: string) { super(500, `A public key for '${email}' doesn't exist in the server pgp keyring!`); } +} + +export class PatchTooLargeError extends ErrorType { + constructor() { + super(500, "Patch is too large for parsing!"); + } +} + +export class DiffTooLargeError extends ErrorType { + constructor() { + super(500, "Diff is too large for parsing!"); + } } \ No newline at end of file diff --git a/packages/server/src/git/patch.ts b/packages/server/src/git/patch.ts index 4239ce4..4527d03 100644 --- a/packages/server/src/git/patch.ts +++ b/packages/server/src/git/patch.ts @@ -1,5 +1,6 @@ import { Diff } from "./diff"; import { ConvenientPatch as NodeGitPatch } from "nodegit"; +import { createError, ErrorWhere, PatchTooLargeError } from "./error"; type Hunk = { new_start: number, @@ -87,10 +88,9 @@ export class Patch { * * These bounds are in the context of it's whole diff * - * @returns A patch bounds instance which contains a start & an end property + * @returns The patch's bounds */ - private async _bounds(): Promise { - const raw_patches = (await this._diff.rawPatches()).split("\n"); + private async _bounds(raw_patches: string[]): Promise { const patch_header_data = await this._diff.patchHeaderData(); return { @@ -104,7 +104,7 @@ export class Patch { */ private async _content(): Promise { const raw_patches = (await this._diff.rawPatches()).split("\n"); - const bounds = await this._bounds(); + const bounds = await this._bounds(raw_patches); return raw_patches.slice(bounds.start, bounds.end).join("\n"); } @@ -115,10 +115,14 @@ export class Patch { * @returns Whether or not the patch is too large */ public async isTooLarge(): Promise { + if(this.additions > 2000 || this.deletions > 2000) { + return true; + } + const content = (await this._content()).split("\n"); const line_lengths = content.map(line => line.length).reduce((result, length) => result + length); - if(content.length > 5000 || line_lengths > 5000) { + if(content.length > 10000 || line_lengths > 10000) { return true; } @@ -131,6 +135,10 @@ export class Patch { * @returns An array of hunk instances */ public async getHunks(): Promise { + if(await this.isTooLarge()) { + throw(createError(ErrorWhere.Patch, PatchTooLargeError)); + } + const content = (await this._content()).split("\n"); const hunks = await this._ng_patch.hunks(); diff --git a/packages/server/src/git/repository.ts b/packages/server/src/git/repository.ts index c1410ab..53245be 100644 --- a/packages/server/src/git/repository.ts +++ b/packages/server/src/git/repository.ts @@ -38,7 +38,7 @@ export class Repository { public name: RepositoryName; public git_dir: string; - public branch_name: string; + private _branch: string; /** * @param repository - An instance of a Nodegit repository @@ -52,7 +52,7 @@ export class Repository { }; this.git_dir = dirname(repository.path()); - this.branch_name = branch; + this._branch = branch; } /** @@ -69,23 +69,31 @@ export class Repository { return getFile(this.git_dir, this.name.full, "owner"); } + get branch(): string { + return this._branch; + } + + set branch(branch: string) { + this._branch = branch; + } + /** * Returns the repository's branch * * @returns An instance of a branch */ - public branch(): Promise { - return Branch.lookup(this, this.branch_name); + public getBranch(): Promise { + return Branch.lookup(this, this._branch); } /** * Returns the repository's commits * - * @param [count=20] - The number of commits to get + * @param [amount=20] - The number of commits to get or whether or not to get all commits * @returns An array of commit instances */ - public async commits(count?: number): Promise { - return Commit.getMultiple(this, count); + public async commits(amount?: number | boolean): Promise { + return Commit.getMultiple(this, amount); } /** @@ -118,6 +126,20 @@ export class Repository { .catch(() => false); } + /** + * Returns this repository instance with a different branch + * + * @param branch - The branch to switch to + * @returns An instance of a repository + */ + public async withBranch(branch: string): Promise { + if(!await Branch.lookupExists(this.ng_repository, branch)) { + throw(createError(ErrorWhere.Repository, NotFoundError, "branch")); + } + + return new Repository(this.ng_repository, branch); + } + /** * Returns the repository's branches * diff --git a/packages/server/src/git/tree_entry.ts b/packages/server/src/git/tree_entry.ts index b03ea9e..cdcb0d3 100644 --- a/packages/server/src/git/tree_entry.ts +++ b/packages/server/src/git/tree_entry.ts @@ -31,11 +31,11 @@ export abstract class BaseTreeEntry { */ public async latestCommit(): Promise { const rev_walk = NodeGitRevwalk.create(this._owner.ng_repository); - rev_walk.pushRef(`refs/heads/${this._owner.branch_name}`); + rev_walk.pushRef(`refs/heads/${this._owner.branch}`); const commit_cnt = (await rev_walk.getCommitsUntil(() => true)).length; - rev_walk.pushRef(`refs/heads/${this._owner.branch_name}`); + rev_walk.pushRef(`refs/heads/${this._owner.branch}`); const file_hist = await rev_walk.fileHistoryWalk(this.path, commit_cnt); return new Commit(this._owner, file_hist[0].commit); @@ -48,11 +48,11 @@ export abstract class BaseTreeEntry { */ public async history(count?: number): Promise { const rev_walk = NodeGitRevwalk.create(this._owner.ng_repository); - rev_walk.pushRef(`refs/heads/${this._owner.branch_name}`); + rev_walk.pushRef(`refs/heads/${this._owner.branch}`); const commit_cnt = (await rev_walk.getCommitsUntil(() => true)).length; - rev_walk.pushRef(`refs/heads/${this._owner.branch_name}`); + rev_walk.pushRef(`refs/heads/${this._owner.branch}`); const file_hist = await rev_walk.fileHistoryWalk(this.path, commit_cnt); const commit_history = await Promise.all(file_hist.map(async hist_entry => new Commit(this._owner, await NodeGitCommit.lookup(this._owner.ng_repository, hist_entry.commit)))); diff --git a/packages/server/src/routes/api/v1/data.ts b/packages/server/src/routes/api/v1/data.ts new file mode 100644 index 0000000..97f07ff --- /dev/null +++ b/packages/server/src/routes/api/v1/data.ts @@ -0,0 +1,119 @@ +import { + LogCommit, + Patch as APIPatch, + Commit as APICommit, Tag as APITag, + RepositorySummary as APIRepositorySummary, + Repository as APIRepository, + BranchSummary, + Branch as APIBranch +} from "api"; +import { Branch, Commit, Patch, Repository, Tag } from "../../../git"; + +export async function getLogCommits(commits: Commit[]): Promise { + return Promise.all(commits.map(async(commit: Commit) => { + const stats = await commit.stats(); + + const is_signed = await commit.isSigned(); + + return { + id: commit.id, + author: { + name: commit.author().name, + email: commit.author().email, + fingerprint: await commit.author().fingerprint().catch(() => null) + }, + isSigned: is_signed, + signatureVerified: is_signed ? await commit.verifySignature().catch(() => false) : null, + message: commit.message, + date: commit.date, + insertions: stats.insertions, + deletions: stats.deletions, + files_changed: stats.files_changed + }; + })); +} + +export async function getCommit(commit: Commit): Promise { + const stats = await commit.stats(); + + const is_signed = await commit.isSigned(); + + const patches = await (await commit.diff()).patches().catch(() => null); + + return { + message: commit.message, + author: { + name: commit.author().name, + email: commit.author().email, + fingerprint: await commit.author().fingerprint().catch(() => null) + }, + isSigned: is_signed, + signatureVerified: is_signed ? await commit.verifySignature().catch(() => false) : null, + date: commit.date, + insertions: stats.insertions, + deletions: stats.deletions, + files_changed: stats.files_changed, + too_large: Boolean(!patches), + diff: patches + ? await Promise.all(patches.map(async(patch: Patch) => { + return { + additions: patch.additions, + deletions: patch.deletions, + from: patch.from, + to: patch.to, + too_large: await patch.isTooLarge(), + hunks: await patch.getHunks().catch(() => null) + }; + })) + : null + }; +} + +export function getTags(tags: Tag[]): Promise { + return Promise.all(tags.map(async(tag: Tag) => { + const author = await tag.author(); + return { + name: tag.name, + author: { + name: author.name, + email: author.email + }, + date: await tag.date() + }; + })); +} + +export function getRepositories(repositories: Repository[]): Promise { + return Promise.all(repositories.map(async repository => { + return { + name: repository.name.short, + description: await repository.description(), + last_updated: (await repository.head()).date + }; + })); +} + +export async function getRepository(repository: Repository): Promise { + return { + name: repository.name.short, + description: await repository.description(), + has_readme: await (await repository.tree()).findExists("README.md") + }; +} + +export function getBranches(branches: Branch[]): BranchSummary[] { + return branches.map(branch => { + return { + id: branch.id, + name: branch.name + }; + }); +} + +export async function getBranch(branch: Branch): Promise { + return { + id: branch.id, + name: branch.name, + latest_commit: await branch.latestCommit() + }; +} \ No newline at end of file diff --git a/packages/server/src/routes/api/v1/index.ts b/packages/server/src/routes/api/v1/index.ts index 4b63435..7997b4d 100644 --- a/packages/server/src/routes/api/v1/index.ts +++ b/packages/server/src/routes/api/v1/index.ts @@ -1,47 +1,24 @@ -import { FastifyInstance, FastifyPluginOptions } from "fastify"; +import { FastifyPluginCallback } from "fastify"; import { Repository } from "../../../git/repository"; -import { Route } from "../../../types/fastify"; +import { FastifyPluginOptions, Route } from "../../../types/fastify"; import repo from "./repo"; import { verifyRepoName } from "../util"; -import { Info as APIInfo, RepositorySummary as APIRepositorySummary, Repository as APIRepository } from "api"; +import { Info as APIInfo } from "api"; import { ServerError } from "../../../git/error"; +import { getRepositories, getRepository } from "./data"; +import { sources } from "../../../cache"; -function setHandlers(fastify: FastifyInstance): void { - fastify.setErrorHandler((err, req, reply) => { - if(err.validation) { - reply.code(400).send({ error: `${err.validation[0].dataPath} ${err.validation[0].message}` }); - return; - } - - console.log(err); - - reply.code(500).send({ error: "Internal server error!" }); - }); - fastify.setNotFoundHandler((req, reply) => { - reply.code(404).send({ error: "Endpoint not found!" }); - }); -} - -function reposEndpoints(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void { +const reposEndpoints: FastifyPluginCallback = (fastify, opts, done) => { fastify.route({ method: "GET", url: "/repos", handler: async(req, reply) => { - const repos = await Repository.openAll(opts.config.settings.git_dir); - - if(!repos) { - reply.send({ data: [] }); - return; - } + const repositories = await Repository.openAll(opts.config.settings.git_dir); reply.send({ - data: await Promise.all(repos.map(async repository => { - return { - name: repository.name.short, - description: await repository.description(), - last_updated: (await repository.head()).date - }; - })) + data: await (opts.config.cache + ? opts.config.cache.receive(sources.RepositoriesSource, repositories) + : getRepositories(repositories)) }); } }); @@ -67,20 +44,30 @@ function reposEndpoints(fastify: FastifyInstance, opts: FastifyPluginOptions, do return; } - const data: APIRepository = { - name: repository.name.short, - description: await repository.description(), - has_readme: await (await repository.tree()).findExists("README.md") - }; - - reply.send({ data: data }); + reply.send({ + data: await (opts.config.cache + ? opts.config.cache.receive(sources.RepositorySource, repository) + : getRepository(repository)) + }); } }); done(); -} +}; + +const api: FastifyPluginCallback = (fastify, opts, done) => { + fastify.setErrorHandler((err, req, reply) => { + if(err.validation) { + reply.code(400).send({ error: `${err.validation[0].dataPath} ${err.validation[0].message}` }); + return; + } -export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void { - setHandlers(fastify); + console.log(err); + + reply.code(500).send({ error: "Internal server error!" }); + }); + fastify.setNotFoundHandler((req, reply) => { + reply.code(404).send({ error: "Endpoint not found!" }); + }); fastify.route({ method: "GET", @@ -95,8 +82,10 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do } }); - fastify.register(reposEndpoints, { config: { settings: opts.config.settings } }); - fastify.register(repo, { prefix: "/repos/:repo", config: { settings: opts.config.settings } }); + fastify.register(reposEndpoints, { config: opts.config }); + fastify.register(repo, { prefix: "/repos/:repo", config: opts.config }); done(); -} \ No newline at end of file +}; + +export default api; \ No newline at end of file diff --git a/packages/server/src/routes/api/v1/repo/branches.ts b/packages/server/src/routes/api/v1/repo/branches.ts index 99f0327..f709f4d 100644 --- a/packages/server/src/routes/api/v1/repo/branches.ts +++ b/packages/server/src/routes/api/v1/repo/branches.ts @@ -1,9 +1,10 @@ -import { FastifyInstance, FastifyPluginOptions } from "fastify"; +import { FastifyPluginCallback } from "fastify"; +import { sources } from "../../../../cache"; import { Branch } from "../../../../git/branch"; -import { Route } from "../../../../types/fastify"; -import { BranchSummary as APIBranchSummary, Branch as APIBranch } from "api"; +import { FastifyPluginOptions, Route } from "../../../../types/fastify"; +import { getBranch, getBranches } from "../data"; -export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void { +const branches: FastifyPluginCallback = (fastify, opts, done) => { fastify.route({ method: "GET", url: "/branches", @@ -11,12 +12,9 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do const branches = await req.repository.branches(); reply.send({ - data: branches.map(branch => { - return { - id: branch.id, - name: branch.name - }; - }) + data: opts.config.cache + ? await opts.config.cache.receive(sources.BranchesSource, req.repository, branches) + : getBranches(branches) }); } }); @@ -32,22 +30,15 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do handler: async(req, reply) => { const branch = await Branch.lookup(req.repository, req.params.branch); - if(!branch) { - reply.code(404).send({ error: "Branch not found!" }); - return; - } - - const data: APIBranch = { - id: branch.id, - name: branch.name, - latest_commit: await branch.latestCommit() - }; - reply.send({ - data: data + data: await (opts.config.cache + ? opts.config.cache.receive(sources.BranchSource, req.repository, branch) + : getBranch(branch)) }); } }); done(); -} \ No newline at end of file +}; + +export default branches; \ No newline at end of file diff --git a/packages/server/src/routes/api/v1/repo/index.ts b/packages/server/src/routes/api/v1/repo/index.ts index 4cd6c51..f8e01d3 100644 --- a/packages/server/src/routes/api/v1/repo/index.ts +++ b/packages/server/src/routes/api/v1/repo/index.ts @@ -1,15 +1,15 @@ -import { CoolFastifyRequest, Route } from "../../../../types/fastify"; -import { FastifyInstance, FastifyPluginOptions } from "fastify"; +import { CoolFastifyRequest, Route, FastifyPluginOptions } from "../../../../types/fastify"; +import { FastifyInstance, FastifyPluginCallback } from "fastify"; import { Repository } from "../../../../git/repository"; -import { Tag } from "../../../../git/tag"; import { BaseTreeEntry, BlobTreeEntry, TreeEntry } from "../../../../git/tree_entry"; import { basename } from "path"; import branches from "./branches"; import log from "./log"; import { verifyRepoName } from "../../util"; -import { Tree as APITree, Tag as APITag, TreeEntry as APITreeEntry } from "api"; +import { Tree as APITree, TreeEntry as APITreeEntry } from "api"; import { ServerError } from "../../../../git/error"; -import { commitMap } from "./map"; +import { getLogCommits, getTags } from "../data"; +import { sources } from "../../../../cache"; declare module "fastify" { interface FastifyRequest { @@ -48,20 +48,11 @@ async function treeEntryMap(entry: BaseTreeEntry) { }; } -async function tagMap(tag: Tag) { - const author = await tag.author(); - return { - name: tag.name, - author: { name: author.name, email: author.email }, - date: await tag.date() - }; -} - -export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void { +const repo: FastifyPluginCallback = (fastify, opts, done) => { addHooks(fastify, opts); - fastify.register(log); - fastify.register(branches); + fastify.register(log, { config: opts.config }); + fastify.register(branches, { config: opts.config }); fastify.route({ method: "GET", @@ -127,7 +118,7 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do const history = await tree_entry.history(Number(req.query.count)); - reply.send({ data: await Promise.all(history.map(commitMap)) }); + reply.send({ data: await getLogCommits(history) }); } }); @@ -137,10 +128,14 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do handler: async(req, reply) => { const tags = await req.repository.tags(); reply.send({ - data: await Promise.all(tags.map(tagMap)) + data: await (opts.config.cache + ? opts.config.cache.receive(sources.TagsSource, req.repository, tags) + : getTags(tags)) }); } }); done(); -} \ No newline at end of file +}; + +export default repo; \ No newline at end of file diff --git a/packages/server/src/routes/api/v1/repo/log.ts b/packages/server/src/routes/api/v1/repo/log.ts index 163cf80..7ad1e11 100644 --- a/packages/server/src/routes/api/v1/repo/log.ts +++ b/packages/server/src/routes/api/v1/repo/log.ts @@ -1,23 +1,11 @@ -import { FastifyInstance, FastifyPluginOptions } from "fastify"; +import { FastifyPluginCallback } from "fastify"; +import { sources } from "../../../../cache"; import { Commit } from "../../../../git/commit"; -import { Patch } from "../../../../git/patch"; -import { Route } from "../../../../types/fastify"; +import { Route, FastifyPluginOptions } from "../../../../types/fastify"; import { verifySHA } from "../../util"; -import { Patch as APIPatch, Commit as APICommit } from "api"; -import { commitMap } from "./map"; +import { getCommit, getLogCommits } from "../data"; -async function patchMap(patch: Patch) { - return { - additions: patch.additions, - deletions: patch.deletions, - from: patch.from, - to: patch.to, - too_large: await patch.isTooLarge(), - hunks: await patch.getHunks() - }; -} - -export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void { +const log: FastifyPluginCallback = (fastify, opts, done) => { fastify.route({ method: "GET", url: "/log", @@ -27,10 +15,12 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do } }, handler: async(req, reply) => { - const commits = await req.repository.commits(Number(req.query.count)); + const commits = await req.repository.commits(Number(req.query.count) || undefined); reply.send({ - data: await Promise.all(commits.map(commitMap)) + data: await (opts.config.cache + ? opts.config.cache.receive(sources.LogCommitsSource, req.repository, Number(req.query.count) || undefined) + : getLogCommits(commits)) }); } }); @@ -51,31 +41,15 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do const commit = await Commit.lookup(req.repository, req.params.commit); - const stats = await commit.stats(); - - const is_signed = await commit.isSigned(); - - const data: APICommit = { - message: commit.message, - author: { - name: commit.author().name, - email: commit.author().email, - fingerprint: await commit.author().fingerprint().catch(() => null) - }, - isSigned: is_signed, - signatureVerified: is_signed ? await commit.verifySignature().catch(() => false) : null, - date: commit.date, - insertions: stats.insertions, - deletions: stats.deletions, - files_changed: stats.files_changed, - diff: await Promise.all((await (await commit.diff()).patches()).map(patchMap)) - }; - reply.send({ - data: data + data: await (opts.config.cache + ? opts.config.cache.receive(sources.CommitSource, req.repository, commit) + : getCommit(commit)) }); } }); done(); -} \ No newline at end of file +}; + +export default log; \ No newline at end of file diff --git a/packages/server/src/routes/api/v1/repo/map.ts b/packages/server/src/routes/api/v1/repo/map.ts deleted file mode 100644 index a544d1a..0000000 --- a/packages/server/src/routes/api/v1/repo/map.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Commit } from "../../../../git/commit"; -import { LogCommit } from "api"; - -export async function commitMap(commit: Commit): Promise { - const stats = await commit.stats(); - - const is_signed = await commit.isSigned(); - - return { - id: commit.id, - author: { - name: commit.author().name, - email: commit.author().email, - fingerprint: await commit.author().fingerprint().catch(() => null) - }, - isSigned: is_signed, - signatureVerified: is_signed ? await commit.verifySignature().catch(() => false) : null, - message: commit.message, - date: commit.date, - insertions: stats.insertions, - deletions: stats.deletions, - files_changed: stats.files_changed - }; -} \ No newline at end of file diff --git a/packages/server/src/routes/repo.ts b/packages/server/src/routes/repo.ts index bb70c68..1088e6b 100644 --- a/packages/server/src/routes/repo.ts +++ b/packages/server/src/routes/repo.ts @@ -1,11 +1,11 @@ import { Repository } from "../git/repository"; -import { CoolFastifyRequest, Route } from "../types/fastify"; +import { CoolFastifyRequest, Route, FastifyPluginOptions } from "../types/fastify"; import { Tag } from "../git/tag"; -import { FastifyInstance, FastifyPluginOptions } from "fastify"; +import { FastifyPluginCallback } from "fastify"; import { verifyRepoName } from "../routes/api/util"; import { ServerError } from "../git/error"; -export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void { +const repo: FastifyPluginCallback = (fastify, opts, done): void => { fastify.addHook("onRequest", async(req: CoolFastifyRequest, reply) => { if(!verifyRepoName(req.params.repo)) { reply.code(400).send("Bad request"); @@ -86,4 +86,6 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do }); done(); -} \ No newline at end of file +}; + +export default repo; \ No newline at end of file diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index 11b3f7f..3056bf5 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -1,41 +1,55 @@ -import { readFileSync, readdirSync } from "fs"; +import { readFile, readdir } from "fs/promises"; import { join } from "path"; import { exit } from "process"; import { Settings } from "./types"; import buildApp from "./app"; +import { ServerCache } from "./cache"; -const settings = JSON.parse(readFileSync(join(__dirname, "/../../../settings.json"), "utf-8")) as Settings; +async function main() { + const settings = JSON.parse(await readFile(join(__dirname, "/../../../settings.json"), { encoding: "utf-8" })) as Settings; -const settings_keys = Object.keys(settings); + const settings_keys = Object.keys(settings); -const mandatory_settings = [ "host", "port", "title", "about", "git_dir" ]; + const mandatory_settings = [ "host", "port", "title", "about", "git_dir" ]; -// Get missing mandatory settings -const settings_not_included = mandatory_settings.filter(x => !settings_keys.includes(x)); + // Get missing mandatory settings + const settings_not_included = mandatory_settings.filter(x => !settings_keys.includes(x)); -// Error out and exit if there's any missing settings -if(settings_not_included.length !== 0) { - console.log(`Error: settings file is missing ${(settings_not_included.length > 1) ? "keys" : "key"}:`); - console.log(settings_not_included.join(", ")); - exit(1); -} - -// Make sure that the git directory specified in the settings actually exists -try { - readdirSync(settings.git_dir); -} -catch { - console.error(`Error: Git directory ${settings.git_dir} doesn't exist!`); - exit(1); -} - -const app = buildApp(settings); + // Error out and exit if there's any missing settings + if(settings_not_included.length !== 0) { + console.log(`Error: settings file is missing ${(settings_not_included.length > 1) ? "keys" : "key"}:`); + console.log(settings_not_included.join(", ")); + exit(1); + } -app.listen(settings.port, settings.host, (err: Error, addr: string) => { - if(err) { - console.error(err); + // Make sure that the git directory specified in the settings actually exists + await readdir(settings.git_dir).catch(() => { + console.error(`Error: Git directory ${settings.git_dir} doesn't exist!`); exit(1); + }); + + const cache = (settings.cache && settings.cache.enabled === true) || settings.cache === undefined || settings.cache.enabled === undefined + ? new ServerCache(settings.cache) + : null; + + if(cache) { + await cache.init(settings.git_dir); + if(!cache.ready) { + console.error("Error: cache failed to initialize!"); + return 1; + } } - console.log(`Githermit is running on ${addr}`); -}); \ No newline at end of file + const app = await buildApp(settings, cache); + + app.listen(settings.port, settings.host, (err: Error, addr: string) => { + if(err) { + console.error(err); + exit(1); + } + + console.log(`Githermit is running on ${addr}`); + }); +} + +main(); \ No newline at end of file diff --git a/packages/server/src/types/fastify.d.ts b/packages/server/src/types/fastify.d.ts index ebaaac2..7c2341d 100644 --- a/packages/server/src/types/fastify.d.ts +++ b/packages/server/src/types/fastify.d.ts @@ -1,5 +1,7 @@ import { FastifyRequest, RequestGenericInterface } from "fastify"; import { ReplyGenericInterface } from "fastify/types/reply"; +import { Settings } from "."; +import { ServerCache } from "../cache"; export interface Request extends RequestGenericInterface { Params: Record, @@ -8,4 +10,11 @@ export interface Request extends RequestGenericInterface { export interface Route extends Request, ReplyGenericInterface {} -export type CoolFastifyRequest = FastifyRequest; \ No newline at end of file +export type CoolFastifyRequest = FastifyRequest; + +export type FastifyPluginOptions = { + config: { + settings: Settings, + cache: ServerCache | null + } +} \ No newline at end of file diff --git a/packages/server/src/types/index.d.ts b/packages/server/src/types/index.d.ts index 8f592f9..ca0743d 100644 --- a/packages/server/src/types/index.d.ts +++ b/packages/server/src/types/index.d.ts @@ -1,10 +1,18 @@ +export type CacheConfig = { + enabled: boolean, + ttl?: number, + max?: number, + refreshThreshold?: number +} + export type Settings = { host: string, port: number, title: string, about: string, git_dir: string, - dev: { + cache?: CacheConfig, + dev?: { port: number } } \ No newline at end of file diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index a98cc40..92c94fc 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -3,4 +3,4 @@ "compilerOptions": { "module": "CommonJS", } -} +} \ No newline at end of file -- cgit v1.2.3-18-g5258