From d1a1b7dc947063aef5f8375a6a1e03246b272c84 Mon Sep 17 00:00:00 2001
From: HampusM <hampus@hampusmat.com>
Date: Wed, 18 Aug 2021 17:29:55 +0200
Subject: Implemented caching for certain API endpoints, Added documentation &
made backend-fixes
---
packages/api/src/commit.d.ts | 3 +-
packages/server/package.json | 2 +
packages/server/src/app.ts | 18 +-
packages/server/src/cache/index.ts | 64 +++++
packages/server/src/cache/sources.ts | 314 +++++++++++++++++++++
packages/server/src/git/branch.ts | 5 +
packages/server/src/git/commit.ts | 17 +-
packages/server/src/git/diff.ts | 6 +-
packages/server/src/git/error/index.ts | 3 +-
packages/server/src/git/error/types.ts | 12 +
packages/server/src/git/patch.ts | 18 +-
packages/server/src/git/repository.ts | 36 ++-
packages/server/src/git/tree_entry.ts | 8 +-
packages/server/src/routes/api/v1/data.ts | 119 ++++++++
packages/server/src/routes/api/v1/index.ts | 81 +++---
packages/server/src/routes/api/v1/repo/branches.ts | 37 +--
packages/server/src/routes/api/v1/repo/index.ts | 35 +--
packages/server/src/routes/api/v1/repo/log.ts | 56 +---
packages/server/src/routes/api/v1/repo/map.ts | 24 --
packages/server/src/routes/repo.ts | 10 +-
packages/server/src/server.ts | 70 +++--
packages/server/src/types/fastify.d.ts | 11 +-
packages/server/src/types/index.d.ts | 10 +-
packages/server/tsconfig.json | 2 +-
24 files changed, 738 insertions(+), 223 deletions(-)
create mode 100644 packages/server/src/cache/index.ts
create mode 100644 packages/server/src/cache/sources.ts
create mode 100644 packages/server/src/routes/api/v1/data.ts
delete mode 100644 packages/server/src/routes/api/v1/repo/map.ts
(limited to 'packages')
diff --git a/packages/api/src/commit.d.ts b/packages/api/src/commit.d.ts
index c9974f3..5b682ce 100644
--- a/packages/api/src/commit.d.ts
+++ b/packages/api/src/commit.d.ts
@@ -30,7 +30,8 @@ export interface Commit {
insertions: number,
deletions: number,
files_changed: number,
- diff: Patch[]
+ too_large: boolean,
+ diff: Patch[] | null
}
export type LogCommit = {
diff --git a/packages/server/package.json b/packages/server/package.json
index 6d213dd..b295966 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -9,6 +9,7 @@
"start": "ts-node src/server.ts"
},
"dependencies": {
+ "cache-manager": "^3.4.4",
"date-fns": "^2.22.1",
"fastify": "^3.17.0",
"fastify-static": "^4.2.2",
@@ -18,6 +19,7 @@
"whatwg-url": "^9.0.0"
},
"devDependencies": {
+ "@types/cache-manager": "^3.4.2",
"@types/js-yaml": "^4.0.1",
"@types/node": "^16.3.1",
"@types/nodegit": "^0.27.2",
diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts
index 42d096a..33c5a5a 100644
--- a/packages/server/src/app.ts
+++ b/packages/server/src/app.ts
@@ -4,10 +4,11 @@ import fastifyStatic from "fastify-static";
import { Settings } from "./types";
import repo from "./routes/repo";
import { join } from "path";
-import { readdirSync } from "fs";
+import { readdir } from "fs/promises";
import { exit } from "process";
+import { ServerCache } from "./cache";
-export default function buildApp(settings: Settings): FastifyInstance {
+export default async function buildApp(settings: Settings, cache: ServerCache | null): Promise<FastifyInstance> {
const fastify = fastifyFactory();
fastify.setErrorHandler((err, req, reply) => {
@@ -20,20 +21,17 @@ export default function buildApp(settings: Settings): FastifyInstance {
reply.code(500).send("Internal server error!");
});
- fastify.setNotFoundHandler({}, function(req, reply) {
+ fastify.setNotFoundHandler({}, (req, reply) => {
reply.code(404).send("Page not found!");
});
if(!settings.dev) {
const dist_dir = join(__dirname, "/../../client/dist");
- try {
- readdirSync(dist_dir);
- }
- catch {
+ await readdir(dist_dir).catch(() => {
console.error("Error: Client dist directory doesn't exist!");
exit(1);
- }
+ });
fastify.register(fastifyStatic, { root: dist_dir });
@@ -49,8 +47,8 @@ export default function buildApp(settings: Settings): FastifyInstance {
fastify.addContentTypeParser("application/x-git-upload-pack-request", (req, payload, done) => done(null, payload));
fastify.addContentTypeParser("application/x-git-receive-pack-request", (req, payload, done) => done(null, payload));
- fastify.register(api, { prefix: "/api/v1", config: { settings: settings } });
- fastify.register(repo, { prefix: "/:repo([a-zA-Z0-9\\.\\-_]+)", config: { settings: settings } });
+ fastify.register(api, { prefix: "/api/v1", config: { settings: settings, cache: cache } });
+ fastify.register(repo, { prefix: "/:repo([a-zA-Z0-9\\.\\-_]+)", config: { settings: settings, cache: cache } });
return fastify;
}
\ No newline at end of file
diff --git a/packages/server/src/cache/index.ts b/packages/server/src/cache/index.ts
new file mode 100644
index 0000000..9bb4abd
--- /dev/null
+++ b/packages/server/src/cache/index.ts
@@ -0,0 +1,64 @@
+/**
+ * Utilities for managing server-side cache
+ *
+ * @module cache
+ */
+
+import { caching, Cache } from "cache-manager";
+import { cacheAllSources, CacheSource } from "./sources";
+import { CacheConfig } from "../types";
+
+export *as sources from "./sources";
+
+export class ServerCache {
+ private _cache: Cache;
+
+ public ready = false;
+
+ /**
+ * @param [config] - Cache configuration from the settings
+ */
+ constructor(config?: Omit<CacheConfig, "enabled">) {
+ this._cache = caching({
+ store: "memory",
+ max: config?.max || 5000000,
+ ttl: config?.ttl || 120,
+ refreshThreshold: config?.refreshThreshold || 80
+ });
+ }
+
+ /**
+ * Returns the cache value specified in the source & caches it if need be
+ *
+ * @template T - The constructor of a cache source
+ * @param Source - Information about where to get the value from
+ * @param args - Source arguments
+ * @returns A value from the cache
+ */
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ public async receive<T extends new(...args: any[]) => CacheSource>(Source: T, ...args: ConstructorParameters<T>): Promise<unknown> {
+ const source = new Source(...args);
+
+ const result = await this._cache.wrap(source.key(), () => source.func()) as T;
+
+ return source.post
+ ? source.post(result) as T
+ : result;
+ }
+
+ /**
+ * Initialize the cache.
+ * This will cache all of the available sources.
+ *
+ * @param git_dir - A git directory
+ */
+ public async init(git_dir: string): Promise<void> {
+ if(this.ready === true) {
+ throw(new Error("Cache has already been initialized!"));
+ }
+
+ await cacheAllSources(this, git_dir);
+
+ this.ready = true;
+ }
+}
\ No newline at end of file
diff --git a/packages/server/src/cache/sources.ts b/packages/server/src/cache/sources.ts
new file mode 100644
index 0000000..9ee953d
--- /dev/null
+++ b/packages/server/src/cache/sources.ts
@@ -0,0 +1,314 @@
+import {
+ LogCommit,
+ Commit as APICommit,
+ Tag as APITag,
+ RepositorySummary,
+ Repository as APIRepository,
+ BranchSummary,
+ Branch as APIBranch
+} from "api";
+import { getBranch, getBranches, getCommit, getLogCommits, getRepositories, getRepository, getTags } from "../routes/api/v1/data";
+import { Branch, Commit, Repository, Tag } from "../git";
+import { ServerCache } from ".";
+
+/**
+ * Holds information about a cache key
+ *
+ * @abstract
+ */
+export abstract class CacheSource {
+ private _key?: string;
+
+ /**
+ * @param [key] - The name of the cache key
+ */
+ constructor(key?: string) {
+ this._key = key;
+ }
+
+ /**
+ * Returns the full cache key name
+ */
+ public key(): string {
+ return `repositories${this._key || ""}`;
+ }
+
+ /**
+ * Returns fresh data
+ */
+ public abstract func(): Promise<unknown> | unknown;
+
+ /**
+ * Returns the input but modified
+ * @param [input] - Variable to modify
+ */
+ public abstract post?(input: unknown): unknown;
+}
+
+/**
+ * Cache source for log commits
+ *
+ * @extends CacheSource
+ */
+export class LogCommitsSource extends CacheSource {
+ private _count: number;
+ private _repository: Repository;
+
+ /**
+ * @param repository - An instance of a repository
+ * @param [count] - The number of commits to return
+ */
+ constructor(repository: Repository, count = 20) {
+ super(`_${repository.name.short}_${repository.branch}_commits`);
+
+ this._repository = repository;
+ this._count = count;
+ }
+
+ /**
+ * @returns An array of log commits
+ */
+ public readonly func = async(): Promise<LogCommit[]> => getLogCommits(await this._repository.commits(true));
+
+ /**
+ * @param commits - An array of log commits
+ * @returns A modified array of log commits
+ */
+ public readonly post = (commits: LogCommit[]): LogCommit[] => commits.slice(0, this._count);
+}
+
+/**
+ * Cache source for a commit
+ *
+ * @extends CacheSource
+ */
+export class CommitSource extends CacheSource {
+ private _commit: Commit;
+
+ /**
+ * @param repository - An instance of a repository
+ * @param commit - An instance of a commit
+ */
+ constructor(repository: Repository, commit: Commit) {
+ super(`_${repository.name.short}_${repository.branch}_commits_${commit.id}`);
+
+ this._commit = commit;
+ }
+
+ /**
+ * @returns An array of API commits
+ */
+ public readonly func = (): Promise<APICommit> => getCommit(this._commit);
+
+ public readonly post = undefined;
+}
+
+/**
+ * Cache source for tags
+ *
+ * @extends CacheSource
+ */
+export class TagsSource extends CacheSource {
+ private _tags: Tag[];
+
+ /**
+ * @param repository - An instance of a repository
+ * @param tags - An array of tag instances
+ */
+ constructor(repository: Repository, tags: Tag[]) {
+ super(`_${repository.name.short}_tags`);
+
+ this._tags = tags;
+ }
+
+ /**
+ * @returns An array of API tags
+ */
+ public readonly func = (): Promise<APITag[]> => getTags(this._tags);
+
+ public readonly post = undefined;
+}
+
+/**
+ * Cache source for repositories
+ *
+ * @extends CacheSource
+ */
+export class RepositoriesSource extends CacheSource {
+ private _repositories: Repository[];
+
+ /**
+ * @param repositories An array of repository instances
+ */
+ constructor(repositories: Repository[]) {
+ super();
+
+ this._repositories = repositories;
+ }
+
+ /**
+ * @returns An array of repository summaries
+ */
+ public readonly func = (): Promise<RepositorySummary[]> => getRepositories(this._repositories);
+
+ public readonly post = undefined;
+}
+
+/**
+ * Cache source for a repository
+ *
+ * @extends CacheSource
+ */
+export class RepositorySource extends CacheSource {
+ private _repository: Repository;
+
+ /**
+ * @param repository - An instance of a repository
+ */
+ constructor(repository: Repository) {
+ super(`_${repository.name.short}`);
+
+ this._repository = repository;
+ }
+
+ /**
+ * @returns A API repository
+ */
+ public readonly func = (): Promise<APIRepository> => getRepository(this._repository);
+
+ public readonly post = undefined;
+}
+
+/**
+ * Cache source for branches
+ *
+ * @extends CacheSource
+ */
+export class BranchesSource extends CacheSource {
+ private _branches: Branch[];
+
+ /**
+ * @param repository - An instance of a repository
+ * @param branches - An array of branch instances
+ */
+ constructor(repository: Repository, branches: Branch[]) {
+ super(`_${repository.name.short}_branches`);
+
+ this._branches = branches;
+ }
+
+ /**
+ * @returns An array of branch summaries
+ */
+ public readonly func = (): BranchSummary[] => getBranches(this._branches);
+
+ public readonly post = undefined;
+}
+
+/**
+ * Cache source for a branch
+ *
+ * @extends CacheSource
+ */
+export class BranchSource extends CacheSource {
+ private _branch: Branch;
+
+ /**
+ * @param repository - An instance of a repository
+ * @param branch - An instance of a branch
+ */
+ constructor(repository: Repository, branch: Branch) {
+ super(`_${repository.name.short}_branches_${branch.name}`);
+
+ this._branch = branch;
+ }
+
+ /**
+ * @returns A API branch
+ */
+ public readonly func = (): Promise<APIBranch> => getBranch(this._branch);
+
+ public readonly post = undefined;
+}
+
+/**
+ * Caches all of the available cache sources
+ *
+ * @param cache - A server cache instance
+ * @param git_dir - A git directory
+ */
+export async function cacheAllSources(cache: ServerCache, git_dir: string): Promise<void> {
+ console.log("Initializing cache... this may take a while\n");
+
+ const repositories = await Repository.openAll(git_dir);
+
+ process.stdout.write("Caching repositories... ");
+
+ await cache.receive(RepositoriesSource, repositories);
+
+ process.stdout.write("done\n\n");
+
+ for(const repository of repositories) {
+ console.log(repository.name.short);
+
+ process.stdout.write("-> Caching repository... ");
+
+ await cache.receive(RepositorySource, repository);
+
+ process.stdout.write("done\n");
+
+ process.stdout.write("-> Caching tags... ");
+
+ const tags = await repository.tags();
+
+ await cache.receive(TagsSource, repository, tags);
+
+ process.stdout.write("done\n");
+
+ process.stdout.write("-> Caching branches... ");
+
+ const branches = await repository.branches();
+
+ await cache.receive(BranchesSource, repository, branches);
+
+ process.stdout.write("done\n");
+
+ for(const branch of branches) {
+ const branch_repository = await branch.repository();
+
+ console.log(`\n-> ${branch.name}`);
+
+ process.stdout.write(" -> Caching branch... ");
+
+ await cache.receive(BranchSource, branch_repository, branch);
+
+ process.stdout.write("done\n");
+
+ process.stdout.write(" -> Caching log commits... ");
+
+ await cache.receive(LogCommitsSource, branch_repository, 0);
+
+ process.stdout.write("done\n");
+
+ const message = " -> Caching commits... ";
+ process.stdout.write(message);
+
+ const commits = await branch_repository.commits(true);
+
+ const commits_cnt = commits.length;
+ for(const commit of commits) {
+ process.stdout.clearLine(1);
+ process.stdout.cursorTo(message.length);
+ process.stdout.write(`${Math.round(commits.indexOf(commit) / commits_cnt * 100)}%`);
+
+ await cache.receive(CommitSource, branch_repository, commit);
+ }
+
+ process.stdout.clearLine(1);
+ process.stdout.cursorTo(message.length);
+ process.stdout.write("done\n");
+ }
+
+ console.log("");
+ }
+}
\ No newline at end of file
diff --git a/packages/server/src/git/branch.ts b/packages/server/src/git/branch.ts
index dacabda..85f83be 100644
--- a/packages/server/src/git/branch.ts
+++ b/packages/server/src/git/branch.ts
@@ -10,6 +10,10 @@ import { createError, ErrorWhere, FailedError, NotFoundError, UnknownError } fro
* @extends Reference
*/
export class Branch extends Reference {
+ public async repository(): Promise<Repository> {
+ return this._owner.withBranch(this.name);
+ }
+
/**
* Returns the branch's latest commit
*
@@ -41,6 +45,7 @@ export class Branch extends Reference {
}
throw(createError(ErrorWhere.Branch, UnknownError));
});
+
return new Branch(owner, reference);
}
diff --git a/packages/server/src/git/commit.ts b/packages/server/src/git/commit.ts
index 6ef02ef..7062304 100644
--- a/packages/server/src/git/commit.ts
+++ b/packages/server/src/git/commit.ts
@@ -207,21 +207,28 @@ export class Commit {
* @returns An instance of a commit
*/
public static async branchCommit(owner: Repository): Promise<Commit> {
- return new Commit(owner, await owner.ng_repository.getBranchCommit(owner.branch_name));
+ return new Commit(owner, await owner.ng_repository.getBranchCommit(owner.branch));
}
/**
* Returns a number of commits in a repository
*
* @param owner - The repository which the commits are in
- * @param [count=20] - The number of commits to get
+ * @param [amount=20] - The number of commits to get or whether or not to get all commits
* @returns An array of commit instances
*/
- public static async getMultiple(owner: Repository, count = 20): Promise<Commit[]> {
+ public static async getMultiple(owner: Repository, amount: number | boolean = 20): Promise<Commit[]> {
const walker = NodeGitRevwalk.create(owner.ng_repository);
- walker.pushRef(`refs/heads/${owner.branch_name}`);
+ walker.pushRef(`refs/heads/${owner.branch}`);
- return Promise.all((await walker.getCommits(count)).map(commit => new Commit(owner, commit)));
+ if(typeof amount === "boolean") {
+ return Promise.all((await (amount
+ ? walker.getCommitsUntil(() => true)
+ : walker.getCommits(20)
+ )).map(commit => new Commit(owner, commit)));
+ }
+
+ return Promise.all((await walker.getCommits(amount)).map(commit => new Commit(owner, commit)));
}
}
\ No newline at end of file
diff --git a/packages/server/src/git/diff.ts b/packages/server/src/git/diff.ts
index d084e5d..a2c8829 100644
--- a/packages/server/src/git/diff.ts
+++ b/packages/server/src/git/diff.ts
@@ -1,5 +1,5 @@
import { Diff as NodeGitDiff } from "nodegit";
-import { createError, ErrorWhere, NotFoundError } from "./error";
+import { createError, DiffTooLargeError, ErrorWhere, NotFoundError } from "./error";
import { Patch } from "./patch";
type PatchHeaderData = {
@@ -63,6 +63,10 @@ export class Diff {
* @returns An array of patch instances
*/
public async patches(): Promise<Patch[]> {
+ if((await this.rawPatches()).split("\n").length > 50000) {
+ throw(createError(ErrorWhere.Diff, DiffTooLargeError));
+ }
+
return (await this.ng_diff.patches()).map((patch, index) => new Patch(this, patch, index));
}
diff --git a/packages/server/src/git/error/index.ts b/packages/server/src/git/error/index.ts
index b8994d3..55a3aef 100644
--- a/packages/server/src/git/error/index.ts
+++ b/packages/server/src/git/error/index.ts
@@ -21,7 +21,8 @@ export enum ErrorWhere {
Commit = "commit",
Diff = "diff",
Misc = "misc",
- Blob = "blob"
+ Blob = "blob",
+ Patch = "patch"
}
/**
diff --git a/packages/server/src/git/error/types.ts b/packages/server/src/git/error/types.ts
index b8c860b..19ad710 100644
--- a/packages/server/src/git/error/types.ts
+++ b/packages/server/src/git/error/types.ts
@@ -42,4 +42,16 @@ export class NotInKeyringError extends ErrorType {
constructor(email: string) {
super(500, `A public key for '${email}' doesn't exist in the server pgp keyring!`);
}
+}
+
+export class PatchTooLargeError extends ErrorType {
+ constructor() {
+ super(500, "Patch is too large for parsing!");
+ }
+}
+
+export class DiffTooLargeError extends ErrorType {
+ constructor() {
+ super(500, "Diff is too large for parsing!");
+ }
}
\ No newline at end of file
diff --git a/packages/server/src/git/patch.ts b/packages/server/src/git/patch.ts
index 4239ce4..4527d03 100644
--- a/packages/server/src/git/patch.ts
+++ b/packages/server/src/git/patch.ts
@@ -1,5 +1,6 @@
import { Diff } from "./diff";
import { ConvenientPatch as NodeGitPatch } from "nodegit";
+import { createError, ErrorWhere, PatchTooLargeError } from "./error";
type Hunk = {
new_start: number,
@@ -87,10 +88,9 @@ export class Patch {
*
* These bounds are in the context of it's whole diff
*
- * @returns A patch bounds instance which contains a start & an end property
+ * @returns The patch's bounds
*/
- private async _bounds(): Promise<PatchBounds> {
- const raw_patches = (await this._diff.rawPatches()).split("\n");
+ private async _bounds(raw_patches: string[]): Promise<PatchBounds> {
const patch_header_data = await this._diff.patchHeaderData();
return {
@@ -104,7 +104,7 @@ export class Patch {
*/
private async _content(): Promise<string> {
const raw_patches = (await this._diff.rawPatches()).split("\n");
- const bounds = await this._bounds();
+ const bounds = await this._bounds(raw_patches);
return raw_patches.slice(bounds.start, bounds.end).join("\n");
}
@@ -115,10 +115,14 @@ export class Patch {
* @returns Whether or not the patch is too large
*/
public async isTooLarge(): Promise<boolean> {
+ if(this.additions > 2000 || this.deletions > 2000) {
+ return true;
+ }
+
const content = (await this._content()).split("\n");
const line_lengths = content.map(line => line.length).reduce((result, length) => result + length);
- if(content.length > 5000 || line_lengths > 5000) {
+ if(content.length > 10000 || line_lengths > 10000) {
return true;
}
@@ -131,6 +135,10 @@ export class Patch {
* @returns An array of hunk instances
*/
public async getHunks(): Promise<Hunk[]> {
+ if(await this.isTooLarge()) {
+ throw(createError(ErrorWhere.Patch, PatchTooLargeError));
+ }
+
const content = (await this._content()).split("\n");
const hunks = await this._ng_patch.hunks();
diff --git a/packages/server/src/git/repository.ts b/packages/server/src/git/repository.ts
index c1410ab..53245be 100644
--- a/packages/server/src/git/repository.ts
+++ b/packages/server/src/git/repository.ts
@@ -38,7 +38,7 @@ export class Repository {
public name: RepositoryName;
public git_dir: string;
- public branch_name: string;
+ private _branch: string;
/**
* @param repository - An instance of a Nodegit repository
@@ -52,7 +52,7 @@ export class Repository {
};
this.git_dir = dirname(repository.path());
- this.branch_name = branch;
+ this._branch = branch;
}
/**
@@ -69,23 +69,31 @@ export class Repository {
return getFile(this.git_dir, this.name.full, "owner");
}
+ get branch(): string {
+ return this._branch;
+ }
+
+ set branch(branch: string) {
+ this._branch = branch;
+ }
+
/**
* Returns the repository's branch
*
* @returns An instance of a branch
*/
- public branch(): Promise<Branch> {
- return Branch.lookup(this, this.branch_name);
+ public getBranch(): Promise<Branch> {
+ return Branch.lookup(this, this._branch);
}
/**
* Returns the repository's commits
*
- * @param [count=20] - The number of commits to get
+ * @param [amount=20] - The number of commits to get or whether or not to get all commits
* @returns An array of commit instances
*/
- public async commits(count?: number): Promise<Commit[]> {
- return Commit.getMultiple(this, count);
+ public async commits(amount?: number | boolean): Promise<Commit[]> {
+ return Commit.getMultiple(this, amount);
}
/**
@@ -118,6 +126,20 @@ export class Repository {
.catch(() => false);
}
+ /**
+ * Returns this repository instance with a different branch
+ *
+ * @param branch - The branch to switch to
+ * @returns An instance of a repository
+ */
+ public async withBranch(branch: string): Promise<Repository> {
+ if(!await Branch.lookupExists(this.ng_repository, branch)) {
+ throw(createError(ErrorWhere.Repository, NotFoundError, "branch"));
+ }
+
+ return new Repository(this.ng_repository, branch);
+ }
+
/**
* Returns the repository's branches
*
diff --git a/packages/server/src/git/tree_entry.ts b/packages/server/src/git/tree_entry.ts
index b03ea9e..cdcb0d3 100644
--- a/packages/server/src/git/tree_entry.ts
+++ b/packages/server/src/git/tree_entry.ts
@@ -31,11 +31,11 @@ export abstract class BaseTreeEntry {
*/
public async latestCommit(): Promise<Commit> {
const rev_walk = NodeGitRevwalk.create(this._owner.ng_repository);
- rev_walk.pushRef(`refs/heads/${this._owner.branch_name}`);
+ rev_walk.pushRef(`refs/heads/${this._owner.branch}`);
const commit_cnt = (await rev_walk.getCommitsUntil(() => true)).length;
- rev_walk.pushRef(`refs/heads/${this._owner.branch_name}`);
+ rev_walk.pushRef(`refs/heads/${this._owner.branch}`);
const file_hist = await rev_walk.fileHistoryWalk(this.path, commit_cnt);
return new Commit(this._owner, file_hist[0].commit);
@@ -48,11 +48,11 @@ export abstract class BaseTreeEntry {
*/
public async history(count?: number): Promise<Commit[]> {
const rev_walk = NodeGitRevwalk.create(this._owner.ng_repository);
- rev_walk.pushRef(`refs/heads/${this._owner.branch_name}`);
+ rev_walk.pushRef(`refs/heads/${this._owner.branch}`);
const commit_cnt = (await rev_walk.getCommitsUntil(() => true)).length;
- rev_walk.pushRef(`refs/heads/${this._owner.branch_name}`);
+ rev_walk.pushRef(`refs/heads/${this._owner.branch}`);
const file_hist = await rev_walk.fileHistoryWalk(this.path, commit_cnt);
const commit_history = await Promise.all(file_hist.map(async hist_entry => new Commit(this._owner, await NodeGitCommit.lookup(this._owner.ng_repository, hist_entry.commit))));
diff --git a/packages/server/src/routes/api/v1/data.ts b/packages/server/src/routes/api/v1/data.ts
new file mode 100644
index 0000000..97f07ff
--- /dev/null
+++ b/packages/server/src/routes/api/v1/data.ts
@@ -0,0 +1,119 @@
+import {
+ LogCommit,
+ Patch as APIPatch,
+ Commit as APICommit, Tag as APITag,
+ RepositorySummary as APIRepositorySummary,
+ Repository as APIRepository,
+ BranchSummary,
+ Branch as APIBranch
+} from "api";
+import { Branch, Commit, Patch, Repository, Tag } from "../../../git";
+
+export async function getLogCommits(commits: Commit[]): Promise<LogCommit[]> {
+ return Promise.all(commits.map(async(commit: Commit) => {
+ const stats = await commit.stats();
+
+ const is_signed = await commit.isSigned();
+
+ return <LogCommit>{
+ id: commit.id,
+ author: {
+ name: commit.author().name,
+ email: commit.author().email,
+ fingerprint: await commit.author().fingerprint().catch(() => null)
+ },
+ isSigned: is_signed,
+ signatureVerified: is_signed ? await commit.verifySignature().catch(() => false) : null,
+ message: commit.message,
+ date: commit.date,
+ insertions: stats.insertions,
+ deletions: stats.deletions,
+ files_changed: stats.files_changed
+ };
+ }));
+}
+
+export async function getCommit(commit: Commit): Promise<APICommit> {
+ const stats = await commit.stats();
+
+ const is_signed = await commit.isSigned();
+
+ const patches = await (await commit.diff()).patches().catch(() => null);
+
+ return {
+ message: commit.message,
+ author: {
+ name: commit.author().name,
+ email: commit.author().email,
+ fingerprint: await commit.author().fingerprint().catch(() => null)
+ },
+ isSigned: is_signed,
+ signatureVerified: is_signed ? await commit.verifySignature().catch(() => false) : null,
+ date: commit.date,
+ insertions: stats.insertions,
+ deletions: stats.deletions,
+ files_changed: stats.files_changed,
+ too_large: Boolean(!patches),
+ diff: patches
+ ? await Promise.all(patches.map(async(patch: Patch) => {
+ return <APIPatch>{
+ additions: patch.additions,
+ deletions: patch.deletions,
+ from: patch.from,
+ to: patch.to,
+ too_large: await patch.isTooLarge(),
+ hunks: await patch.getHunks().catch(() => null)
+ };
+ }))
+ : null
+ };
+}
+
+export function getTags(tags: Tag[]): Promise<APITag[]> {
+ return Promise.all(tags.map(async(tag: Tag) => {
+ const author = await tag.author();
+ return <APITag>{
+ name: tag.name,
+ author: {
+ name: author.name,
+ email: author.email
+ },
+ date: await tag.date()
+ };
+ }));
+}
+
+export function getRepositories(repositories: Repository[]): Promise<APIRepositorySummary[]> {
+ return Promise.all(repositories.map(async repository => {
+ return <APIRepositorySummary>{
+ name: repository.name.short,
+ description: await repository.description(),
+ last_updated: (await repository.head()).date
+ };
+ }));
+}
+
+export async function getRepository(repository: Repository): Promise<APIRepository> {
+ return <APIRepository>{
+ name: repository.name.short,
+ description: await repository.description(),
+ has_readme: await (await repository.tree()).findExists("README.md")
+ };
+}
+
+export function getBranches(branches: Branch[]): BranchSummary[] {
+ return branches.map(branch => {
+ return <BranchSummary>{
+ id: branch.id,
+ name: branch.name
+ };
+ });
+}
+
+export async function getBranch(branch: Branch): Promise<APIBranch> {
+ return {
+ id: branch.id,
+ name: branch.name,
+ latest_commit: await branch.latestCommit()
+ };
+}
\ No newline at end of file
diff --git a/packages/server/src/routes/api/v1/index.ts b/packages/server/src/routes/api/v1/index.ts
index 4b63435..7997b4d 100644
--- a/packages/server/src/routes/api/v1/index.ts
+++ b/packages/server/src/routes/api/v1/index.ts
@@ -1,47 +1,24 @@
-import { FastifyInstance, FastifyPluginOptions } from "fastify";
+import { FastifyPluginCallback } from "fastify";
import { Repository } from "../../../git/repository";
-import { Route } from "../../../types/fastify";
+import { FastifyPluginOptions, Route } from "../../../types/fastify";
import repo from "./repo";
import { verifyRepoName } from "../util";
-import { Info as APIInfo, RepositorySummary as APIRepositorySummary, Repository as APIRepository } from "api";
+import { Info as APIInfo } from "api";
import { ServerError } from "../../../git/error";
+import { getRepositories, getRepository } from "./data";
+import { sources } from "../../../cache";
-function setHandlers(fastify: FastifyInstance): void {
- fastify.setErrorHandler((err, req, reply) => {
- if(err.validation) {
- reply.code(400).send({ error: `${err.validation[0].dataPath} ${err.validation[0].message}` });
- return;
- }
-
- console.log(err);
-
- reply.code(500).send({ error: "Internal server error!" });
- });
- fastify.setNotFoundHandler((req, reply) => {
- reply.code(404).send({ error: "Endpoint not found!" });
- });
-}
-
-function reposEndpoints(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void {
+const reposEndpoints: FastifyPluginCallback<FastifyPluginOptions> = (fastify, opts, done) => {
fastify.route({
method: "GET",
url: "/repos",
handler: async(req, reply) => {
- const repos = await Repository.openAll(opts.config.settings.git_dir);
-
- if(!repos) {
- reply.send({ data: [] });
- return;
- }
+ const repositories = await Repository.openAll(opts.config.settings.git_dir);
reply.send({
- data: await Promise.all(repos.map(async repository => {
- return <APIRepositorySummary>{
- name: repository.name.short,
- description: await repository.description(),
- last_updated: (await repository.head()).date
- };
- }))
+ data: await (opts.config.cache
+ ? opts.config.cache.receive(sources.RepositoriesSource, repositories)
+ : getRepositories(repositories))
});
}
});
@@ -67,20 +44,30 @@ function reposEndpoints(fastify: FastifyInstance, opts: FastifyPluginOptions, do
return;
}
- const data: APIRepository = {
- name: repository.name.short,
- description: await repository.description(),
- has_readme: await (await repository.tree()).findExists("README.md")
- };
-
- reply.send({ data: data });
+ reply.send({
+ data: await (opts.config.cache
+ ? opts.config.cache.receive(sources.RepositorySource, repository)
+ : getRepository(repository))
+ });
}
});
done();
-}
+};
+
+const api: FastifyPluginCallback<FastifyPluginOptions> = (fastify, opts, done) => {
+ fastify.setErrorHandler((err, req, reply) => {
+ if(err.validation) {
+ reply.code(400).send({ error: `${err.validation[0].dataPath} ${err.validation[0].message}` });
+ return;
+ }
-export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void {
- setHandlers(fastify);
+ console.log(err);
+
+ reply.code(500).send({ error: "Internal server error!" });
+ });
+ fastify.setNotFoundHandler((req, reply) => {
+ reply.code(404).send({ error: "Endpoint not found!" });
+ });
fastify.route({
method: "GET",
@@ -95,8 +82,10 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do
}
});
- fastify.register(reposEndpoints, { config: { settings: opts.config.settings } });
- fastify.register(repo, { prefix: "/repos/:repo", config: { settings: opts.config.settings } });
+ fastify.register(reposEndpoints, { config: opts.config });
+ fastify.register(repo, { prefix: "/repos/:repo", config: opts.config });
done();
-}
\ No newline at end of file
+};
+
+export default api;
\ No newline at end of file
diff --git a/packages/server/src/routes/api/v1/repo/branches.ts b/packages/server/src/routes/api/v1/repo/branches.ts
index 99f0327..f709f4d 100644
--- a/packages/server/src/routes/api/v1/repo/branches.ts
+++ b/packages/server/src/routes/api/v1/repo/branches.ts
@@ -1,9 +1,10 @@
-import { FastifyInstance, FastifyPluginOptions } from "fastify";
+import { FastifyPluginCallback } from "fastify";
+import { sources } from "../../../../cache";
import { Branch } from "../../../../git/branch";
-import { Route } from "../../../../types/fastify";
-import { BranchSummary as APIBranchSummary, Branch as APIBranch } from "api";
+import { FastifyPluginOptions, Route } from "../../../../types/fastify";
+import { getBranch, getBranches } from "../data";
-export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void {
+const branches: FastifyPluginCallback<FastifyPluginOptions> = (fastify, opts, done) => {
fastify.route<Route>({
method: "GET",
url: "/branches",
@@ -11,12 +12,9 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do
const branches = await req.repository.branches();
reply.send({
- data: branches.map(branch => {
- return <APIBranchSummary>{
- id: branch.id,
- name: branch.name
- };
- })
+ data: opts.config.cache
+ ? await opts.config.cache.receive(sources.BranchesSource, req.repository, branches)
+ : getBranches(branches)
});
}
});
@@ -32,22 +30,15 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do
handler: async(req, reply) => {
const branch = await Branch.lookup(req.repository, req.params.branch);
- if(!branch) {
- reply.code(404).send({ error: "Branch not found!" });
- return;
- }
-
- const data: APIBranch = {
- id: branch.id,
- name: branch.name,
- latest_commit: await branch.latestCommit()
- };
-
reply.send({
- data: data
+ data: await (opts.config.cache
+ ? opts.config.cache.receive(sources.BranchSource, req.repository, branch)
+ : getBranch(branch))
});
}
});
done();
-}
\ No newline at end of file
+};
+
+export default branches;
\ No newline at end of file
diff --git a/packages/server/src/routes/api/v1/repo/index.ts b/packages/server/src/routes/api/v1/repo/index.ts
index 4cd6c51..f8e01d3 100644
--- a/packages/server/src/routes/api/v1/repo/index.ts
+++ b/packages/server/src/routes/api/v1/repo/index.ts
@@ -1,15 +1,15 @@
-import { CoolFastifyRequest, Route } from "../../../../types/fastify";
-import { FastifyInstance, FastifyPluginOptions } from "fastify";
+import { CoolFastifyRequest, Route, FastifyPluginOptions } from "../../../../types/fastify";
+import { FastifyInstance, FastifyPluginCallback } from "fastify";
import { Repository } from "../../../../git/repository";
-import { Tag } from "../../../../git/tag";
import { BaseTreeEntry, BlobTreeEntry, TreeEntry } from "../../../../git/tree_entry";
import { basename } from "path";
import branches from "./branches";
import log from "./log";
import { verifyRepoName } from "../../util";
-import { Tree as APITree, Tag as APITag, TreeEntry as APITreeEntry } from "api";
+import { Tree as APITree, TreeEntry as APITreeEntry } from "api";
import { ServerError } from "../../../../git/error";
-import { commitMap } from "./map";
+import { getLogCommits, getTags } from "../data";
+import { sources } from "../../../../cache";
declare module "fastify" {
interface FastifyRequest {
@@ -48,20 +48,11 @@ async function treeEntryMap(entry: BaseTreeEntry) {
};
}
-async function tagMap(tag: Tag) {
- const author = await tag.author();
- return <APITag>{
- name: tag.name,
- author: { name: author.name, email: author.email },
- date: await tag.date()
- };
-}
-
-export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void {
+const repo: FastifyPluginCallback<FastifyPluginOptions> = (fastify, opts, done) => {
addHooks(fastify, opts);
- fastify.register(log);
- fastify.register(branches);
+ fastify.register(log, { config: opts.config });
+ fastify.register(branches, { config: opts.config });
fastify.route<Route>({
method: "GET",
@@ -127,7 +118,7 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do
const history = await tree_entry.history(Number(req.query.count));
- reply.send({ data: await Promise.all(history.map(commitMap)) });
+ reply.send({ data: await getLogCommits(history) });
}
});
@@ -137,10 +128,14 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do
handler: async(req, reply) => {
const tags = await req.repository.tags();
reply.send({
- data: await Promise.all(tags.map(tagMap))
+ data: await (opts.config.cache
+ ? opts.config.cache.receive(sources.TagsSource, req.repository, tags)
+ : getTags(tags))
});
}
});
done();
-}
\ No newline at end of file
+};
+
+export default repo;
\ No newline at end of file
diff --git a/packages/server/src/routes/api/v1/repo/log.ts b/packages/server/src/routes/api/v1/repo/log.ts
index 163cf80..7ad1e11 100644
--- a/packages/server/src/routes/api/v1/repo/log.ts
+++ b/packages/server/src/routes/api/v1/repo/log.ts
@@ -1,23 +1,11 @@
-import { FastifyInstance, FastifyPluginOptions } from "fastify";
+import { FastifyPluginCallback } from "fastify";
+import { sources } from "../../../../cache";
import { Commit } from "../../../../git/commit";
-import { Patch } from "../../../../git/patch";
-import { Route } from "../../../../types/fastify";
+import { Route, FastifyPluginOptions } from "../../../../types/fastify";
import { verifySHA } from "../../util";
-import { Patch as APIPatch, Commit as APICommit } from "api";
-import { commitMap } from "./map";
+import { getCommit, getLogCommits } from "../data";
-async function patchMap(patch: Patch) {
- return <APIPatch>{
- additions: patch.additions,
- deletions: patch.deletions,
- from: patch.from,
- to: patch.to,
- too_large: await patch.isTooLarge(),
- hunks: await patch.getHunks()
- };
-}
-
-export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void {
+const log: FastifyPluginCallback<FastifyPluginOptions> = (fastify, opts, done) => {
fastify.route<Route>({
method: "GET",
url: "/log",
@@ -27,10 +15,12 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do
}
},
handler: async(req, reply) => {
- const commits = await req.repository.commits(Number(req.query.count));
+ const commits = await req.repository.commits(Number(req.query.count) || undefined);
reply.send({
- data: await Promise.all(commits.map(commitMap))
+ data: await (opts.config.cache
+ ? opts.config.cache.receive(sources.LogCommitsSource, req.repository, Number(req.query.count) || undefined)
+ : getLogCommits(commits))
});
}
});
@@ -51,31 +41,15 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do
const commit = await Commit.lookup(req.repository, req.params.commit);
- const stats = await commit.stats();
-
- const is_signed = await commit.isSigned();
-
- const data: APICommit = {
- message: commit.message,
- author: {
- name: commit.author().name,
- email: commit.author().email,
- fingerprint: await commit.author().fingerprint().catch(() => null)
- },
- isSigned: is_signed,
- signatureVerified: is_signed ? await commit.verifySignature().catch(() => false) : null,
- date: commit.date,
- insertions: stats.insertions,
- deletions: stats.deletions,
- files_changed: stats.files_changed,
- diff: await Promise.all((await (await commit.diff()).patches()).map(patchMap))
- };
-
reply.send({
- data: data
+ data: await (opts.config.cache
+ ? opts.config.cache.receive(sources.CommitSource, req.repository, commit)
+ : getCommit(commit))
});
}
});
done();
-}
\ No newline at end of file
+};
+
+export default log;
\ No newline at end of file
diff --git a/packages/server/src/routes/api/v1/repo/map.ts b/packages/server/src/routes/api/v1/repo/map.ts
deleted file mode 100644
index a544d1a..0000000
--- a/packages/server/src/routes/api/v1/repo/map.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-import { Commit } from "../../../../git/commit";
-import { LogCommit } from "api";
-
-export async function commitMap(commit: Commit): Promise<LogCommit> {
- const stats = await commit.stats();
-
- const is_signed = await commit.isSigned();
-
- return <LogCommit>{
- id: commit.id,
- author: {
- name: commit.author().name,
- email: commit.author().email,
- fingerprint: await commit.author().fingerprint().catch(() => null)
- },
- isSigned: is_signed,
- signatureVerified: is_signed ? await commit.verifySignature().catch(() => false) : null,
- message: commit.message,
- date: commit.date,
- insertions: stats.insertions,
- deletions: stats.deletions,
- files_changed: stats.files_changed
- };
-}
\ No newline at end of file
diff --git a/packages/server/src/routes/repo.ts b/packages/server/src/routes/repo.ts
index bb70c68..1088e6b 100644
--- a/packages/server/src/routes/repo.ts
+++ b/packages/server/src/routes/repo.ts
@@ -1,11 +1,11 @@
import { Repository } from "../git/repository";
-import { CoolFastifyRequest, Route } from "../types/fastify";
+import { CoolFastifyRequest, Route, FastifyPluginOptions } from "../types/fastify";
import { Tag } from "../git/tag";
-import { FastifyInstance, FastifyPluginOptions } from "fastify";
+import { FastifyPluginCallback } from "fastify";
import { verifyRepoName } from "../routes/api/util";
import { ServerError } from "../git/error";
-export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, done: (err?: Error) => void): void {
+const repo: FastifyPluginCallback<FastifyPluginOptions> = (fastify, opts, done): void => {
fastify.addHook("onRequest", async(req: CoolFastifyRequest, reply) => {
if(!verifyRepoName(req.params.repo)) {
reply.code(400).send("Bad request");
@@ -86,4 +86,6 @@ export default function(fastify: FastifyInstance, opts: FastifyPluginOptions, do
});
done();
-}
\ No newline at end of file
+};
+
+export default repo;
\ No newline at end of file
diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts
index 11b3f7f..3056bf5 100644
--- a/packages/server/src/server.ts
+++ b/packages/server/src/server.ts
@@ -1,41 +1,55 @@
-import { readFileSync, readdirSync } from "fs";
+import { readFile, readdir } from "fs/promises";
import { join } from "path";
import { exit } from "process";
import { Settings } from "./types";
import buildApp from "./app";
+import { ServerCache } from "./cache";
-const settings = JSON.parse(readFileSync(join(__dirname, "/../../../settings.json"), "utf-8")) as Settings;
+async function main() {
+ const settings = JSON.parse(await readFile(join(__dirname, "/../../../settings.json"), { encoding: "utf-8" })) as Settings;
-const settings_keys = Object.keys(settings);
+ const settings_keys = Object.keys(settings);
-const mandatory_settings = [ "host", "port", "title", "about", "git_dir" ];
+ const mandatory_settings = [ "host", "port", "title", "about", "git_dir" ];
-// Get missing mandatory settings
-const settings_not_included = mandatory_settings.filter(x => !settings_keys.includes(x));
+ // Get missing mandatory settings
+ const settings_not_included = mandatory_settings.filter(x => !settings_keys.includes(x));
-// Error out and exit if there's any missing settings
-if(settings_not_included.length !== 0) {
- console.log(`Error: settings file is missing ${(settings_not_included.length > 1) ? "keys" : "key"}:`);
- console.log(settings_not_included.join(", "));
- exit(1);
-}
-
-// Make sure that the git directory specified in the settings actually exists
-try {
- readdirSync(settings.git_dir);
-}
-catch {
- console.error(`Error: Git directory ${settings.git_dir} doesn't exist!`);
- exit(1);
-}
-
-const app = buildApp(settings);
+ // Error out and exit if there's any missing settings
+ if(settings_not_included.length !== 0) {
+ console.log(`Error: settings file is missing ${(settings_not_included.length > 1) ? "keys" : "key"}:`);
+ console.log(settings_not_included.join(", "));
+ exit(1);
+ }
-app.listen(settings.port, settings.host, (err: Error, addr: string) => {
- if(err) {
- console.error(err);
+ // Make sure that the git directory specified in the settings actually exists
+ await readdir(settings.git_dir).catch(() => {
+ console.error(`Error: Git directory ${settings.git_dir} doesn't exist!`);
exit(1);
+ });
+
+ const cache = (settings.cache && settings.cache.enabled === true) || settings.cache === undefined || settings.cache.enabled === undefined
+ ? new ServerCache(settings.cache)
+ : null;
+
+ if(cache) {
+ await cache.init(settings.git_dir);
+ if(!cache.ready) {
+ console.error("Error: cache failed to initialize!");
+ return 1;
+ }
}
- console.log(`Githermit is running on ${addr}`);
-});
\ No newline at end of file
+ const app = await buildApp(settings, cache);
+
+ app.listen(settings.port, settings.host, (err: Error, addr: string) => {
+ if(err) {
+ console.error(err);
+ exit(1);
+ }
+
+ console.log(`Githermit is running on ${addr}`);
+ });
+}
+
+main();
\ No newline at end of file
diff --git a/packages/server/src/types/fastify.d.ts b/packages/server/src/types/fastify.d.ts
index ebaaac2..7c2341d 100644
--- a/packages/server/src/types/fastify.d.ts
+++ b/packages/server/src/types/fastify.d.ts
@@ -1,5 +1,7 @@
import { FastifyRequest, RequestGenericInterface } from "fastify";
import { ReplyGenericInterface } from "fastify/types/reply";
+import { Settings } from ".";
+import { ServerCache } from "../cache";
export interface Request extends RequestGenericInterface {
Params: Record<string, string>,
@@ -8,4 +10,11 @@ export interface Request extends RequestGenericInterface {
export interface Route extends Request, ReplyGenericInterface {}
-export type CoolFastifyRequest = FastifyRequest<Route>;
\ No newline at end of file
+export type CoolFastifyRequest = FastifyRequest<Route>;
+
+export type FastifyPluginOptions = {
+ config: {
+ settings: Settings,
+ cache: ServerCache | null
+ }
+}
\ No newline at end of file
diff --git a/packages/server/src/types/index.d.ts b/packages/server/src/types/index.d.ts
index 8f592f9..ca0743d 100644
--- a/packages/server/src/types/index.d.ts
+++ b/packages/server/src/types/index.d.ts
@@ -1,10 +1,18 @@
+export type CacheConfig = {
+ enabled: boolean,
+ ttl?: number,
+ max?: number,
+ refreshThreshold?: number
+}
+
export type Settings = {
host: string,
port: number,
title: string,
about: string,
git_dir: string,
- dev: {
+ cache?: CacheConfig,
+ dev?: {
port: number
}
}
\ No newline at end of file
diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json
index a98cc40..92c94fc 100644
--- a/packages/server/tsconfig.json
+++ b/packages/server/tsconfig.json
@@ -3,4 +3,4 @@
"compilerOptions": {
"module": "CommonJS",
}
-}
+}
\ No newline at end of file
--
cgit v1.2.3-18-g5258