From c1935fb36025099ed8d3d75f1508724dabc4019f Mon Sep 17 00:00:00 2001 From: JOLIMAITRE Matthieu Date: Sun, 14 Apr 2024 16:28:13 +0200 Subject: [PATCH 1/3] source refactor --- build.sh | 2 +- main.ts | 138 ---------------------------------------------- src/lib/client.ts | 36 ++++++++++++ src/lib/crypto.ts | 20 +++++++ src/lib/parser.ts | 17 ++++++ src/lib/utils.ts | 10 ++++ src/okimeter.ts | 53 ++++++++++++++++++ 7 files changed, 137 insertions(+), 139 deletions(-) delete mode 100755 main.ts create mode 100644 src/lib/client.ts create mode 100644 src/lib/crypto.ts create mode 100644 src/lib/parser.ts create mode 100644 src/lib/utils.ts create mode 100755 src/okimeter.ts diff --git a/build.sh b/build.sh index f5558cf..868cbc9 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,7 @@ set -e cd "$(dirname "$(realpath "$0")")" -MAIN="main.ts" +MAIN="src/okimeter.ts" ARGS="--allow-net" BIN="okimeter" diff --git a/main.ts b/main.ts deleted file mode 100755 index 4467e1a..0000000 --- a/main.ts +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/env -S deno run -A - -import { encodeBase64 } from "https://deno.land/std@0.222.1/encoding/base64.ts"; -import { assert } from "https://deno.land/std@0.222.1/assert/assert.ts"; -import { assertEquals } from "https://deno.land/std@0.222.1/assert/assert_equals.ts"; -import { crayon } from "https://deno.land/x/crayon@3.3.3/mod.ts"; -import { z } from "https://deno.land/x/zod@v3.22.4/mod.ts"; - -async function main() { - let { token } = parse_args(Deno.args); - if (token === undefined) token = prompt_args(); - const client = new ApiClient("gitlab.cri.epita.fr", token); - const merge_requests = await get_all_merge_requests(client); - const total = sum(await parallel(merge_requests, (mr) => count_in_mr(mr, client))); - console.log("Total", total, "oki"); -} - -function parse_args(args: string[]) { - const [token_part] = args; - const token = token_part as string | undefined; - return { token }; -} - -function prompt_args() { - const [bl, gr] = [crayon.blue, crayon.lightBlack]; - const message = ` -${bl("Please enter a read-allowed gitlab token :")} -${gr("You can get one from https://gitlab.cri.epita.fr/-/user_settings/personal_access_tokens")} -${gr("with at least the permission 'read_api', format is :")} -${gr("> xxxxxxxxxxxxxxxxxxxx")} ->`.trim(); - const result = prompt(message); - assert(result !== null); - return result.trim(); -} - -class ApiClient { - url; - token; - constructor(hostname: string, token: string) { - this.url = `https://${hostname}`; - this.token = token; - } - - async fetch(path: string) { - console.log(crayon.lightBlack("\tfetching", crayon.bold(path))); - const headers = new Headers([["PRIVATE-TOKEN", this.token]]); - const result = await fetch(this.url + "/" + path, { headers }); - const parsed = await result.json(); - return parsed as unknown; - } - - async fetch_parse(path: string, parser: z.ZodType) { - const unkn = await this.fetch(path); - return parser.parse(unkn); - } - - async fetch_parse_paginated(path: string, parser: z.ZodType) { - const result = [] as T[]; - let page = 1; - while (true) { - const fetched = await this.fetch_parse(path + "?per_page=100&page=" + page, parser.array()); - if (fetched.length === 0) break; - result.push(...fetched); - page += 1; - } - return result; - } -} - -type MergeRequest = z.infer; -const merge_request_parser = z.object({ - iid: z.number(), - project_id: z.number(), - reviewers: z.array(z.object({ - username: z.string(), - })), -}); - -async function get_all_merge_requests(client: ApiClient): Promise { - return await client.fetch_parse_paginated(`api/v4/merge_requests`, merge_request_parser); -} - -function sum(numbers: Iterable) { - let result = 0; - for (const item of numbers) result += item; - return result; -} - -Deno.test("test_sum", () => { - assertEquals(sum([1, 2, 3]), 6); -}); - -async function parallel(inputs: I[], operation: (item: I) => Promise) { - const promises = [] as Promise[]; - for (const input of inputs) promises.push(operation(input)); - return await Promise.all(promises); -} - -async function count_in_mr(mr: MergeRequest, client: ApiClient) { - let total = 0; - const mr_path = `api/v4/projects/${mr.project_id}/merge_requests/${mr.iid}`; - const all_notes = await client.fetch_parse_paginated(`${mr_path}/notes`, note_parser); - for (const c of all_notes) { - if (!await is_relevant(c.author.username)) continue; - if (c.body.toLowerCase().includes("oki")) total += 1; - } - return total; -} - -const note_parser = z.object({ - body: z.string(), - author: z.object({ - username: z.string(), - }), -}); - -async function hash(namespace: string) { - const buffer = new TextEncoder().encode(namespace); - const hash = await crypto.subtle.digest("SHA-512", buffer); - const base64 = encodeBase64(hash); - return base64; -} - -async function is_relevant(text: string) { - return (await hash(text)) === relevant_hash; -} - -const relevant_hash = "Qa4xy8yRPnBsQf8U2UUE7tse2u4Fi/Aqxbt" + - "6aq56m5ofcmCkTi8PbgMBF1OtHC6qkXDF2tz2qKprYCIAMzTSQQ=="; - -Deno.test("encode_namespace", async () => { - const namespace = "CHANGEME"; - const base64 = await hash(namespace); - console.log({ namespace, base64 }); -}); - -if (import.meta.main) await main(); diff --git a/src/lib/client.ts b/src/lib/client.ts new file mode 100644 index 0000000..91fcc58 --- /dev/null +++ b/src/lib/client.ts @@ -0,0 +1,36 @@ +import { crayon } from "https://deno.land/x/crayon@3.3.3/mod.ts"; +import { z } from "https://deno.land/x/zod@v3.22.4/mod.ts"; + +export class ApiClient { + url; + token; + constructor(hostname: string, token: string) { + this.url = `https://${hostname}`; + this.token = token; + } + + async fetch(path: string) { + console.log(crayon.lightBlack("\tfetching", crayon.bold(path))); + const headers = new Headers([["PRIVATE-TOKEN", this.token]]); + const result = await fetch(this.url + "/" + path, { headers }); + const parsed = await result.json(); + return parsed as unknown; + } + + async fetch_parse(path: string, parser: z.ZodType) { + const unkn = await this.fetch(path); + return parser.parse(unkn); + } + + async fetch_parse_paginated(path: string, parser: z.ZodType) { + const result = [] as T[]; + let page = 1; + while (true) { + const fetched = await this.fetch_parse(path + "?per_page=100&page=" + page, parser.array()); + result.push(...fetched); + if (fetch.length < 100) break; + page += 1; + } + return result; + } +} diff --git a/src/lib/crypto.ts b/src/lib/crypto.ts new file mode 100644 index 0000000..218910d --- /dev/null +++ b/src/lib/crypto.ts @@ -0,0 +1,20 @@ +import { encodeBase64 } from "https://deno.land/std@0.222.1/encoding/base64.ts"; + +export async function is_relevant(text: string) { + return (await hash(text)) === relevant_hash; +} + +async function hash(namespace: string) { + const buffer = new TextEncoder().encode(namespace); + const hash = await crypto.subtle.digest("SHA-512", buffer); + const base64 = encodeBase64(hash); + return base64; +} + +const relevant_hash = "Qa4xy8yRPnBsQf8U2UUE7tse2u4Fi/Aqxbt" + + "6aq56m5ofcmCkTi8PbgMBF1OtHC6qkXDF2tz2qKprYCIAMzTSQQ=="; +Deno.test("encode_namespace", async () => { + const namespace = "CHANGEME"; + const base64 = await hash(namespace); + console.log({ namespace, base64 }); +}); diff --git a/src/lib/parser.ts b/src/lib/parser.ts new file mode 100644 index 0000000..8dd7572 --- /dev/null +++ b/src/lib/parser.ts @@ -0,0 +1,17 @@ +import { z } from "https://deno.land/x/zod@v3.22.4/mod.ts"; + +export type MergeRequest = z.infer; +export const merge_request_parser = z.object({ + iid: z.number(), + project_id: z.number(), + reviewers: z.array(z.object({ + username: z.string(), + })), +}); + +export const note_parser = z.object({ + body: z.string(), + author: z.object({ + username: z.string(), + }), +}); diff --git a/src/lib/utils.ts b/src/lib/utils.ts new file mode 100644 index 0000000..c4f0259 --- /dev/null +++ b/src/lib/utils.ts @@ -0,0 +1,10 @@ +export function sum(numbers: Iterable) { + let result = 0; + for (const item of numbers) result += item; + return result; +} +export async function parallel(inputs: I[], operation: (item: I) => Promise) { + const promises = [] as Promise[]; + for (const input of inputs) promises.push(operation(input)); + return await Promise.all(promises); +} diff --git a/src/okimeter.ts b/src/okimeter.ts new file mode 100755 index 0000000..a336742 --- /dev/null +++ b/src/okimeter.ts @@ -0,0 +1,53 @@ +#!/bin/env -S deno run -A + +import { assert } from "https://deno.land/std@0.222.1/assert/assert.ts"; +import { crayon } from "https://deno.land/x/crayon@3.3.3/mod.ts"; +import { is_relevant } from "./lib/crypto.ts"; +import { ApiClient } from "./lib/client.ts"; +import { parallel, sum } from "./lib/utils.ts"; +import { merge_request_parser, MergeRequest, note_parser } from "./lib/parser.ts"; + +async function main() { + let { token } = parse_args(Deno.args); + if (token === undefined) token = prompt_args(); + const client = new ApiClient("gitlab.cri.epita.fr", token); + const merge_requests = await get_all_merge_requests(client); + const total = sum(await parallel(merge_requests, (mr) => count_in_mr(mr, client))); + console.log("Total", total, "oki"); +} + +function parse_args(args: string[]) { + const [token_part] = args; + const token = token_part as string | undefined; + return { token }; +} + +function prompt_args() { + const [bl, gr] = [crayon.blue, crayon.lightBlack]; + const message = ` +${bl("Please enter a read-allowed gitlab token :")} +${gr("You can get one from https://gitlab.cri.epita.fr/-/user_settings/personal_access_tokens")} +${gr("with at least the permission 'read_api', format is :")} +${gr("> xxxxxxxxxxxxxxxxxxxx")} +>`.trim(); + const result = prompt(message); + assert(result !== null); + return result.trim(); +} + +async function get_all_merge_requests(client: ApiClient): Promise { + return await client.fetch_parse_paginated(`api/v4/merge_requests`, merge_request_parser); +} + +async function count_in_mr(mr: MergeRequest, client: ApiClient) { + let total = 0; + const mr_path = `api/v4/projects/${mr.project_id}/merge_requests/${mr.iid}`; + const all_notes = await client.fetch_parse_paginated(`${mr_path}/notes`, note_parser); + for (const c of all_notes) { + if (!await is_relevant(c.author.username)) continue; + if (c.body.toLowerCase().includes("oki")) total += 1; + } + return total; +} + +if (import.meta.main) await main(); From 90a7af344eeecd705cc277061a171606fe770ea1 Mon Sep 17 00:00:00 2001 From: JOLIMAITRE Matthieu Date: Mon, 15 Apr 2024 10:19:38 +0200 Subject: [PATCH 2/3] code format & renaming --- src/lib/crypto.ts | 1 + src/okimeter.ts | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lib/crypto.ts b/src/lib/crypto.ts index 218910d..685edcc 100644 --- a/src/lib/crypto.ts +++ b/src/lib/crypto.ts @@ -13,6 +13,7 @@ async function hash(namespace: string) { const relevant_hash = "Qa4xy8yRPnBsQf8U2UUE7tse2u4Fi/Aqxbt" + "6aq56m5ofcmCkTi8PbgMBF1OtHC6qkXDF2tz2qKprYCIAMzTSQQ=="; + Deno.test("encode_namespace", async () => { const namespace = "CHANGEME"; const base64 = await hash(namespace); diff --git a/src/okimeter.ts b/src/okimeter.ts index a336742..4312aae 100755 --- a/src/okimeter.ts +++ b/src/okimeter.ts @@ -12,7 +12,7 @@ async function main() { if (token === undefined) token = prompt_args(); const client = new ApiClient("gitlab.cri.epita.fr", token); const merge_requests = await get_all_merge_requests(client); - const total = sum(await parallel(merge_requests, (mr) => count_in_mr(mr, client))); + const total = sum(await parallel(merge_requests, (mr) => count_in_merge_request(mr, client))); console.log("Total", total, "oki"); } @@ -35,17 +35,17 @@ ${gr("> xxxxxxxxxxxxxxxxxxxx")} return result.trim(); } -async function get_all_merge_requests(client: ApiClient): Promise { +async function get_all_merge_requests(client: ApiClient) { return await client.fetch_parse_paginated(`api/v4/merge_requests`, merge_request_parser); } -async function count_in_mr(mr: MergeRequest, client: ApiClient) { +async function count_in_merge_request(merge_request: MergeRequest, client: ApiClient) { let total = 0; - const mr_path = `api/v4/projects/${mr.project_id}/merge_requests/${mr.iid}`; - const all_notes = await client.fetch_parse_paginated(`${mr_path}/notes`, note_parser); - for (const c of all_notes) { - if (!await is_relevant(c.author.username)) continue; - if (c.body.toLowerCase().includes("oki")) total += 1; + const merge_request_path = `api/v4/projects/${merge_request.project_id}/merge_requests/${merge_request.iid}`; + const all_notes = await client.fetch_parse_paginated(`${merge_request_path}/notes`, note_parser); + for (const note of all_notes) { + if (!await is_relevant(note.author.username)) continue; + if (note.body.toLowerCase().includes("oki")) total += 1; } return total; } From 9641973def3cb28e6d0a16b6e3990aac3da261ce Mon Sep 17 00:00:00 2001 From: JOLIMAITRE Matthieu Date: Mon, 15 Apr 2024 10:39:46 +0200 Subject: [PATCH 3/3] refactor for more async work --- src/lib/client.ts | 6 ++---- src/lib/utils.ts | 10 ++++++++-- src/okimeter.ts | 16 ++++++++++------ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/lib/client.ts b/src/lib/client.ts index 91fcc58..9a608ba 100644 --- a/src/lib/client.ts +++ b/src/lib/client.ts @@ -22,15 +22,13 @@ export class ApiClient { return parser.parse(unkn); } - async fetch_parse_paginated(path: string, parser: z.ZodType) { - const result = [] as T[]; + async *fetch_parse_paginated(path: string, parser: z.ZodType): AsyncIterable { let page = 1; while (true) { const fetched = await this.fetch_parse(path + "?per_page=100&page=" + page, parser.array()); - result.push(...fetched); + yield* fetched; if (fetch.length < 100) break; page += 1; } - return result; } } diff --git a/src/lib/utils.ts b/src/lib/utils.ts index c4f0259..d757cd9 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -3,8 +3,14 @@ export function sum(numbers: Iterable) { for (const item of numbers) result += item; return result; } -export async function parallel(inputs: I[], operation: (item: I) => Promise) { +export async function parallel(inputs: AsyncIterable, operation: (item: I) => Promise) { const promises = [] as Promise[]; - for (const input of inputs) promises.push(operation(input)); + for await (const input of inputs) promises.push(operation(input)); return await Promise.all(promises); } + +export async function collect(gen: AsyncIterable) { + const result = [] as T[]; + for await (const item of gen) result.push(item); + return result; +} diff --git a/src/okimeter.ts b/src/okimeter.ts index 4312aae..a16a0eb 100755 --- a/src/okimeter.ts +++ b/src/okimeter.ts @@ -11,7 +11,7 @@ async function main() { let { token } = parse_args(Deno.args); if (token === undefined) token = prompt_args(); const client = new ApiClient("gitlab.cri.epita.fr", token); - const merge_requests = await get_all_merge_requests(client); + const merge_requests = get_all_merge_requests(client); const total = sum(await parallel(merge_requests, (mr) => count_in_merge_request(mr, client))); console.log("Total", total, "oki"); } @@ -35,19 +35,23 @@ ${gr("> xxxxxxxxxxxxxxxxxxxx")} return result.trim(); } -async function get_all_merge_requests(client: ApiClient) { - return await client.fetch_parse_paginated(`api/v4/merge_requests`, merge_request_parser); +function get_all_merge_requests(client: ApiClient) { + return client.fetch_parse_paginated(`api/v4/merge_requests`, merge_request_parser); } async function count_in_merge_request(merge_request: MergeRequest, client: ApiClient) { + const all_notes = get_all_merge_request_notes(merge_request, client); let total = 0; - const merge_request_path = `api/v4/projects/${merge_request.project_id}/merge_requests/${merge_request.iid}`; - const all_notes = await client.fetch_parse_paginated(`${merge_request_path}/notes`, note_parser); - for (const note of all_notes) { + for await (const note of all_notes) { if (!await is_relevant(note.author.username)) continue; if (note.body.toLowerCase().includes("oki")) total += 1; } return total; } +function get_all_merge_request_notes(merge_request: MergeRequest, client: ApiClient) { + const merge_request_path = `api/v4/projects/${merge_request.project_id}/merge_requests/${merge_request.iid}`; + return client.fetch_parse_paginated(`${merge_request_path}/notes`, note_parser); +} + if (import.meta.main) await main();