├── nop.js ├── .gitignore ├── README.md ├── .gitmodules ├── svg_test.js ├── install_deno.sh ├── wrk.js ├── generate_comment.js └── main.js /nop.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | benchmark.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deno GitHub bench bot 2 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "third_party"] 2 | path = third_party 3 | url = https://github.com/denoland/deno_third_party 4 | -------------------------------------------------------------------------------- /svg_test.js: -------------------------------------------------------------------------------- 1 | import { svgChart } from "./generate_comment.js"; 2 | 3 | console.log(svgChart({ "deno-pr": 100, "deno-canary": 200 })); 4 | -------------------------------------------------------------------------------- /install_deno.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright 2019 the Deno authors. All rights reserved. MIT license. 3 | # TODO(everyone): Keep this script simple and easily auditable. 4 | 5 | set -e 6 | 7 | if ! command -v unzip >/dev/null; then 8 | echo "Error: unzip is required to install Deno (see: https://github.com/denoland/deno_install#unzip-is-required)." 1>&2 9 | exit 1 10 | fi 11 | 12 | if [ "$OS" = "Windows_NT" ]; then 13 | target="x86_64-pc-windows-msvc" 14 | else 15 | case $(uname -sm) in 16 | "Darwin x86_64") target="x86_64-apple-darwin" ;; 17 | "Darwin arm64") target="aarch64-apple-darwin" ;; 18 | "Linux aarch64") 19 | echo "Error: Official Deno builds for Linux aarch64 are not available. (https://github.com/denoland/deno/issues/1846)" 1>&2 20 | exit 1 21 | ;; 22 | *) target="x86_64-unknown-linux-gnu" ;; 23 | esac 24 | fi 25 | 26 | if [ $# -eq 0 ]; then 27 | deno_uri="https://github.com/denoland/deno/releases/latest/download/deno-${target}.zip" 28 | else 29 | deno_uri="https://github.com/denoland/deno/releases/download/${1}/deno-${target}.zip" 30 | fi 31 | 32 | deno_install="${DENO_INSTALL:-$HOME/.deno}" 33 | bin_dir="$deno_install/bin" 34 | exe="$bin_dir/deno" 35 | 36 | if [ ! -d "$bin_dir" ]; then 37 | mkdir -p "$bin_dir" 38 | fi 39 | 40 | curl --fail --location --progress-bar --output "$exe.zip" "$deno_uri" 41 | unzip -d "$bin_dir" -o "$exe.zip" 42 | chmod +x "$exe" 43 | rm "$exe.zip" 44 | 45 | echo "Deno was installed successfully to $exe" 46 | if command -v deno >/dev/null; then 47 | echo "Run 'deno --help' to get started" 48 | else 49 | case $SHELL in 50 | /bin/zsh) shell_profile=".zshrc" ;; 51 | *) shell_profile=".bashrc" ;; 52 | esac 53 | echo "Manually add the directory to your \$HOME/$shell_profile (or similar)" 54 | echo " export DENO_INSTALL=\"$deno_install\"" 55 | echo " export PATH=\"\$DENO_INSTALL/bin:\$PATH\"" 56 | echo "Run '$exe --help' to get started" 57 | fi 58 | echo 59 | echo "Stuck? Join our Discord https://discord.gg/deno" -------------------------------------------------------------------------------- /wrk.js: -------------------------------------------------------------------------------- 1 | function parseLatency(str1, unit) { 2 | let latency = parseFloat(str1); 3 | switch (unit) { 4 | case "ms": 5 | latency *= 1000; 6 | break; 7 | case "s": 8 | latency *= 1000000; 9 | break; 10 | default: 11 | break; 12 | } 13 | return latency; 14 | } 15 | 16 | const rxLatency = 17 | /Latency\s+([\d\.]+)(\w+)\s+([\d\.]+)(\w+)\s+([\d\.]+)(\w+)\s+([\d\.]+)\%/; 18 | const rxReq = /\s+(\d+) requests in (\d+\.\d+)(\w+), (\d+\.\d+)(\w+) read/; 19 | const rxRPS = /Requests\/sec:\s+([\d\.]+)\s/; 20 | const rxThread = /Thread: (\d), (\d+): (\d+)\s?/; 21 | const rxPercent = /([\d\.]+)\%,(\d+)/; 22 | 23 | function parseWrks(text) { 24 | const result = { latency: {} }; 25 | let rps = 0; 26 | let match = text.match(rxLatency); 27 | if (match && match.length > 2) { 28 | result.latency.average = parseLatency(match[1], match[2]); 29 | result.latency.stdev = parseLatency(match[3], match[4]); 30 | result.latency.max = parseLatency(match[5], match[6]); 31 | result.latency.variance = parseLatency(match[7]); 32 | } 33 | match = text.match(rxReq); 34 | if (match && match.length > 1) { 35 | const [requests, time, tunit, read, unit] = match.slice(1); 36 | result.requests = parseInt(requests, 10); 37 | if (tunit.toLowerCase() === "s") { 38 | result.time = parseFloat(time); 39 | } else if (tunit.toLowerCase() === "m") { 40 | result.time = parseFloat(time) * 60; 41 | } else { 42 | result.time = parseFloat(time) * (60 * 60); 43 | } 44 | if (unit.toLowerCase() === "mb") { 45 | result.bytes = parseFloat(read) * 1000000; 46 | } else if (unit.toLowerCase() === "gb") { 47 | result.bytes = parseFloat(read) * 1000000000; 48 | } else { 49 | result.bytes = parseFloat(read); 50 | } 51 | } 52 | match = text.match(rxRPS); 53 | if (match && match.length > 1) { 54 | rps = parseInt(match[1], 10); 55 | } 56 | result.rps = rps; 57 | const lines = text.split("\n"); 58 | const statuses = {}; 59 | const threads = {}; 60 | const percentiles = {}; 61 | for (const line of lines) { 62 | const parts = line.match(rxThread); 63 | if (parts && parts.length > 2) { 64 | const [, id, status, count] = parts; 65 | threads[id] = threads[id] || {}; 66 | threads[id][status] = count; 67 | if (statuses[status]) { 68 | statuses[status] += parseInt(count, 10); 69 | } else { 70 | statuses[status] = parseInt(count, 10); 71 | } 72 | } else { 73 | const parts = line.match(rxPercent); 74 | if (parts && parts.length > 2) { 75 | const [, percentile, count] = parts; 76 | percentiles[percentile] = parseInt(count, 10); 77 | } 78 | } 79 | } 80 | result.percentiles = percentiles; 81 | result.threads = threads; 82 | result.statuses = statuses; 83 | return result; 84 | } 85 | -------------------------------------------------------------------------------- /generate_comment.js: -------------------------------------------------------------------------------- 1 | import "https://deno.land/std/dotenv/load.ts"; 2 | 3 | const repo = Deno.args[0] || "denoland/deno"; 4 | const pullNumber = Deno.args[1]; 5 | const artifactID = Deno.args[2]; 6 | const benchmarkType = Deno.args[3]; 7 | const artifactName = `deno-${pullNumber}`; 8 | 9 | const token = Deno.env.get("GITHUB_TOKEN"); 10 | const equinixToken = Deno.env.get("EQUINIX_TOKEN"); 11 | 12 | const osDir = Deno.build.os === "linux" ? "linux64" : "mac"; 13 | const hyperfineBin = 14 | `equinix-metal-test/third_party/prebuilt/${osDir}/hyperfine`; 15 | 16 | export function svgChart(means) { 17 | const body = 18 | `![](https://quickchart.io/chart?c={type:%27bar%27,data:{labels:${ 19 | JSON.stringify(Object.keys(means)) 20 | },datasets:[{label:%27Units%27,data:${ 21 | JSON.stringify(Object.values(means)) 22 | }}]}}) 23 | `; 24 | return body; 25 | } 26 | 27 | export async function generateComment(body, pullNumber) { 28 | const comment = { 29 | body, 30 | }; 31 | const response = await fetch( 32 | `https://api.github.com/repos/${repo}/issues/${pullNumber}/comments`, 33 | { 34 | method: "POST", 35 | headers: { 36 | "Content-Type": "application/json", 37 | "Authorization": `token ${token}`, 38 | }, 39 | body: JSON.stringify(comment), 40 | }, 41 | ); 42 | return response.json(); 43 | } 44 | 45 | async function downloadArtifact() { 46 | const redirected = await fetch( 47 | `https://api.github.com/repos/denoland/deno/actions/artifacts/${artifactID}/zip`, 48 | { 49 | headers: { 50 | "Authorization": `token ${token}`, 51 | }, 52 | }, 53 | ); 54 | const location = redirected.url; 55 | // curl 56 | const result = await Deno.run({ 57 | cmd: [ 58 | "curl", 59 | "-L", 60 | "-H", 61 | `Authorization: token ${token}`, 62 | "-o", 63 | "artifact.zip", 64 | location, 65 | ], 66 | }); 67 | await result.status(); 68 | result.close(); 69 | // unzip 70 | const unzip = await Deno.run({ 71 | cmd: ["unzip", "artifact.zip"], 72 | }); 73 | await unzip.status(); 74 | unzip.close(); 75 | // chmod 76 | const chmod = await Deno.run({ 77 | cmd: ["chmod", "+x", "./deno"], 78 | }); 79 | await chmod.status(); 80 | chmod.close(); 81 | } 82 | 83 | async function getInstanceMetadata() { 84 | const resp = await fetch( 85 | `https://metadata.platformequinix.com/metadata`, 86 | ); 87 | return resp.json(); 88 | } 89 | 90 | async function terminateInstance() { 91 | const { id } = await getInstanceMetadata(); 92 | const resp = await fetch( 93 | `https://api.equinix.com/metal/v1/devices/${id}?force_delete=true`, 94 | { 95 | method: "DELETE", 96 | headers: { 97 | "X-Auth-Token": equinixToken, 98 | }, 99 | }, 100 | ); 101 | return resp.text(); 102 | } 103 | 104 | async function runHyperfine() { 105 | const result = await Deno.run({ 106 | cmd: [ 107 | hyperfineBin, 108 | "--warmup", 109 | "5", 110 | "--show-output", 111 | "--export-json", 112 | "hyperfine.json", 113 | "deno run equinix-metal-test/nop.js", 114 | `./deno run equinix-metal-test/nop.js`, 115 | ], 116 | }); 117 | await result.status(); 118 | } 119 | 120 | async function hyperfine() { 121 | await runHyperfine(); 122 | const { results } = JSON.parse(await Deno.readTextFile("hyperfine.json")); 123 | const means = { "deno-main": results[0].mean, "deno-pr": results[1].mean }; 124 | console.log(await generateComment(svgChart(means), pullNumber)); 125 | } 126 | 127 | async function wrk() { 128 | const result = await Deno.run({ 129 | cmd: [ 130 | "wrk", 131 | "-t", 132 | 2, 133 | "-d", 134 | 30, 135 | "-c", 136 | 256, 137 | `http://127.0.0.1:8080/`, 138 | ], 139 | }); 140 | 141 | await result.status(); 142 | } 143 | 144 | const benchmarkTypes = { 145 | hyperfine, 146 | wrk, 147 | }; 148 | 149 | if (import.meta.main) { 150 | try { 151 | await downloadArtifact(); 152 | const run = benchmarkTypes[benchmarkType]; 153 | if (run) run(); 154 | else await generateComment(`benchmark type invalid: ${benchmarkType}`); 155 | } catch (e) { 156 | await generateComment(e.toString(), pullNumber); 157 | } finally { 158 | console.log(await terminateInstance()); 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /main.js: -------------------------------------------------------------------------------- 1 | import "https://deno.land/std/dotenv/load.ts"; 2 | import { serve } from "https://deno.land/std/http/server.ts"; 3 | import { router } from "https://deno.land/x/rutt/mod.ts"; 4 | import { encode } from "https://deno.land/std/encoding/hex.ts"; 5 | import { generateComment } from "./generate_comment.js"; 6 | 7 | const webhookSecret = Deno.env.get("WEBHOOK_SECRET"); 8 | const equinixProjectId = Deno.env.get("EQUINIX_PROJECT_ID"); 9 | const equinixToken = Deno.env.get("EQUINIX_TOKEN"); 10 | const githubToken = Deno.env.get("GITHUB_TOKEN"); 11 | 12 | async function createSpotMarketRequest(prNumber, type_) { 13 | const artifactId = await getArtifactId(prNumber); 14 | const resp = await fetch( 15 | `https://api.equinix.com/metal/v1/projects/${equinixProjectId}/spot-market-requests`, 16 | { 17 | method: "POST", 18 | headers: { 19 | "Content-Type": "application/json", 20 | "X-Auth-Token": equinixToken, 21 | }, 22 | body: JSON.stringify({ 23 | "devices_max": 1, 24 | "devices_min": 1, 25 | "max_bid_price": 0.2, 26 | "instance_parameters": { 27 | "hostname": "divy2", 28 | "plan": "m3.small.x86", 29 | "operating_system": "ubuntu_22_04", 30 | "userdata": createBenchScript(prNumber, artifactId, type_), 31 | }, 32 | }), 33 | }, 34 | ); 35 | return resp.json(); 36 | } 37 | 38 | async function getSpotMarketRequest(id) { 39 | const resp = await fetch( 40 | `https://api.equinix.com/metal/v1/projects/${equinixProjectId}/spot-market-requests`, 41 | { 42 | headers: { 43 | "X-Auth-Token": equinixToken, 44 | }, 45 | }, 46 | ); 47 | const result = await resp.json(); 48 | return result.spot_market_requests.find((r) => r.id === id); 49 | } 50 | 51 | async function getArtifactId(prNumber) { 52 | const artifactName = `deno-${prNumber}`; 53 | const resp = await fetch( 54 | `https://api.github.com/repos/denoland/deno/actions/artifacts?per_page=100`, 55 | { 56 | headers: { 57 | "Authorization": `token ${githubToken}`, 58 | }, 59 | }, 60 | ); 61 | const result = await resp.json(); 62 | const artifact = result.artifacts.find((a) => a.name === artifactName); 63 | if (!artifact) console.error("CI pending or PR marked as draft"); 64 | return artifact.id; 65 | } 66 | 67 | function createBenchScript(prNumber, artifactId, type_) { 68 | return `#!/bin/bash 69 | apt-get install -y unzip git 70 | export PATH=$HOME/.deno/bin:$PATH 71 | git clone --depth=1 --recurse-submodules https://github.com/littledivy/equinix-metal-test 72 | sh equinix-metal-test/install_deno.sh 73 | deno upgrade --canary 74 | GITHUB_TOKEN=${githubToken} EQUINIX_TOKEN=${equinixToken} deno run -A --unstable equinix-metal-test/generate_comment.js denoland/deno ${prNumber} ${artifactId} ${type_} 75 | `; 76 | } 77 | 78 | const enc = new TextEncoder(); 79 | const dec = new TextDecoder(); 80 | 81 | const key = await crypto.subtle.importKey( 82 | "raw", 83 | enc.encode(webhookSecret), 84 | { name: "HMAC", hash: "SHA-1" }, 85 | false, 86 | ["verify", "sign"], 87 | ); 88 | 89 | async function sign(data) { 90 | const s = await crypto.subtle.sign("HMAC", key, new Uint8Array(data)); 91 | return `sha1=${dec.decode(encode(new Uint8Array(s)))}`; 92 | } 93 | 94 | function benchmarkType(arg) { 95 | if (!arg) return "hyperfine"; 96 | arg = arg.trim(); 97 | return arg; 98 | } 99 | 100 | const authorizedRoles = ["OWNER", "MEMBER"]; 101 | 102 | async function handler(req) { 103 | const event = req.headers.get("x-github-event"); 104 | if (!event) return new Response("No event", { status: 400 }); 105 | 106 | const signature = req.headers.get("x-hub-signature"); 107 | const body = await req.arrayBuffer(); 108 | const digest = await sign(body); 109 | if (signature !== digest) { 110 | return new Response("Invalid signature", { status: 401 }); 111 | } 112 | 113 | const fn = ({ 114 | "ping": () => {}, 115 | "issue_comment": async (event) => { 116 | if (event.action === "created") { 117 | const id = event.issue.number; 118 | const comment = event.comment.body.trim(); 119 | const authorized = authorizedRoles.includes( 120 | event.comment.author_association, 121 | ); 122 | if ( 123 | authorized && 124 | comment.startsWith("+bench") && 125 | !comment.startsWith("+bench status") 126 | ) { 127 | const args = comment.split(" ")[1]; 128 | const type_ = benchmarkType(args); 129 | console.log("Creating spot market request"); 130 | const request = await createSpotMarketRequest(id, type_); 131 | if (request.errors) { 132 | await generateComment(`❌ ${request.errors[0]}`, id); 133 | return; 134 | } 135 | await generateComment( 136 | `⏳ Provisioning metal...\n\n id: \`${request.id}\`\n metro: \`${ 137 | request.metro ?? "unknown" 138 | }\`\n\n Use \`+bench status \` for status `, 139 | id, 140 | ); 141 | } 142 | 143 | if ( 144 | authorized && 145 | comment.startsWith("+bench status") 146 | ) { 147 | const reqid = comment.split(" ")[2]; 148 | if (!reqid) { 149 | return; 150 | } 151 | 152 | const request = await getSpotMarketRequest(reqid.trim()); 153 | if (request.errors || request.error) return; 154 | const device = request.devices[0]; 155 | if (device) { 156 | const d = await fetch(`https://api.equinix.com${device.href}`, { 157 | headers: { 158 | "X-Auth-Token": equinixToken, 159 | }, 160 | }); 161 | const res = await d.json(); 162 | let metro = res.metro 163 | ? `metro: ${res.metro.name} (${res.metro.country})` 164 | : "unknown"; 165 | const percentage = `${ 166 | Math.round(res.provisioning_percentage || 100) 167 | }%`; 168 | await generateComment( 169 | `✅ Device provisioned ${percentage}\n\n${metro}`, 170 | id, 171 | ); 172 | } else { 173 | await generateComment( 174 | `⏳ No device provisioned yet\n\ncreated_at: \`${request.created_at}\``, 175 | id, 176 | ); 177 | } 178 | } 179 | } 180 | }, 181 | })[event]; 182 | 183 | if (!fn) return new Response("No handler for event", { status: 400 }); 184 | const info = JSON.parse(dec.decode(body)); 185 | await fn(info); 186 | return new Response("OK"); 187 | } 188 | 189 | serve(router({ 190 | "/hooks/github": handler, 191 | })); 192 | --------------------------------------------------------------------------------