nick-doc/scripts/profile-mongo-api.mjs

#!/usr/bin/env node

import { execFileSync, spawnSync } from "node:child_process";
import { existsSync } from "node:fs";
import { mkdir, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";

const docRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");

const config = {
  baseUrl: process.env.BASE_URL || "https://dev.manwe.qzz.io",
  sshHost: process.env.SSH_HOST || "root@5.78.213.189",
  sshKey: expandHome(process.env.SSH_KEY || "~/CascadeProjects/wzp"),
  mongoContainer: process.env.MONGO_CONTAINER || "amanat-dev-mongodb",
  mongoDb: process.env.MONGO_DB || "marketplace",
  mongoUser: process.env.MONGO_USER || "admin",
  mongoPassword: process.env.MONGO_PASSWORD || "password123",
  mongoAuthDb: process.env.MONGO_AUTH_DB || "admin",
  backendContainer: process.env.BACKEND_CONTAINER || "amanat-dev-backend",
  resetBackendLimiter: ["1", "true", "yes"].includes(
    String(process.env.RESET_BACKEND_LIMITER || "").toLowerCase(),
  ),
  npxBin: process.env.NPX_BIN || "npx",
  buyerEmail: process.env.BUYER_EMAIL || "buyer@marketplace.com",
  buyerPassword: process.env.BUYER_PASSWORD || "Moji6364",
  templateShareableLink: process.env.TEMPLATE_SHAREABLE_LINK || "logo-design-template",
  outputDir:
    process.env.OUT_DIR ||
    path.join(
      docRoot,
      "09 - Audits",
      "Mongo API Profiles",
      new Date().toISOString().replace(/[:.]/g, "-"),
    ),
};

const containers = (process.env.PROFILE_CONTAINERS || [
  "amanat-dev-nginx",
  "amanat-dev-backend",
  "amanat-dev-frontend",
  "amanat-dev-postgres",
  "amanat-dev-mongodb",
  "amanat-dev-redis",
  "amanat-dev-scanner",
].join(","))
  .split(",")
  .map((value) => value.trim())
  .filter(Boolean);

const endpointMatrix = [
  { name: "health", path: "/api/health", concurrency: 1, amount: 5 },
  { name: "categories", path: "/api/marketplace/categories", concurrency: 2, amount: 10 },
  { name: "categories_tree", path: "/api/marketplace/categories/tree", concurrency: 2, amount: 10 },
  { name: "sellers", path: "/api/marketplace/sellers", concurrency: 2, amount: 10 },
  {
    name: "template_public",
    path: `/api/marketplace/request-templates/public/${encodeURIComponent(config.templateShareableLink)}`,
    concurrency: 2,
    amount: 10,
  },
  {
    name: "payment_options_template",
    path: null,
    concurrency: 5,
    amount: 50,
    auth: true,
  },
  { name: "addresses_me", path: "/api/addresses", concurrency: 2, amount: 10, auth: true },
  {
    name: "purchase_requests_my",
    path: "/api/marketplace/purchase-requests/my",
    concurrency: 2,
    amount: 10,
    auth: true,
  },
  {
    name: "auth_login",
    path: "/api/auth/login",
    concurrency: 1,
    amount: 5,
    method: "POST",
    headers: ["Content-Type: application/json"],
    body: () => ({ email: config.buyerEmail, password: config.buyerPassword }),
  },
];

const sshBaseArgs = ["-i", config.sshKey, "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", config.sshHost];

if (!existsSync(config.sshKey)) {
  throw new Error(`SSH key not found: ${config.sshKey}`);
}

await mkdir(config.outputDir, { recursive: true });

let profilerEnabled = false;

try {
  if (config.resetBackendLimiter) {
    console.error(`restarting ${config.backendContainer} to reset process-local rate limits`);
    restartBackendContainer();
    await waitForHealth();
  }

  const authToken = await login();
  const template = getTemplateContext();

  const matrix = endpointMatrix.map((test, index) => {
    const pathValue =
      test.name === "payment_options_template"
        ? `/api/payment/request-network/options?currency=USD&amount=0.01&sellerId=${template.sellerId}&templateId=${template.templateId}`
        : test.path;

    return {
      ...test,
      path: pathValue,
      headers: [
        ...(test.headers || []),
        // Counts are intentionally low to avoid profiling the in-memory global limiter.
        `X-Forwarded-For: 203.0.113.${10 + index}`,
      ],
    };
  });

  const results = [];

  for (const test of matrix) {
    console.error(`profiling ${test.name} ${test.path}`);
    enableProfiler();
    const beforeBlockIo = readDockerBlockIo();
    const bench = runAutocannon(test, authToken);
    const afterBlockIo = readDockerBlockIo();
    const mongoProfile = collectMongoProfile();

    results.push({
      name: test.name,
      method: test.method || "GET",
      path: test.path,
      requestCount: bench.requests.total,
      rps: bench.requests.average,
      latency: {
        averageMs: bench.latency.average,
        p50Ms: bench.latency.p50,
        p90Ms: bench.latency.p90,
        p95Ms: bench.latency.p95 ?? bench.latency.p97_5,
        p99Ms: bench.latency.p99,
        maxMs: bench.latency.max,
      },
      non2xx: bench.non2xx || 0,
      statusCodeStats: bench.statusCodeStats || {},
      mongoProfile,
      blockIoDelta: diffBlockIo(beforeBlockIo, afterBlockIo),
    });
  }

  disableProfiler();

  const report = {
    generatedAt: new Date().toISOString(),
    config: {
      baseUrl: config.baseUrl,
      sshHost: config.sshHost,
      mongoContainer: config.mongoContainer,
      mongoDb: config.mongoDb,
      mongoAuthDb: config.mongoAuthDb,
      backendContainer: config.backendContainer,
      resetBackendLimiter: config.resetBackendLimiter,
      containers,
      templateShareableLink: config.templateShareableLink,
      outputDir: config.outputDir,
    },
    results,
  };

  const jsonPath = path.join(config.outputDir, "mongo-api-profile.json");
  const markdownPath = path.join(config.outputDir, "summary.md");
  await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`);
  await writeFile(markdownPath, renderMarkdown(report));

  console.log(`Wrote ${path.relative(docRoot, jsonPath)}`);
  console.log(`Wrote ${path.relative(docRoot, markdownPath)}`);
} catch (error) {
  if (profilerEnabled) {
    try {
      disableProfiler();
    } catch (disableError) {
      console.error(`failed to disable profiler: ${disableError.message}`);
    }
  }
  throw error;
}

function expandHome(value) {
  if (!value.startsWith("~/")) return value;
  return path.join(os.homedir(), value.slice(2));
}

function shellQuote(value) {
  return `'${String(value).replace(/'/g, "'\\''")}'`;
}

function ssh(command, options = {}) {
  return execFileSync("ssh", [...sshBaseArgs, command], {
    encoding: "utf8",
    maxBuffer: options.maxBuffer || 100 * 1024 * 1024,
  });
}

function mongoEval(js) {
  const command = [
    "docker exec",
    shellQuote(config.mongoContainer),
    "mongosh --quiet",
    "-u",
    shellQuote(config.mongoUser),
    "-p",
    shellQuote(config.mongoPassword),
    "--authenticationDatabase",
    shellQuote(config.mongoAuthDb),
    shellQuote(config.mongoDb),
    "--eval",
    shellQuote(js),
  ].join(" ");
  return ssh(command);
}

function restartBackendContainer() {
  ssh(`docker restart ${shellQuote(config.backendContainer)}`, { maxBuffer: 1024 * 1024 });
}

async function waitForHealth() {
  const deadline = Date.now() + 90_000;
  let lastError = "";

  while (Date.now() < deadline) {
    try {
      const response = await fetch(`${config.baseUrl}/api/health`);
      const body = await response.json();
      if (response.ok && body?.status === "ok") return;
      lastError = `status=${response.status} body=${JSON.stringify(body)}`;
    } catch (error) {
      lastError = error.message;
    }
    await new Promise((resolve) => setTimeout(resolve, 2_000));
  }

  throw new Error(`backend did not become healthy after restart: ${lastError}`);
}

async function login() {
  const response = await fetch(`${config.baseUrl}/api/auth/login`, {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({ email: config.buyerEmail, password: config.buyerPassword }),
  });
  const body = await response.json();
  if (!body?.success || !body?.data?.tokens?.accessToken) {
    throw new Error(`login failed with status ${response.status}: ${JSON.stringify(body)}`);
  }
  return body.data.tokens.accessToken;
}

function getTemplateContext() {
  const raw = mongoEval(`
    const doc = db.requesttemplates.findOne(
      { shareableLink: ${JSON.stringify(config.templateShareableLink)} },
      { _id: 1, sellerId: 1 }
    );
    print(JSON.stringify(doc));
  `).trim();

  if (!raw || raw === "null") {
    throw new Error(`template not found: ${config.templateShareableLink}`);
  }

  const doc = JSON.parse(raw);
  const templateId = doc._id?.$oid || doc._id;
  const sellerId = doc.sellerId?.$oid || doc.sellerId;
  if (!templateId || !sellerId) {
    throw new Error(`template missing _id/sellerId: ${raw}`);
  }
  return { templateId, sellerId };
}

function enableProfiler() {
  mongoEval(`
    db.setProfilingLevel(0);
    try { db.system.profile.drop(); } catch (error) {}
    db.setProfilingLevel(2, { slowms: 0, sampleRate: 1 });
    print(JSON.stringify(db.getProfilingStatus()));
  `);
  profilerEnabled = true;
}

function disableProfiler() {
  mongoEval(`db.setProfilingLevel(0); print(JSON.stringify(db.getProfilingStatus()));`);
  profilerEnabled = false;
}

function collectMongoProfile() {
  const output = mongoEval(`
    db.setProfilingLevel(0);
    const docs = db.system.profile.find({ ns: /^${escapeRegExp(config.mongoDb)}\\./ }).toArray();

    function commandName(doc) {
      const command = doc.command || {};
      for (const key of Object.keys(command)) {
        if (!['lsid', '$db', '$clusterTime', 'readConcern', 'writeConcern', 'maxTimeMS'].includes(key)) {
          return key;
        }
      }
      return doc.op || 'unknown';
    }

    function collectionName(doc) {
      const command = doc.command || {};
      return command.find ||
        command.aggregate ||
        command.count ||
        command.distinct ||
        command.update ||
        command.delete ||
        command.findAndModify ||
        command.insert ||
        doc.ns.replace(/^${escapeRegExp(config.mongoDb)}\\./, '');
    }

    function shapeValue(value) {
      if (value === null) return 'null';
      if (value === undefined) return 'undefined';
      if (Array.isArray(value)) return '[' + value.map(shapeValue).join(',') + ']';
      if (typeof value === 'object') {
        if (value._bsontype) return value._bsontype;
        return '{' + Object.keys(value).sort().map((key) => key + ':' + shapeValue(value[key])).join(',') + '}';
      }
      return typeof value;
    }

    function queryShape(doc) {
      const command = doc.command || {};
      const parts = [];
      if (command.filter) parts.push('filter=' + shapeValue(command.filter));
      if (command.query) parts.push('query=' + shapeValue(command.query));
      if (command.pipeline) parts.push('pipeline=' + shapeValue(command.pipeline));
      if (command.sort) parts.push('sort=' + shapeValue(command.sort));
      if (command.projection) parts.push('projection=' + shapeValue(command.projection));
      if (command.update) parts.push('update=' + shapeValue(command.update));
      return parts.join(' ');
    }

    const groups = new Map();
    for (const doc of docs) {
      const key = [
        doc.ns,
        doc.op,
        commandName(doc),
        collectionName(doc),
        doc.planSummary || '',
        doc.queryHash || '',
        doc.planCacheKey || '',
        queryShape(doc),
      ].join(' | ');

      let group = groups.get(key);
      if (!group) {
        group = {
          namespace: doc.ns,
          operation: doc.op,
          command: commandName(doc),
          collection: collectionName(doc),
          planSummary: doc.planSummary || '',
          queryHash: doc.queryHash || '',
          planCacheKey: doc.planCacheKey || '',
          queryShape: queryShape(doc),
          count: 0,
          millisTotal: 0,
          millisMax: 0,
          millisValues: [],
          docsExamined: 0,
          keysExamined: 0,
          nreturned: 0,
          ninserted: 0,
          nMatched: 0,
          nModified: 0,
          responseLength: 0,
          numYield: 0,
        };
        groups.set(key, group);
      }

      const millis = Number(doc.millis || 0);
      group.count += 1;
      group.millisTotal += millis;
      group.millisMax = Math.max(group.millisMax, millis);
      group.millisValues.push(millis);
      group.docsExamined += Number(doc.docsExamined || 0);
      group.keysExamined += Number(doc.keysExamined || 0);
      group.nreturned += Number(doc.nreturned || 0);
      group.ninserted += Number(doc.ninserted || 0);
      group.nMatched += Number(doc.nMatched || 0);
      group.nModified += Number(doc.nModified || 0);
      group.responseLength += Number(doc.responseLength || 0);
      group.numYield += Number(doc.numYield || 0);
    }

    function percentile(values, p) {
      if (!values.length) return 0;
      values.sort((a, b) => a - b);
      return values[Math.min(values.length - 1, Math.floor((p / 100) * values.length))];
    }

    const groupsOut = Array.from(groups.values())
      .map((group) => ({
        namespace: group.namespace,
        operation: group.operation,
        command: group.command,
        collection: group.collection,
        planSummary: group.planSummary,
        queryHash: group.queryHash,
        planCacheKey: group.planCacheKey,
        queryShape: group.queryShape,
        count: group.count,
        millisTotal: group.millisTotal,
        millisAverage: group.count ? group.millisTotal / group.count : 0,
        millisP50: percentile(group.millisValues, 50),
        millisP95: percentile(group.millisValues, 95),
        millisMax: group.millisMax,
        docsExamined: group.docsExamined,
        keysExamined: group.keysExamined,
        nreturned: group.nreturned,
        ninserted: group.ninserted,
        nMatched: group.nMatched,
        nModified: group.nModified,
        responseLength: group.responseLength,
        numYield: group.numYield,
      }))
      .sort((a, b) => b.millisTotal - a.millisTotal || b.count - a.count);

    print(JSON.stringify({
      totalOperations: docs.length,
      totalMillis: groupsOut.reduce((sum, group) => sum + group.millisTotal, 0),
      groups: groupsOut,
    }));
  `);

  return JSON.parse(output.trim().split(/\n/).pop());
}

function runAutocannon(test, authToken) {
  const args = ["-y", "autocannon@8.0.0"];
  if (test.amount) args.push("-a", String(test.amount));
  if (test.duration) args.push("-d", String(test.duration));
  args.push("-c", String(test.concurrency || 1), "--json");
  for (const header of test.headers || []) args.push("-H", header);
  if (test.auth) args.push("-H", `Authorization: Bearer ${authToken}`);
  if (test.method) args.push("-m", test.method);
  const body = typeof test.body === "function" ? test.body() : test.body;
  if (body) args.push("-b", JSON.stringify(body));
  args.push(`${config.baseUrl}${test.path}`);

  const result = spawnSync(config.npxBin, args, {
    encoding: "utf8",
    maxBuffer: 100 * 1024 * 1024,
  });

  if (result.status !== 0) {
    throw new Error(`autocannon failed for ${test.name}\n${result.stderr}\n${result.stdout}`);
  }

  return JSON.parse(result.stdout);
}

function readDockerBlockIo() {
  const output = ssh(
    `docker stats --no-stream --format '{{json .}}' ${containers.map(shellQuote).join(" ")}`,
  );
  const rows = output.trim().split(/\n/).filter(Boolean).map((line) => JSON.parse(line));
  const map = {};
  for (const row of rows) {
    const [readRaw, writeRaw] = String(row.BlockIO || "0B / 0B").split("/").map((item) => item.trim());
    map[row.Name] = {
      readBytes: parseBytes(readRaw),
      writeBytes: parseBytes(writeRaw),
      raw: row.BlockIO,
    };
  }
  return map;
}

function diffBlockIo(before, after) {
  const diff = {};
  for (const [name, value] of Object.entries(after)) {
    diff[name] = {
      readBytes: Math.max(0, value.readBytes - (before[name]?.readBytes || 0)),
      writeBytes: Math.max(0, value.writeBytes - (before[name]?.writeBytes || 0)),
    };
  }
  return diff;
}

function parseBytes(value) {
  const match = String(value || "").trim().match(/^([0-9.]+)\s*([KMGT]?i?B|B)$/i);
  if (!match) return 0;
  const number = Number(match[1]);
  const unit = match[2].toLowerCase();
  const multiplier = {
    b: 1,
    kb: 1_000,
    mb: 1_000_000,
    gb: 1_000_000_000,
    tb: 1_000_000_000_000,
    kib: 1024,
    mib: 1024 ** 2,
    gib: 1024 ** 3,
    tib: 1024 ** 4,
  }[unit] || 1;
  return number * multiplier;
}

function formatBytes(value) {
  if (!value) return "0 B";
  const units = ["B", "KB", "MB", "GB"];
  let next = value;
  let index = 0;
  while (next >= 1000 && index < units.length - 1) {
    next /= 1000;
    index += 1;
  }
  return `${next.toFixed(next >= 10 || index === 0 ? 0 : 1)} ${units[index]}`;
}

function renderMarkdown(report) {
  const lines = [];
  lines.push("# Mongo API Query Profile");
  lines.push("");
  lines.push(`Generated: ${report.generatedAt}`);
  lines.push(`Base URL: \`${report.config.baseUrl}\``);
  lines.push(`Mongo: \`${report.config.mongoContainer}/${report.config.mongoDb}\``);
  lines.push("");
  lines.push("This is a query-shape profile, not a max-throughput test. Request counts are intentionally small so the backend rate limiter does not dominate the profile.");
  lines.push("");
  lines.push("## Endpoint Summary");
  lines.push("");
  lines.push("| Endpoint | Requests | Avg | P95 | P99 | Non-2xx | Mongo ops | Top Mongo query |");
  lines.push("|---|---:|---:|---:|---:|---:|---:|---|");
  for (const result of report.results) {
    const top = result.mongoProfile.groups[0];
    lines.push(
      [
        `\`${result.method} ${result.path}\``,
        result.requestCount,
        `${result.latency.averageMs}ms`,
        `${result.latency.p95Ms}ms`,
        `${result.latency.p99Ms}ms`,
        result.non2xx,
        result.mongoProfile.totalOperations,
        top ? `\`${top.collection}\` ${top.command} (${top.count}x, ${top.planSummary || "no plan"})` : "-",
      ].join(" | ").replace(/^/, "| ").replace(/$/, " |"),
    );
  }
  lines.push("");
  lines.push("## Query Groups");
  for (const result of report.results) {
    lines.push("");
    lines.push(`### ${result.name}`);
    lines.push("");
    lines.push(`Path: \`${result.method} ${result.path}\``);
    lines.push(`Status codes: \`${JSON.stringify(result.statusCodeStats)}\``);
    lines.push("");
    if (!result.mongoProfile.groups.length) {
      lines.push("No Mongo operations captured in this endpoint window.");
      continue;
    }
    lines.push("| Collection | Command | Count | Total ms | Avg ms | P95 ms | Plan | Docs | Keys | Returned | Shape |");
    lines.push("|---|---|---:|---:|---:|---:|---|---:|---:|---:|---|");
    for (const group of result.mongoProfile.groups.slice(0, 12)) {
      lines.push(
        [
          `\`${group.collection}\``,
          `\`${group.command}\``,
          group.count,
          group.millisTotal,
          round(group.millisAverage),
          group.millisP95,
          `\`${group.planSummary || "-"}\``,
          group.docsExamined,
          group.keysExamined,
          group.nreturned,
          `\`${truncate(group.queryShape || "-", 140)}\``,
        ].join(" | ").replace(/^/, "| ").replace(/$/, " |"),
      );
    }
  }
  lines.push("");
  lines.push("## Block I/O Deltas");
  for (const result of report.results) {
    const active = Object.entries(result.blockIoDelta)
      .filter(([, value]) => value.readBytes || value.writeBytes)
      .map(([name, value]) => `${name}: read ${formatBytes(value.readBytes)}, write ${formatBytes(value.writeBytes)}`);
    lines.push(`- ${result.name}: ${active.length ? active.join("; ") : "no container block I/O delta"}`);
  }
  lines.push("");
  return `${lines.join("\n")}\n`;
}

function truncate(value, max) {
  return value.length > max ? `${value.slice(0, max - 3)}...` : value;
}

function round(value) {
  return Math.round(value * 1000) / 1000;
}

function escapeRegExp(value) {
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}