docs: sync documentation with latest codebase state (merged)

- Update Activity Log with 108 missing commits (48 backend + 60 frontend)
- Update version references: backend v2.8.79, frontend v2.8.94
- Update migration count: 18 migrations (0000-0017)
- Update Telegram Mini App Flow to v2.8.94
- Update Payment Flow - Scanner to 2026-06-05
- Update all architectural and database references
- Add MongoDB removal handoff document with updated versions

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Siavash Sameni
2026-06-05 07:51:00 +04:00
parent e51236af91
commit c98c31dc24
8 changed files with 2013 additions and 4 deletions

617
scripts/profile-mongo-api.mjs Executable file
View File

@@ -0,0 +1,617 @@
#!/usr/bin/env node
import { execFileSync, spawnSync } from "node:child_process";
import { existsSync } from "node:fs";
import { mkdir, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
const docRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const config = {
baseUrl: process.env.BASE_URL || "https://dev.manwe.qzz.io",
sshHost: process.env.SSH_HOST || "root@5.78.213.189",
sshKey: expandHome(process.env.SSH_KEY || "~/CascadeProjects/wzp"),
mongoContainer: process.env.MONGO_CONTAINER || "amanat-dev-mongodb",
mongoDb: process.env.MONGO_DB || "marketplace",
mongoUser: process.env.MONGO_USER || "admin",
mongoPassword: process.env.MONGO_PASSWORD || "password123",
mongoAuthDb: process.env.MONGO_AUTH_DB || "admin",
backendContainer: process.env.BACKEND_CONTAINER || "amanat-dev-backend",
resetBackendLimiter: ["1", "true", "yes"].includes(
String(process.env.RESET_BACKEND_LIMITER || "").toLowerCase(),
),
npxBin: process.env.NPX_BIN || "npx",
buyerEmail: process.env.BUYER_EMAIL || "buyer@marketplace.com",
buyerPassword: process.env.BUYER_PASSWORD || "Moji6364",
templateShareableLink: process.env.TEMPLATE_SHAREABLE_LINK || "logo-design-template",
outputDir:
process.env.OUT_DIR ||
path.join(
docRoot,
"09 - Audits",
"Mongo API Profiles",
new Date().toISOString().replace(/[:.]/g, "-"),
),
};
const containers = (process.env.PROFILE_CONTAINERS || [
"amanat-dev-nginx",
"amanat-dev-backend",
"amanat-dev-frontend",
"amanat-dev-postgres",
"amanat-dev-mongodb",
"amanat-dev-redis",
"amanat-dev-scanner",
].join(","))
.split(",")
.map((value) => value.trim())
.filter(Boolean);
const endpointMatrix = [
{ name: "health", path: "/api/health", concurrency: 1, amount: 5 },
{ name: "categories", path: "/api/marketplace/categories", concurrency: 2, amount: 10 },
{ name: "categories_tree", path: "/api/marketplace/categories/tree", concurrency: 2, amount: 10 },
{ name: "sellers", path: "/api/marketplace/sellers", concurrency: 2, amount: 10 },
{
name: "template_public",
path: `/api/marketplace/request-templates/public/${encodeURIComponent(config.templateShareableLink)}`,
concurrency: 2,
amount: 10,
},
{
name: "payment_options_template",
path: null,
concurrency: 5,
amount: 50,
auth: true,
},
{ name: "addresses_me", path: "/api/addresses", concurrency: 2, amount: 10, auth: true },
{
name: "purchase_requests_my",
path: "/api/marketplace/purchase-requests/my",
concurrency: 2,
amount: 10,
auth: true,
},
{
name: "auth_login",
path: "/api/auth/login",
concurrency: 1,
amount: 5,
method: "POST",
headers: ["Content-Type: application/json"],
body: () => ({ email: config.buyerEmail, password: config.buyerPassword }),
},
];
const sshBaseArgs = ["-i", config.sshKey, "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", config.sshHost];
if (!existsSync(config.sshKey)) {
throw new Error(`SSH key not found: ${config.sshKey}`);
}
await mkdir(config.outputDir, { recursive: true });
let profilerEnabled = false;
try {
if (config.resetBackendLimiter) {
console.error(`restarting ${config.backendContainer} to reset process-local rate limits`);
restartBackendContainer();
await waitForHealth();
}
const authToken = await login();
const template = getTemplateContext();
const matrix = endpointMatrix.map((test, index) => {
const pathValue =
test.name === "payment_options_template"
? `/api/payment/request-network/options?currency=USD&amount=0.01&sellerId=${template.sellerId}&templateId=${template.templateId}`
: test.path;
return {
...test,
path: pathValue,
headers: [
...(test.headers || []),
// Counts are intentionally low to avoid profiling the in-memory global limiter.
`X-Forwarded-For: 203.0.113.${10 + index}`,
],
};
});
const results = [];
for (const test of matrix) {
console.error(`profiling ${test.name} ${test.path}`);
enableProfiler();
const beforeBlockIo = readDockerBlockIo();
const bench = runAutocannon(test, authToken);
const afterBlockIo = readDockerBlockIo();
const mongoProfile = collectMongoProfile();
results.push({
name: test.name,
method: test.method || "GET",
path: test.path,
requestCount: bench.requests.total,
rps: bench.requests.average,
latency: {
averageMs: bench.latency.average,
p50Ms: bench.latency.p50,
p90Ms: bench.latency.p90,
p95Ms: bench.latency.p95 ?? bench.latency.p97_5,
p99Ms: bench.latency.p99,
maxMs: bench.latency.max,
},
non2xx: bench.non2xx || 0,
statusCodeStats: bench.statusCodeStats || {},
mongoProfile,
blockIoDelta: diffBlockIo(beforeBlockIo, afterBlockIo),
});
}
disableProfiler();
const report = {
generatedAt: new Date().toISOString(),
config: {
baseUrl: config.baseUrl,
sshHost: config.sshHost,
mongoContainer: config.mongoContainer,
mongoDb: config.mongoDb,
mongoAuthDb: config.mongoAuthDb,
backendContainer: config.backendContainer,
resetBackendLimiter: config.resetBackendLimiter,
containers,
templateShareableLink: config.templateShareableLink,
outputDir: config.outputDir,
},
results,
};
const jsonPath = path.join(config.outputDir, "mongo-api-profile.json");
const markdownPath = path.join(config.outputDir, "summary.md");
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`);
await writeFile(markdownPath, renderMarkdown(report));
console.log(`Wrote ${path.relative(docRoot, jsonPath)}`);
console.log(`Wrote ${path.relative(docRoot, markdownPath)}`);
} catch (error) {
if (profilerEnabled) {
try {
disableProfiler();
} catch (disableError) {
console.error(`failed to disable profiler: ${disableError.message}`);
}
}
throw error;
}
function expandHome(value) {
if (!value.startsWith("~/")) return value;
return path.join(os.homedir(), value.slice(2));
}
function shellQuote(value) {
return `'${String(value).replace(/'/g, "'\\''")}'`;
}
function ssh(command, options = {}) {
return execFileSync("ssh", [...sshBaseArgs, command], {
encoding: "utf8",
maxBuffer: options.maxBuffer || 100 * 1024 * 1024,
});
}
function mongoEval(js) {
const command = [
"docker exec",
shellQuote(config.mongoContainer),
"mongosh --quiet",
"-u",
shellQuote(config.mongoUser),
"-p",
shellQuote(config.mongoPassword),
"--authenticationDatabase",
shellQuote(config.mongoAuthDb),
shellQuote(config.mongoDb),
"--eval",
shellQuote(js),
].join(" ");
return ssh(command);
}
function restartBackendContainer() {
ssh(`docker restart ${shellQuote(config.backendContainer)}`, { maxBuffer: 1024 * 1024 });
}
async function waitForHealth() {
const deadline = Date.now() + 90_000;
let lastError = "";
while (Date.now() < deadline) {
try {
const response = await fetch(`${config.baseUrl}/api/health`);
const body = await response.json();
if (response.ok && body?.status === "ok") return;
lastError = `status=${response.status} body=${JSON.stringify(body)}`;
} catch (error) {
lastError = error.message;
}
await new Promise((resolve) => setTimeout(resolve, 2_000));
}
throw new Error(`backend did not become healthy after restart: ${lastError}`);
}
async function login() {
const response = await fetch(`${config.baseUrl}/api/auth/login`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ email: config.buyerEmail, password: config.buyerPassword }),
});
const body = await response.json();
if (!body?.success || !body?.data?.tokens?.accessToken) {
throw new Error(`login failed with status ${response.status}: ${JSON.stringify(body)}`);
}
return body.data.tokens.accessToken;
}
function getTemplateContext() {
const raw = mongoEval(`
const doc = db.requesttemplates.findOne(
{ shareableLink: ${JSON.stringify(config.templateShareableLink)} },
{ _id: 1, sellerId: 1 }
);
print(JSON.stringify(doc));
`).trim();
if (!raw || raw === "null") {
throw new Error(`template not found: ${config.templateShareableLink}`);
}
const doc = JSON.parse(raw);
const templateId = doc._id?.$oid || doc._id;
const sellerId = doc.sellerId?.$oid || doc.sellerId;
if (!templateId || !sellerId) {
throw new Error(`template missing _id/sellerId: ${raw}`);
}
return { templateId, sellerId };
}
function enableProfiler() {
mongoEval(`
db.setProfilingLevel(0);
try { db.system.profile.drop(); } catch (error) {}
db.setProfilingLevel(2, { slowms: 0, sampleRate: 1 });
print(JSON.stringify(db.getProfilingStatus()));
`);
profilerEnabled = true;
}
function disableProfiler() {
mongoEval(`db.setProfilingLevel(0); print(JSON.stringify(db.getProfilingStatus()));`);
profilerEnabled = false;
}
function collectMongoProfile() {
const output = mongoEval(`
db.setProfilingLevel(0);
const docs = db.system.profile.find({ ns: /^${escapeRegExp(config.mongoDb)}\\./ }).toArray();
function commandName(doc) {
const command = doc.command || {};
for (const key of Object.keys(command)) {
if (!['lsid', '$db', '$clusterTime', 'readConcern', 'writeConcern', 'maxTimeMS'].includes(key)) {
return key;
}
}
return doc.op || 'unknown';
}
function collectionName(doc) {
const command = doc.command || {};
return command.find ||
command.aggregate ||
command.count ||
command.distinct ||
command.update ||
command.delete ||
command.findAndModify ||
command.insert ||
doc.ns.replace(/^${escapeRegExp(config.mongoDb)}\\./, '');
}
function shapeValue(value) {
if (value === null) return 'null';
if (value === undefined) return 'undefined';
if (Array.isArray(value)) return '[' + value.map(shapeValue).join(',') + ']';
if (typeof value === 'object') {
if (value._bsontype) return value._bsontype;
return '{' + Object.keys(value).sort().map((key) => key + ':' + shapeValue(value[key])).join(',') + '}';
}
return typeof value;
}
function queryShape(doc) {
const command = doc.command || {};
const parts = [];
if (command.filter) parts.push('filter=' + shapeValue(command.filter));
if (command.query) parts.push('query=' + shapeValue(command.query));
if (command.pipeline) parts.push('pipeline=' + shapeValue(command.pipeline));
if (command.sort) parts.push('sort=' + shapeValue(command.sort));
if (command.projection) parts.push('projection=' + shapeValue(command.projection));
if (command.update) parts.push('update=' + shapeValue(command.update));
return parts.join(' ');
}
const groups = new Map();
for (const doc of docs) {
const key = [
doc.ns,
doc.op,
commandName(doc),
collectionName(doc),
doc.planSummary || '',
doc.queryHash || '',
doc.planCacheKey || '',
queryShape(doc),
].join(' | ');
let group = groups.get(key);
if (!group) {
group = {
namespace: doc.ns,
operation: doc.op,
command: commandName(doc),
collection: collectionName(doc),
planSummary: doc.planSummary || '',
queryHash: doc.queryHash || '',
planCacheKey: doc.planCacheKey || '',
queryShape: queryShape(doc),
count: 0,
millisTotal: 0,
millisMax: 0,
millisValues: [],
docsExamined: 0,
keysExamined: 0,
nreturned: 0,
ninserted: 0,
nMatched: 0,
nModified: 0,
responseLength: 0,
numYield: 0,
};
groups.set(key, group);
}
const millis = Number(doc.millis || 0);
group.count += 1;
group.millisTotal += millis;
group.millisMax = Math.max(group.millisMax, millis);
group.millisValues.push(millis);
group.docsExamined += Number(doc.docsExamined || 0);
group.keysExamined += Number(doc.keysExamined || 0);
group.nreturned += Number(doc.nreturned || 0);
group.ninserted += Number(doc.ninserted || 0);
group.nMatched += Number(doc.nMatched || 0);
group.nModified += Number(doc.nModified || 0);
group.responseLength += Number(doc.responseLength || 0);
group.numYield += Number(doc.numYield || 0);
}
function percentile(values, p) {
if (!values.length) return 0;
values.sort((a, b) => a - b);
return values[Math.min(values.length - 1, Math.floor((p / 100) * values.length))];
}
const groupsOut = Array.from(groups.values())
.map((group) => ({
namespace: group.namespace,
operation: group.operation,
command: group.command,
collection: group.collection,
planSummary: group.planSummary,
queryHash: group.queryHash,
planCacheKey: group.planCacheKey,
queryShape: group.queryShape,
count: group.count,
millisTotal: group.millisTotal,
millisAverage: group.count ? group.millisTotal / group.count : 0,
millisP50: percentile(group.millisValues, 50),
millisP95: percentile(group.millisValues, 95),
millisMax: group.millisMax,
docsExamined: group.docsExamined,
keysExamined: group.keysExamined,
nreturned: group.nreturned,
ninserted: group.ninserted,
nMatched: group.nMatched,
nModified: group.nModified,
responseLength: group.responseLength,
numYield: group.numYield,
}))
.sort((a, b) => b.millisTotal - a.millisTotal || b.count - a.count);
print(JSON.stringify({
totalOperations: docs.length,
totalMillis: groupsOut.reduce((sum, group) => sum + group.millisTotal, 0),
groups: groupsOut,
}));
`);
return JSON.parse(output.trim().split(/\n/).pop());
}
function runAutocannon(test, authToken) {
const args = ["-y", "autocannon@8.0.0"];
if (test.amount) args.push("-a", String(test.amount));
if (test.duration) args.push("-d", String(test.duration));
args.push("-c", String(test.concurrency || 1), "--json");
for (const header of test.headers || []) args.push("-H", header);
if (test.auth) args.push("-H", `Authorization: Bearer ${authToken}`);
if (test.method) args.push("-m", test.method);
const body = typeof test.body === "function" ? test.body() : test.body;
if (body) args.push("-b", JSON.stringify(body));
args.push(`${config.baseUrl}${test.path}`);
const result = spawnSync(config.npxBin, args, {
encoding: "utf8",
maxBuffer: 100 * 1024 * 1024,
});
if (result.status !== 0) {
throw new Error(`autocannon failed for ${test.name}\n${result.stderr}\n${result.stdout}`);
}
return JSON.parse(result.stdout);
}
function readDockerBlockIo() {
const output = ssh(
`docker stats --no-stream --format '{{json .}}' ${containers.map(shellQuote).join(" ")}`,
);
const rows = output.trim().split(/\n/).filter(Boolean).map((line) => JSON.parse(line));
const map = {};
for (const row of rows) {
const [readRaw, writeRaw] = String(row.BlockIO || "0B / 0B").split("/").map((item) => item.trim());
map[row.Name] = {
readBytes: parseBytes(readRaw),
writeBytes: parseBytes(writeRaw),
raw: row.BlockIO,
};
}
return map;
}
function diffBlockIo(before, after) {
const diff = {};
for (const [name, value] of Object.entries(after)) {
diff[name] = {
readBytes: Math.max(0, value.readBytes - (before[name]?.readBytes || 0)),
writeBytes: Math.max(0, value.writeBytes - (before[name]?.writeBytes || 0)),
};
}
return diff;
}
function parseBytes(value) {
const match = String(value || "").trim().match(/^([0-9.]+)\s*([KMGT]?i?B|B)$/i);
if (!match) return 0;
const number = Number(match[1]);
const unit = match[2].toLowerCase();
const multiplier = {
b: 1,
kb: 1_000,
mb: 1_000_000,
gb: 1_000_000_000,
tb: 1_000_000_000_000,
kib: 1024,
mib: 1024 ** 2,
gib: 1024 ** 3,
tib: 1024 ** 4,
}[unit] || 1;
return number * multiplier;
}
function formatBytes(value) {
if (!value) return "0 B";
const units = ["B", "KB", "MB", "GB"];
let next = value;
let index = 0;
while (next >= 1000 && index < units.length - 1) {
next /= 1000;
index += 1;
}
return `${next.toFixed(next >= 10 || index === 0 ? 0 : 1)} ${units[index]}`;
}
function renderMarkdown(report) {
const lines = [];
lines.push("# Mongo API Query Profile");
lines.push("");
lines.push(`Generated: ${report.generatedAt}`);
lines.push(`Base URL: \`${report.config.baseUrl}\``);
lines.push(`Mongo: \`${report.config.mongoContainer}/${report.config.mongoDb}\``);
lines.push("");
lines.push("This is a query-shape profile, not a max-throughput test. Request counts are intentionally small so the backend rate limiter does not dominate the profile.");
lines.push("");
lines.push("## Endpoint Summary");
lines.push("");
lines.push("| Endpoint | Requests | Avg | P95 | P99 | Non-2xx | Mongo ops | Top Mongo query |");
lines.push("|---|---:|---:|---:|---:|---:|---:|---|");
for (const result of report.results) {
const top = result.mongoProfile.groups[0];
lines.push(
[
`\`${result.method} ${result.path}\``,
result.requestCount,
`${result.latency.averageMs}ms`,
`${result.latency.p95Ms}ms`,
`${result.latency.p99Ms}ms`,
result.non2xx,
result.mongoProfile.totalOperations,
top ? `\`${top.collection}\` ${top.command} (${top.count}x, ${top.planSummary || "no plan"})` : "-",
].join(" | ").replace(/^/, "| ").replace(/$/, " |"),
);
}
lines.push("");
lines.push("## Query Groups");
for (const result of report.results) {
lines.push("");
lines.push(`### ${result.name}`);
lines.push("");
lines.push(`Path: \`${result.method} ${result.path}\``);
lines.push(`Status codes: \`${JSON.stringify(result.statusCodeStats)}\``);
lines.push("");
if (!result.mongoProfile.groups.length) {
lines.push("No Mongo operations captured in this endpoint window.");
continue;
}
lines.push("| Collection | Command | Count | Total ms | Avg ms | P95 ms | Plan | Docs | Keys | Returned | Shape |");
lines.push("|---|---|---:|---:|---:|---:|---|---:|---:|---:|---|");
for (const group of result.mongoProfile.groups.slice(0, 12)) {
lines.push(
[
`\`${group.collection}\``,
`\`${group.command}\``,
group.count,
group.millisTotal,
round(group.millisAverage),
group.millisP95,
`\`${group.planSummary || "-"}\``,
group.docsExamined,
group.keysExamined,
group.nreturned,
`\`${truncate(group.queryShape || "-", 140)}\``,
].join(" | ").replace(/^/, "| ").replace(/$/, " |"),
);
}
}
lines.push("");
lines.push("## Block I/O Deltas");
for (const result of report.results) {
const active = Object.entries(result.blockIoDelta)
.filter(([, value]) => value.readBytes || value.writeBytes)
.map(([name, value]) => `${name}: read ${formatBytes(value.readBytes)}, write ${formatBytes(value.writeBytes)}`);
lines.push(`- ${result.name}: ${active.length ? active.join("; ") : "no container block I/O delta"}`);
}
lines.push("");
return `${lines.join("\n")}\n`;
}
function truncate(value, max) {
return value.length > max ? `${value.slice(0, max - 3)}...` : value;
}
function round(value) {
return Math.round(value * 1000) / 1000;
}
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}