diff --git a/.obsidian/graph.json b/.obsidian/graph.json index 42385c9..850762e 100644 --- a/.obsidian/graph.json +++ b/.obsidian/graph.json @@ -17,6 +17,6 @@ "repelStrength": 10, "linkStrength": 1, "linkDistance": 250, - "scale": 0.5219627776444189, + "scale": 0.27561052639457884, "close": true } \ No newline at end of file diff --git a/09 - Audits/DB Migration Audit Report (2026-06-02).md b/09 - Audits/DB Migration Audit Report (2026-06-02).md new file mode 100644 index 0000000..3066a31 --- /dev/null +++ b/09 - Audits/DB Migration Audit Report (2026-06-02).md @@ -0,0 +1,272 @@ +# DB Migration Audit Report — Amanat Escrow (Mongo→PG) +**Date:** 2026-06-02 | **Scope:** Full Mongo→PG migration audit — schemas, indexes, constraints, dual-write coverage, backfill, verify harness, and service-layer Mongo-idiomatic patterns + +--- + +## Executive Summary + +The migration is **~50% complete** and **NOT ready for PG-primary cutover**. Schema and backfill scaffolding are mature (all 13 in-scope Mongo collections have Drizzle tables and backfill scripts), but three categories block cutover: + +1. **Migration correctness** — `0004_funds_ledger_entries.sql` is unjournaled (silently skips `funds_ledger_entries` on fresh DBs); `shadowRead()` exists but is never called from any read path so the soak window is completely blind. +2. **Financial integrity gaps** — missing `CHECK (amount > 0)` / `fx_rate > 0` constraints, ~20 FK columns declared in `relations()` only (never as physical FKs), backfill that silently writes `amount = '0'` for NULL Mongo amounts. +3. **Service-layer rework is far bigger than the schema work** — the factory (`createRepositories`) has **zero callers**; 30+ services still import Mongoose directly and contain ~50 Mongo-idiomatic patterns (N+1 loops, full-fetch+JS-filter, read-modify-write without locking, multi-table writes with no transaction) that will cause real money errors and lost updates under concurrent load. + +--- + +## Critical Issues (must fix before cutover) + +| # | Dimension | Table/File → Issue | Fix | +|---|---|---|---| +| CR-1 | Migration | `0004_funds_ledger_entries.sql` has NO `_journal.json` entry; `funds_ledger_entries` DDL conflicts between 0003 and 0004 — silently skipped on fresh DB, `ALTER` fails with "constraint already exists" if run | Reduce 0004 to trigger-only DDL; register in journal | +| CR-2 | Arch | `shadowRead.ts` exists and is complete, but no `DualWrite*Repo` read method ever calls it — soak window measures zero signal | Wire `shadowRead()` + `ShadowReadMetrics` into all 4 DualWrite read paths | +| CR-3 | Arch | Factory `createRepositories` has **zero callers** outside `src/db/repositories/` — every `REPO_*=dual` env flag changes env but routes zero traffic | Inject factory into 30+ service files (`paymentController`, `paymentCoordinator`, marketplace/user/points services) | +| CR-4 | Backfill | `backfill-payments.ts`: `extractDecimalString(null)` returns `'0'` — NULL Mongo payment amount silently inserted as `amount = 0` (money integrity violation) | Make `extractDecimalString(null)` throw, or skip + warn | +| CR-5 | Backfill | `backfill-derivedDestinations.ts`: `require()` failure falls back to `strict:false` model — all fields become `undefined`, all rows skipped silently, exits 0 | Make import failure `throw` and exit 1; never fall back to schema-less model | +| CR-6 | Backfill | `backfill-fundsLedger.ts` + `backfill-pointTransactions.ts`: `upsertIdMap` + `INSERT` not in one transaction — interruption leaves orphan id_map rows; re-run `DO NOTHING` never inserts data row → unrecoverable | Wrap `upsertIdMap` + data INSERT in one PG transaction | +| CR-7 | Schema | `payment_quotes`: no `CHECK (offer_amount > 0 AND fx_rate > 0 AND token_price_usd > 0)` — zero/negative FX rate → divide-by-zero in settlement | Add three CHECK constraints | +| CR-8 | Schema | `payments`: no `CHECK (amount > 0)` | `ALTER TABLE payments ADD CONSTRAINT ck_payments_amount_pos CHECK (amount > 0)` | +| CR-9 | Verify | `checksums.ts`: `.catch(() => [])` silently returns `[]` on DB connection failure → `hasMismatch=false`, gate passes green | Propagate errors; never swallow in comparison path | +| CR-10 | Verify | `shadowRead.ts`: Decimal128 detection (`constructor.name === 'Decimal128'`) breaks on `.lean()` POJO results (`{$numberDecimal:...}`) — every numeric field appears equal, silent false-negative | Normalize amounts to strings before compare; detect `$numberDecimal` key | +| CR-11 | Verify | `migration-fk-idmap.test.ts`: `skipIfUnreachable` returns early without `test.skip()` — money-safety tests PASS when DB is unreachable; CI exit 0 | Call `test.skip()` when `!isReachable`; CI must assert `MONEY_SAFETY_TESTS_SKIPPED` absence | +| CR-12 | Verify | `rowCounts.ts`: id_map coverage check exists for `payments` only — dropped id_map entry for other collections → dangling FK silently | Add id_map coverage checks for users/purchaseRequests/sellerOffers/fundsLedger/pointTransactions | +| CR-13 | Code | `paymentCoordinator.ts` `executePaymentUpdate`: read `status` → JS guard → write — two concurrent webhooks both read `pending`, both write (lost update) | `UPDATE payments SET status=... WHERE id=... AND status NOT IN ('completed','cancelled','refunded') RETURNING *`; 0 rows → abort | +| CR-14 | Code | `paymentCoordinator.ts` dispute gate: `isReleaseBlockedById(prId)` read, then payment update — dispute raised in the gap bypasses gate | `SELECT ... FOR UPDATE` on PR row + payment update in one transaction | +| CR-15 | Code | `paymentCoordinator.ts` `executePaymentUpdate`: payment update + ledger append + PR backfill + `acceptOffer` + duplicate cancel + template delete run with **NO transaction** — step 3/4 failure leaves payment completed but offer not accepted | Wrap all side effects in one DB transaction | +| CR-16 | Code | `DisputeService.ts` `createDispute`: dispute create → chat create → save → `setChatId` with no transaction — partial failure → orphaned dispute/chat, UI crashes | Wrap all four ops in one PG transaction | +| CR-17 | Code | `SellerOfferService.ts` `withdrawOffer` + `marketplaceController.ts` `validateStatusTransition`: read-validate-write status machine with no atomic guard | `UPDATE ... WHERE id=... AND status=... RETURNING *`; 0 rows → 409 Conflict | +| CR-18 | Code | `PointsService.ts` `getReferrals`/`collectDeliveredReferralOrders`: per-referred-user `while(true)` skip/limit loop + per-row offer lookup → 510+ queries/user; 14+ s per leaderboard on WAN | Replace with single CTE: `LEFT JOIN purchase_requests/seller_offers/point_transactions ... GROUP BY u.id` | +| CR-19 | Code | `PurchaseRequestService.ts` `searchPurchaseRequests`: `findPurchaseRequests({limit:100})` then JS `.filter().slice(0,20)` — catastrophic at 10k rows | `WHERE title ILIKE $s OR description ILIKE $s LIMIT 20`, or `tsvector` generated column + GIN | +| CR-20 | Code | `Chat` model: `messages[]`/`participants[]`/`unreadCounts[]` as JSONB — no FK integrity, unbounded row bloat, non-indexable | Child tables `chat_messages`, `chat_participants`, `chat_message_reactions`; rewrite `ChatService` as SQL | + +--- + +## High-Priority Issues + +### Schema — Missing Physical FKs +All declared via Drizzle `relations()` only, never as `foreignKey()`/`.references()`. Zero referential integrity enforcement in DB. + +- `users.referred_by_id` → add FK `ON DELETE SET NULL` +- `purchase_requests.buyer_id`, `category_id`, `selected_offer_id` +- All PR child tables (`purchase_request_delivery_info`, `_delivery_address`, `_seller_delivery_info`, `_service_info`, `_specifications`, `_preferred_sellers`) — `purchase_request_id`/`delivery_info_id` +- `delivery_attempts.delivery_info_id`, `seller_id` +- `derived_destinations.buyer_id`, `seller_id`, `seller_offer_id` +- `derived_destination_sweeps.destination_id` +- `trezor_accounts.user_id`; `trezor_derived_addresses.trezor_account_id` +- `funds_ledger_entries.purchase_request_id`, `payment_id` (deferred since 0003, never added) +- `point_transactions.user_id`, `referred_user_id` + +### Schema — Other HIGH +- `users`: no `CHECK (points_available >= 0 AND points_available <= points_total)` +- `point_transactions`: no `CHECK (balance >= 0)` +- `payment_quotes`: no `CHECK (settle_amount >= raw_settle_amount)` (snap-up invariant unenforced) +- `purchase_request_preferred_sellers`: no composite PK, only uniqueIndex +- `seller_offers.price_amount numeric(18,8)` vs project-wide `numeric(38,18)` (precision gap in settlement) + +### Migration — HIGH +- All 70+ `CREATE INDEX` are non-`CONCURRENTLY` (blocking SHARE lock on live data for all) +- All FK `ADD CONSTRAINT` run validating (no `NOT VALID` + later `VALIDATE`) — prolonged ACCESS EXCLUSIVE lock +- `blog_posts` and `notifications` exported from schema barrel but **no migration creates them** +- `disputes`/`chats` use `text` (not `uuid`) for FK columns — zero referential integrity +- Migration 0009: three sequential `UPDATE` DML steps not in `BEGIN/COMMIT` — partial failure leaves inconsistent category re-parenting + +### Backfill — HIGH +- `String(number)` for `numeric` columns risks scientific notation in `backfill-purchaseRequests.ts` (budget), `backfill-sellerOffers.ts` (price.amount), `backfill-requestTemplates.ts` (budget, proposal.price) +- `backfill-users.ts`: `email ?? null` fails if `users.email` is NOT NULL for OAuth-only users +- `backfill-fundsLedger.ts`: missing `d.entryType` (no default) → NOT NULL violation +- `run-backfill.ts`: `requestTemplates` runs in Tier B but runbook documents it last (inconsistency) + +### Verify — HIGH +- `reconcile.ts`: no double-refund detection; no `escrow_state`↔last-ledger-entry check; `LIMIT 1000` silently truncates +- `rowCounts.ts`: `estimatedDocumentCount()` is approximate — use `countDocuments({})` +- `checksums.ts`: no Mongo-side per-user points balance comparison during dual-write window +- `ledgerImmutability.ts`: `TRUNCATE` bypasses row-level trigger — add `BEFORE TRUNCATE` statement-level trigger +- Enum-value completeness verified nowhere + +### Code — HIGH +- `SellerOfferService.ts` `acceptOffer`: per-rejected-seller `createNotification` loop (use `createNotificationsBulk`); multi-UPDATE repo needs transaction +- `RequestTemplateService.ts` `batchConvertTemplates`: ~50 sequential queries per 10-item cart; no transaction per item → orphan PRs with no offer, oversold templates +- `paymentService.ts` `createPaymentRecord`: `String(metadata?.sellerId || createLegacyObjectIdString())` injects random fake ObjectIds as FKs → PG FK violation +- `userController.ts` `getUsersList`: `$regex` on name/email → PG seqscan; needs `pg_trgm` GIN index + `ILIKE` +- `PurchaseRequestService.ts` `updatePurchaseRequestStatus` (completed): non-idempotent double-points risk; no transaction + +--- + +## Medium Issues + +**Schema:** dual unique indexes on `categories.name` (drop raw, keep partial `WHERE is_active`); missing `payments(purchase_request_id, status)` composite index; missing `seller_offers(seller_id,status)`, `derived_destinations(address, chain_id)`, `trezor_derived_addresses.address` indexes; `id_map` no PK and `new_id` no unique constraint; `request_templates` no `CHECK (usage_count <= max_usage)`. + +**Migration:** `wallet_type` enum created but used in no column (dead DDL); `ALTER TYPE offer_currency ADD VALUE 'TRY'` requires PG 12+ in-transaction; `ck_pr_budget_currency_crypto` add(0006)/drop(0007) round-trip fails on rows with non-crypto values; `chats.participants` JSONB has no GIN index. + +**Backfill:** enum default mismatches (`provider:'request.network'` vs `request_network`); `escrow_state ?? null` may hit NOT NULL; `derivedDestinations.lastKnownBalance` via JS Number loses precision above 2^53 for wei. + +**Code:** `dataCleanupService.getCollectionStats` — 13 sequential `countDocuments()` (should be single subselect); `userController.updateUserProfile` writes arbitrary `profile.${key}` (whitelist needed); `paymentCoordinator` metadata read-spread-write overwrites concurrent keys (use `metadata || jsonb_build_object(...)`); skip/limit pagination in `getOffersBySeller`/`getUsersList`. + +--- + +## Index & Constraint Punch List + +| Table | Missing | Recommended DDL | +|---|---|---| +| payments | CHECK amount > 0 | `ALTER TABLE payments ADD CONSTRAINT ck_payments_amount_pos CHECK (amount > 0);` | +| payments | (purchase_request_id, status) | `CREATE INDEX CONCURRENTLY idx_payments_pr_status ON payments (purchase_request_id, status);` | +| payments | disputed partial | `CREATE INDEX CONCURRENTLY idx_payments_disputed ON payments (id) WHERE disputed = true;` | +| payment_quotes | CHECK money fields | `ALTER TABLE payment_quotes ADD CONSTRAINT ck_pq_pos CHECK (offer_amount > 0 AND fx_rate > 0 AND token_price_usd > 0);` | +| payment_quotes | CHECK snap-up | `ALTER TABLE payment_quotes ADD CONSTRAINT ck_pq_settle CHECK (settle_amount >= raw_settle_amount);` | +| users | referred_by_id FK | `ALTER TABLE users ADD CONSTRAINT users_referred_by_fk FOREIGN KEY (referred_by_id) REFERENCES users(id) ON DELETE SET NULL NOT VALID;` then `VALIDATE` | +| users | CHECK points | `ALTER TABLE users ADD CONSTRAINT ck_users_points CHECK (points_available >= 0 AND points_used >= 0 AND points_total >= 0 AND points_available <= points_total);` | +| point_transactions | CHECK balance | `ALTER TABLE point_transactions ADD CONSTRAINT ck_pt_balance CHECK (balance >= 0);` | +| funds_ledger_entries | FK pr + payment | `ALTER TABLE funds_ledger_entries ADD CONSTRAINT fle_pr_fk FOREIGN KEY (purchase_request_id) REFERENCES purchase_requests(id) NOT VALID;` then `VALIDATE` | +| funds_ledger_entries | TRUNCATE trigger | `CREATE TRIGGER funds_ledger_no_truncate BEFORE TRUNCATE ON funds_ledger_entries FOR EACH STATEMENT EXECUTE FUNCTION funds_ledger_immutable_fn();` | +| trezor_accounts | user_id FK | `ALTER TABLE trezor_accounts ADD CONSTRAINT ta_user_fk FOREIGN KEY (user_id) REFERENCES users(id) NOT VALID;` then `VALIDATE` | +| derived_destinations | buyer/seller/offer FK + (address,chain_id) | add 3 FKs `NOT VALID`; `CREATE INDEX CONCURRENTLY idx_dd_addr_chain ON derived_destinations (address, chain_id);` | +| purchase_requests | buyer/category/offer FK + (status,created_at) | add 3 FKs `NOT VALID`; `CREATE INDEX CONCURRENTLY idx_pr_status_created ON purchase_requests (status, created_at DESC);` | +| seller_offers | (seller_id,status) + (purchase_request_id,status) | `CREATE INDEX CONCURRENTLY idx_so_seller_status ON seller_offers (seller_id, status);` | +| id_map | PK + new_id unique | `ALTER TABLE id_map ADD PRIMARY KEY (collection, legacy_object_id); CREATE UNIQUE INDEX id_map_new_id_uq ON id_map (new_id);` | +| users (search) | trigram | `CREATE INDEX CONCURRENTLY idx_users_name_trgm ON users USING GIN (lower(first_name\|\|' '\|\|last_name\|\|' '\|\|coalesce(email,'')) gin_trgm_ops);` | +| purchase_requests | tags GIN | `CREATE INDEX CONCURRENTLY idx_pr_tags ON purchase_requests USING GIN (tags);` | + +--- + +## Repository Coverage Matrix + +| Interface | Drizzle Impl | Dual-Write | Status | +|---|---|---|---| +| PaymentRepo | Yes | Yes (shadow read NOT wired) | PARTIAL | +| UserRepo | Yes | Yes (shadow read NOT wired) | PARTIAL | +| MarketplaceRepo | Yes | Yes (shadow read NOT wired) | PARTIAL | +| PointsRepo | Yes | Yes (shadow read NOT wired) | PARTIAL | +| ReleaseHoldRepo | Yes | — | No dual-write | +| TrezorAccountRepo | Yes | — | No dual-write | +| DerivedDestinationRepo | Yes | — | No dual-write | + +**Factory has zero application callers (CR-3) — most critical architecture gap.** + +--- + +## Backfill Coverage Matrix + +| Mongo Collection | Backfill Script | Ordering | Status | +|---|---|---|---| +| users | backfill-users.ts | Tier A | OK (email NOT NULL risk) | +| categories | backfill-categories.ts | Tier A | OK | +| requestTemplates | backfill-requestTemplates.ts | Tier B | OK (String() decimals; runbook order mismatch) | +| purchaseRequests | backfill-purchaseRequests.ts (2-pass) | Tier B | OK (String() decimals; silent preferred-seller skips) | +| sellerOffers | backfill-sellerOffers.ts | Tier B | OK (String() price.amount) | +| payments | backfill-payments.ts | Tier C | **RISK** — NULL amount → '0' (CR-4) | +| fundsLedger | backfill-fundsLedger.ts | Tier C | **RISK** — non-txn idMap (CR-6); entryType NOT NULL | +| derivedDestinations | backfill-derivedDestinations.ts | Tier C | **RISK** — schema-less fallback (CR-5); wei precision | +| trezorAccounts | backfill-trezorAccounts.ts | Tier C | OK | +| pointTransactions | backfill-pointTransactions.ts | Tier C | **RISK** — non-txn idMap (CR-6); String() decimals | +| id_map | (infra — `_idMap.ts`) | n/a | CORRECT | +| payment_quotes | (none — runtime-generated) | n/a | EXPECTED | +| pg_dualwrite_gaps | (none — operational log) | n/a | EXPECTED | + +--- + +## Verification Coverage Matrix + +| Concern | Covered By | Gap | +|---|---|---| +| Row-count parity | rowCounts.ts (9/~23 collections) | `estimatedDocumentCount()` approximate; id_map not counted | +| ID-mapping completeness | rowCounts.ts (payments only) | **CRITICAL** — no check for users/PR/sellerOffers/FLE/pointTransactions | +| FK integrity | rowCounts.ts (7 pairs) | `seller_offers.seller_id`, `trezor_accounts→users` missing | +| Money sum accuracy | checksums.ts | `.catch(()=>[])` silent pass on conn failure (CR-9) | +| Ledger reconciliation | reconcile.ts | No double-refund; no `escrow_state`↔last-entry; LIMIT 1000 truncation | +| Ledger immutability | ledgerImmutability.ts | TRUNCATE bypass; no schema filter on `pg_proc` | +| Shadow read fidelity | shadowRead.ts | Decimal128 lean false-negative (CR-10); not wired (CR-2) | +| Enum completeness | — | **Not covered anywhere** | +| Timestamp precision/TZ | — | Not covered | +| CI gate output | boolean only | No JSON stdout; tests pass-not-skip on unreachable DB (CR-11) | + +--- + +## Models Not Yet in PG Schema + +| Mongo Model | Fields | Actively Used | Effort | Notes | +|---|---|---|---|---| +| Dispute | ~15 + 3 embedded arrays | Yes (DisputeService, releaseHoldService) | **L** | `evidence[]`/`timeline[]`/`messages[]` → child tables; pre-save timeline hook → service | +| Notification | 11 | Yes (all services, high frequency) | S schema / M migration | `userId` as text → uuid FK backfill; TTL index → pg_cron | +| ShopSettings | ~12 | Yes (marketplace template pages) | S | `paymentConfig.allowedChains int[]`, `socialLinks` → 4 columns | +| ConfigSetting (+History) | 4 (+audit) | Yes (walletMonitor, scanner threshold) | S | key-value; history child table | +| LevelConfig | ~10 | Yes (PointsService) | S | flatten `benefits{}` to 4 columns | +| Address | 10 | Yes (dataCleanup, delivery flows) | S | `addressType` pgEnum; one-primary partial unique | +| Review | 9 | Admin CMS only | S | polymorphic `subjectId` → ref_kind discriminator | +| TelegramLink | ~12 | Yes (auth) | S | two unique constraints; (userId, status) idx | +| TelegramSession | ~10 | Yes (auth middleware) | S | TTL `expiresAt` → pg_cron | +| BlogPost | ~20 | Admin CMS only | S | `videos[]` child table; slug/publishedAt pre-save → service | +| TempVerification | 8 | Registration only | S | TTL cleanup | + +--- + +## Mongo-Idiomatic Code Refactoring Tracker + +| Pattern | File | Function | Severity | Fix | +|---|---|---|---|---| +| N+1 | PointsService.ts | getReferrals / collectDeliveredReferralOrders | CRITICAL | Single CTE with LEFT JOINs + GROUP BY | +| N+1 | SellerOfferService.ts | acceptOffer | HIGH | `createNotificationsBulk` + single seller-id query | +| N+1 | RequestTemplateService.ts | batchConvertTemplates | HIGH | Batch SELECT ANY($links), batch INSERT…VALUES, single usage UPDATE | +| N+1 | dataCleanupService.ts | getCollectionStats | MEDIUM | Single subselect count query | +| Full-fetch+filter | PurchaseRequestService.ts | searchPurchaseRequests | CRITICAL | ILIKE/tsvector WHERE + LIMIT 20 | +| Full-fetch+filter | PurchaseRequestService.ts | createPurchaseRequest (dup detect) | HIGH | WHERE buyer/title/description/created_at LIMIT 1 | +| Full-fetch+filter | paymentCoordinator.ts | executePaymentUpdate (template cleanup) | HIGH | Push JSONB conditions into WHERE/DELETE | +| Full-fetch+filter | userController.ts | getUsersList | HIGH | pg_trgm GIN + ILIKE | +| JSONB no join table | Chat | messages/participants/unreadCounts | CRITICAL | 3 child tables (CR-20) | +| JSONB no join table | Dispute | evidence/timeline/messages | HIGH | 3 child tables on migration | +| JSONB schemaless | Payment | metadata | HIGH | Promote `is_template_checkout`, `rn_request_id` to typed columns | +| In-memory agg | PointsService.ts | sumDeliveredReferralSpend | CRITICAL | SUM in CTE | +| In-memory agg | SellerOfferService.ts | getOfferStatistics | MEDIUM | COUNT(*) OVER() / ROLLUP | +| Lost update | paymentCoordinator.ts | executePaymentUpdate | CRITICAL | UPDATE…WHERE status NOT IN (terminal) RETURNING | +| Lost update | SellerOfferService.ts | updateOffer | HIGH | UPDATE…WHERE status='pending' RETURNING | +| TOCTOU | SellerOfferService.ts | withdrawOffer | CRITICAL | UPDATE…WHERE id AND seller AND status='pending' | +| TOCTOU | marketplaceController.ts | validateStatusTransition | CRITICAL | UPDATE…WHERE status=$expected; 0 rows → 409 | +| TOCTOU | paymentCoordinator.ts | dispute gate | CRITICAL | FOR UPDATE on PR + same txn | +| TOCTOU | PurchaseRequestService.ts | updatePurchaseRequestStatus | HIGH | UPDATE…WHERE status=$old RETURNING | +| Missing txn | paymentCoordinator.ts | executePaymentUpdate | CRITICAL | One txn for all side effects | +| Missing txn | DisputeService.ts | createDispute | CRITICAL | One txn for dispute+chat+link | +| Missing txn | SellerOfferService.ts | acceptOffer (repo) | HIGH | Txn for accept/reject/PR update | +| Missing txn | RequestTemplateService.ts | batchConvertTemplates | HIGH | Txn (or savepoint) per cart item | +| Missing txn | PurchaseRequestService.ts | updatePurchaseRequestStatus (completed) | HIGH | Txn or outbox for referral reward | +| Schemaless write | paymentService.ts | createPaymentRecord | HIGH | Remove fake-ObjectId FK fallback | +| Schemaless write | userController.ts | updateUserProfile | MEDIUM | Whitelist + jsonb \|\| merge | +| Skip/limit pagination | PointsService.ts | collectDeliveredReferralOrders | CRITICAL | Replace loop with aggregate | +| Skip/limit pagination | PurchaseRequestService.ts | searchPurchaseRequests | HIGH | Keyset on (created_at, id) | +| Skip/limit pagination | SellerOfferService.ts / userController.ts | getOffersBySeller / getUsersList | MEDIUM | Keyset + cap limit 100 | +| Virtual/hook | Chat | addMessage/markAsRead/getUnreadCount | CRITICAL | SQL ops in ChatRepository | +| Pre-save hook | FundsLedgerEntry | immutability | HIGH | Apply trigger DDL now | + +--- + +## Migration Completion Assessment + +| Layer | % | +|---|---| +| Schema (Drizzle tables vs Mongo collections) | 90% | +| Repository layer | 70% | +| Backfill scripts | 85% | +| Verification harness | 75% | +| **Service layer (Mongo→RDBMS patterns)** | **5%** | +| **Overall** | **~50%** | + +### Top 5 Blockers for PG-Primary Cutover + +1. **Service-layer rework not started + factory uncalled (CR-3)** — flag flips route zero traffic; ~50 patterns including lost-update/missing-txn money bugs +2. **Transaction + locking defects on payment/escrow paths (CR-13–17)** — real money errors and lost updates under concurrent webhooks +3. **Shadow read unwired (CR-2)** — soak window is blind; cutover decision would be based on no signal +4. **Migration correctness: 0004 unjournaled + duplicate ledger DDL (CR-1)** — fresh-DB apply silently omits `funds_ledger_entries` +5. **Money-integrity gaps + verification silent-passes (CR-4/7/8/9/10/11/12)** — corruption can occur and pass green + +--- + +## Recommended Next Actions + +| # | Action | Files | Effort | +|---|---|---|---| +| 1 | Fix 0004 journal collision | `0004_funds_ledger_entries.sql`, `_journal.json` | S | +| 2 | Add money CHECK constraints + apply ledger TRUNCATE trigger | new migration on payments/payment_quotes/users/point_transactions/funds_ledger_entries | S | +| 3 | Fix backfill money/integrity defects (NULL amount, schema-less fallback, non-txn idMap, `_decimal.ts`) | backfill-payments/derivedDestinations/fundsLedger/pointTransactions/purchaseRequests | M | +| 4 | Close verification silent-passes (checksums, shadowRead, test.skip, id-map/enum/FK coverage, `--json` gate) | checksums.ts, shadowRead.ts, reconcile.ts, rowCounts.ts, migration-fk-idmap.test.ts | M | +| 5 | Add all deferred physical FKs `NOT VALID` + `VALIDATE`; rebuild blocking indexes `CONCURRENTLY` | new migration | M | +| 6 | Wire shadow reads into all 4 DualWrite read paths | DualWritePayment/User/Marketplace/PointsRepo | M | +| 7 | Inject factory into services + fix money/escrow concurrency (txn + `UPDATE…WHERE…RETURNING`) | paymentCoordinator, DisputeService, SellerOfferService, marketplaceController, PurchaseRequestService | **L** | +| 8 | Eliminate N+1 / full-fetch / skip-limit hotpaths | PointsService, searchPurchaseRequests, batchConvertTemplates, getUsersList | L | +| 9 | Schema + backfill for unmodeled active models | Dispute (L), Notification (M), ShopSettings/ConfigSetting/Address/Telegram* (S each) | L | diff --git a/09 - Audits/Mongo API Profiles/2026-05-31T14-26-19-969Z/mongo-api-profile.json b/09 - Audits/Mongo API Profiles/2026-05-31T14-26-19-969Z/mongo-api-profile.json new file mode 100644 index 0000000..f7049be --- /dev/null +++ b/09 - Audits/Mongo API Profiles/2026-05-31T14-26-19-969Z/mongo-api-profile.json @@ -0,0 +1,872 @@ +{ + "generatedAt": "2026-05-31T14:29:51.927Z", + "config": { + "baseUrl": "https://dev.manwe.qzz.io", + "sshHost": "root@5.78.213.189", + "mongoContainer": "amanat-dev-mongodb", + "mongoDb": "marketplace", + "mongoAuthDb": "admin", + "backendContainer": "amanat-dev-backend", + "resetBackendLimiter": true, + "containers": [ + "amanat-dev-nginx", + "amanat-dev-backend", + "amanat-dev-frontend", + "amanat-dev-postgres", + "amanat-dev-mongodb", + "amanat-dev-redis", + "amanat-dev-scanner" + ], + "templateShareableLink": "logo-design-template", + "outputDir": "/Users/manwe/CascadeProjects/escrow/nick-doc/09 - Audits/Mongo API Profiles/2026-05-31T14-26-19-969Z" + }, + "results": [ + { + "name": "health", + "method": "GET", + "path": "/api/health", + "requestCount": 5, + "rps": 2.5, + "latency": { + "averageMs": 327.2, + "p50Ms": 233, + "p90Ms": 707, + "p95Ms": 707, + "p99Ms": 707, + "maxMs": 707 + }, + "non2xx": 0, + "statusCodeStats": { + "200": { + "count": 5 + } + }, + "mongoProfile": { + "totalOperations": 0, + "totalMillis": 0, + "groups": [] + }, + "blockIoDelta": { + "amanat-dev-nginx": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-backend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-frontend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-postgres": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-mongodb": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-redis": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-scanner": { + "readBytes": 0, + "writeBytes": 10000 + } + } + }, + { + "name": "categories", + "method": "GET", + "path": "/api/marketplace/categories", + "requestCount": 10, + "rps": 3.34, + "latency": { + "averageMs": 390.6, + "p50Ms": 232, + "p90Ms": 731, + "p95Ms": 1308, + "p99Ms": 1308, + "maxMs": 1308 + }, + "non2xx": 0, + "statusCodeStats": { + "200": { + "count": 10 + } + }, + "mongoProfile": { + "totalOperations": 0, + "totalMillis": 0, + "groups": [] + }, + "blockIoDelta": { + "amanat-dev-nginx": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-backend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-frontend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-postgres": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-mongodb": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-redis": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-scanner": { + "readBytes": 0, + "writeBytes": 0 + } + } + }, + { + "name": "categories_tree", + "method": "GET", + "path": "/api/marketplace/categories/tree", + "requestCount": 10, + "rps": 5, + "latency": { + "averageMs": 342.5, + "p50Ms": 240, + "p90Ms": 742, + "p95Ms": 752, + "p99Ms": 752, + "maxMs": 752 + }, + "non2xx": 0, + "statusCodeStats": { + "200": { + "count": 10 + } + }, + "mongoProfile": { + "totalOperations": 10, + "totalMillis": 0, + "groups": [ + { + "namespace": "marketplace.categories", + "operation": "query", + "command": "find", + "collection": "categories", + "planSummary": "IXSCAN { isActive: 1 }", + "queryHash": "35A725FF", + "planCacheKey": "80333596", + "queryShape": "filter={isActive:boolean} sort={name:number,order:number}", + "count": 10, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 240, + "keysExamined": 240, + "nreturned": 240, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 65670, + "numYield": 0 + } + ] + }, + "blockIoDelta": { + "amanat-dev-nginx": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-backend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-frontend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-postgres": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-mongodb": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-redis": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-scanner": { + "readBytes": 0, + "writeBytes": 0 + } + } + }, + { + "name": "sellers", + "method": "GET", + "path": "/api/marketplace/sellers", + "requestCount": 10, + "rps": 5, + "latency": { + "averageMs": 341.6, + "p50Ms": 245, + "p90Ms": 729, + "p95Ms": 733, + "p99Ms": 733, + "maxMs": 733 + }, + "non2xx": 0, + "statusCodeStats": { + "200": { + "count": 10 + } + }, + "mongoProfile": { + "totalOperations": 10, + "totalMillis": 0, + "groups": [ + { + "namespace": "marketplace.users", + "operation": "query", + "command": "find", + "collection": "users", + "planSummary": "IXSCAN { role: 1 }", + "queryHash": "BA1E76D1", + "planCacheKey": "0CB19E91", + "queryShape": "filter={isEmailVerified:boolean,role:string} projection={_id:number,email:number,firstName:number,lastName:number,profile.avatar:number}", + "count": 10, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 20, + "keysExamined": 20, + "nreturned": 20, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 3610, + "numYield": 0 + } + ] + }, + "blockIoDelta": { + "amanat-dev-nginx": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-backend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-frontend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-postgres": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-mongodb": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-redis": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-scanner": { + "readBytes": 0, + "writeBytes": 20000 + } + } + }, + { + "name": "template_public", + "method": "GET", + "path": "/api/marketplace/request-templates/public/logo-design-template", + "requestCount": 10, + "rps": 5, + "latency": { + "averageMs": 340.3, + "p50Ms": 241, + "p90Ms": 734, + "p95Ms": 740, + "p99Ms": 740, + "maxMs": 740 + }, + "non2xx": 0, + "statusCodeStats": { + "200": { + "count": 10 + } + }, + "mongoProfile": { + "totalOperations": 30, + "totalMillis": 0, + "groups": [ + { + "namespace": "marketplace.requesttemplates", + "operation": "query", + "command": "find", + "collection": "requesttemplates", + "planSummary": "IXSCAN { shareableLink: 1 }", + "queryHash": "69A943C9", + "planCacheKey": "7C668FB5", + "queryShape": "filter={$or:[{expiresAt:null},{expiresAt:{$gt:{}}}],isActive:boolean,shareableLink:string}", + "count": 10, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 10, + "keysExamined": 10, + "nreturned": 10, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 15470, + "numYield": 0 + }, + { + "namespace": "marketplace.users", + "operation": "query", + "command": "find", + "collection": "users", + "planSummary": "IXSCAN { _id: 1 }", + "queryHash": "39E03FF8", + "planCacheKey": "AED36A0D", + "queryShape": "filter={_id:{$in:[ObjectId]}} projection={email:number,firstName:number,lastName:number}", + "count": 10, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 10, + "keysExamined": 10, + "nreturned": 10, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 2180, + "numYield": 0 + }, + { + "namespace": "marketplace.categories", + "operation": "query", + "command": "find", + "collection": "categories", + "planSummary": "IXSCAN { _id: 1 }", + "queryHash": "ABAD6477", + "planCacheKey": "E494D204", + "queryShape": "filter={_id:{$in:[ObjectId]}} projection={name:number,nameEn:number}", + "count": 10, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 10, + "keysExamined": 10, + "nreturned": 10, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 1890, + "numYield": 0 + } + ] + }, + "blockIoDelta": { + "amanat-dev-nginx": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-backend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-frontend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-postgres": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-mongodb": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-redis": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-scanner": { + "readBytes": 0, + "writeBytes": 0 + } + } + }, + { + "name": "payment_options_template", + "method": "GET", + "path": "/api/payment/request-network/options?currency=USD&amount=0.01&sellerId=6a1bfd1400e8b8205e86db9e&templateId=6a1c4512d07eb576c3509690", + "requestCount": 50, + "rps": 12.5, + "latency": { + "averageMs": 303.52, + "p50Ms": 255, + "p90Ms": 273, + "p95Ms": 753, + "p99Ms": 758, + "maxMs": 758 + }, + "non2xx": 0, + "statusCodeStats": { + "200": { + "count": 50 + } + }, + "mongoProfile": { + "totalOperations": 100, + "totalMillis": 0, + "groups": [ + { + "namespace": "marketplace.requesttemplates", + "operation": "query", + "command": "find", + "collection": "requesttemplates", + "planSummary": "IDHACK", + "queryHash": "3B008735", + "planCacheKey": "", + "queryShape": "filter={_id:ObjectId} projection={paymentConfig:number}", + "count": 50, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 50, + "keysExamined": 50, + "nreturned": 50, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 12850, + "numYield": 0 + }, + { + "namespace": "marketplace.shopsettings", + "operation": "query", + "command": "find", + "collection": "shopsettings", + "planSummary": "IXSCAN { sellerId: 1 }", + "queryHash": "BF51CF8A", + "planCacheKey": "9CF87C58", + "queryShape": "filter={sellerId:ObjectId} projection={paymentConfig:number}", + "count": 50, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 0, + "keysExamined": 0, + "nreturned": 0, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 5650, + "numYield": 0 + } + ] + }, + "blockIoDelta": { + "amanat-dev-nginx": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-backend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-frontend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-postgres": { + "readBytes": 100000, + "writeBytes": 0 + }, + "amanat-dev-mongodb": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-redis": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-scanner": { + "readBytes": 0, + "writeBytes": 0 + } + } + }, + { + "name": "addresses_me", + "method": "GET", + "path": "/api/addresses", + "requestCount": 10, + "rps": 5, + "latency": { + "averageMs": 330.9, + "p50Ms": 239, + "p90Ms": 707, + "p95Ms": 715, + "p99Ms": 715, + "maxMs": 715 + }, + "non2xx": 0, + "statusCodeStats": { + "200": { + "count": 10 + } + }, + "mongoProfile": { + "totalOperations": 10, + "totalMillis": 0, + "groups": [ + { + "namespace": "marketplace.addresses", + "operation": "query", + "command": "find", + "collection": "addresses", + "planSummary": "IXSCAN { userId: 1 }", + "queryHash": "6935090D", + "planCacheKey": "C80BED60", + "queryShape": "filter={userId:ObjectId} sort={createdAt:number,primary:number}", + "count": 10, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 30, + "keysExamined": 30, + "nreturned": 30, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 13800, + "numYield": 0 + } + ] + }, + "blockIoDelta": { + "amanat-dev-nginx": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-backend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-frontend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-postgres": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-mongodb": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-redis": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-scanner": { + "readBytes": 0, + "writeBytes": 0 + } + } + }, + { + "name": "purchase_requests_my", + "method": "GET", + "path": "/api/marketplace/purchase-requests/my", + "requestCount": 10, + "rps": 5, + "latency": { + "averageMs": 353.3, + "p50Ms": 256, + "p90Ms": 747, + "p95Ms": 753, + "p99Ms": 753, + "maxMs": 753 + }, + "non2xx": 0, + "statusCodeStats": { + "200": { + "count": 10 + } + }, + "mongoProfile": { + "totalOperations": 30, + "totalMillis": 1, + "groups": [ + { + "namespace": "marketplace.purchaserequests", + "operation": "query", + "command": "find", + "collection": "purchaserequests", + "planSummary": "IXSCAN { createdAt: -1 }", + "queryHash": "6F3C3F41", + "planCacheKey": "A22CDD0E", + "queryShape": "filter={buyerId:ObjectId} sort={createdAt:number}", + "count": 10, + "millisTotal": 1, + "millisAverage": 0.1, + "millisP50": 0, + "millisP95": 1, + "millisMax": 1, + "docsExamined": 0, + "keysExamined": 0, + "nreturned": 0, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 1170, + "numYield": 0 + }, + { + "namespace": "marketplace.purchaserequests", + "operation": "command", + "command": "aggregate", + "collection": "purchaserequests", + "planSummary": "COUNT_SCAN { buyerId: 1 }", + "queryHash": "C22625EF", + "planCacheKey": "BD75157B", + "queryShape": "pipeline=[{$match:{buyerId:ObjectId}},{$group:{_id:number,n:{$sum:number}}}]", + "count": 10, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 0, + "keysExamined": 10, + "nreturned": 0, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 1170, + "numYield": 0 + }, + { + "namespace": "marketplace.payments", + "operation": "query", + "command": "find", + "collection": "payments", + "planSummary": "IXSCAN { status: 1, createdAt: -1 }, IXSCAN { status: 1, createdAt: -1 }, IXSCAN { status: 1, createdAt: -1 }, IXSCAN { status: 1, createdAt: -1 }", + "queryHash": "3B29FB2B", + "planCacheKey": "8762DEE5", + "queryShape": "filter={purchaseRequestId:{$in:[]},status:{$in:[string,string,string,string]}} sort={createdAt:number}", + "count": 10, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 0, + "keysExamined": 0, + "nreturned": 0, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 1090, + "numYield": 0 + } + ] + }, + "blockIoDelta": { + "amanat-dev-nginx": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-backend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-frontend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-postgres": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-mongodb": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-redis": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-scanner": { + "readBytes": 0, + "writeBytes": 20000 + } + } + }, + { + "name": "auth_login", + "method": "POST", + "path": "/api/auth/login", + "requestCount": 5, + "rps": 1.25, + "latency": { + "averageMs": 724.2, + "p50Ms": 636, + "p90Ms": 1090, + "p95Ms": 1090, + "p99Ms": 1090, + "maxMs": 1090 + }, + "non2xx": 0, + "statusCodeStats": { + "200": { + "count": 5 + } + }, + "mongoProfile": { + "totalOperations": 15, + "totalMillis": 0, + "groups": [ + { + "namespace": "marketplace.users", + "operation": "query", + "command": "find", + "collection": "users", + "planSummary": "IXSCAN { email: 1 }", + "queryHash": "106ECB7C", + "planCacheKey": "AB4716E0", + "queryShape": "filter={email:string,status:string}", + "count": 5, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 5, + "keysExamined": 5, + "nreturned": 5, + "ninserted": 0, + "nMatched": 0, + "nModified": 0, + "responseLength": 17735, + "numYield": 0 + }, + { + "namespace": "marketplace.users", + "operation": "update", + "command": "q", + "collection": "users", + "planSummary": "IDHACK", + "queryHash": "", + "planCacheKey": "", + "queryShape": "", + "count": 5, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 5, + "keysExamined": 5, + "nreturned": 0, + "ninserted": 0, + "nMatched": 5, + "nModified": 5, + "responseLength": 0, + "numYield": 0 + }, + { + "namespace": "marketplace.users", + "operation": "update", + "command": "q", + "collection": "users", + "planSummary": "IXSCAN { _id: 1 }", + "queryHash": "E515C562", + "planCacheKey": "5EA96075", + "queryShape": "", + "count": 5, + "millisTotal": 0, + "millisAverage": 0, + "millisP50": 0, + "millisP95": 0, + "millisMax": 0, + "docsExamined": 5, + "keysExamined": 5, + "nreturned": 0, + "ninserted": 0, + "nMatched": 5, + "nModified": 5, + "responseLength": 0, + "numYield": 0 + } + ] + }, + "blockIoDelta": { + "amanat-dev-nginx": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-backend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-frontend": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-postgres": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-mongodb": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-redis": { + "readBytes": 0, + "writeBytes": 0 + }, + "amanat-dev-scanner": { + "readBytes": 0, + "writeBytes": 0 + } + } + } + ] +} diff --git a/09 - Audits/Mongo API Profiles/2026-05-31T14-26-19-969Z/summary.md b/09 - Audits/Mongo API Profiles/2026-05-31T14-26-19-969Z/summary.md new file mode 100644 index 0000000..f3d0e96 --- /dev/null +++ b/09 - Audits/Mongo API Profiles/2026-05-31T14-26-19-969Z/summary.md @@ -0,0 +1,119 @@ +# Mongo API Query Profile + +Generated: 2026-05-31T14:29:51.927Z +Base URL: `https://dev.manwe.qzz.io` +Mongo: `amanat-dev-mongodb/marketplace` + +This is a query-shape profile, not a max-throughput test. Request counts are intentionally small so the backend rate limiter does not dominate the profile. + +## Endpoint Summary + +| Endpoint | Requests | Avg | P95 | P99 | Non-2xx | Mongo ops | Top Mongo query | +|---|---:|---:|---:|---:|---:|---:|---| +| `GET /api/health` | 5 | 327.2ms | 707ms | 707ms | 0 | 0 | - | +| `GET /api/marketplace/categories` | 10 | 390.6ms | 1308ms | 1308ms | 0 | 0 | - | +| `GET /api/marketplace/categories/tree` | 10 | 342.5ms | 752ms | 752ms | 0 | 10 | `categories` find (10x, IXSCAN { isActive: 1 }) | +| `GET /api/marketplace/sellers` | 10 | 341.6ms | 733ms | 733ms | 0 | 10 | `users` find (10x, IXSCAN { role: 1 }) | +| `GET /api/marketplace/request-templates/public/logo-design-template` | 10 | 340.3ms | 740ms | 740ms | 0 | 30 | `requesttemplates` find (10x, IXSCAN { shareableLink: 1 }) | +| `GET /api/payment/request-network/options?currency=USD&amount=0.01&sellerId=6a1bfd1400e8b8205e86db9e&templateId=6a1c4512d07eb576c3509690` | 50 | 303.52ms | 753ms | 758ms | 0 | 100 | `requesttemplates` find (50x, IDHACK) | +| `GET /api/addresses` | 10 | 330.9ms | 715ms | 715ms | 0 | 10 | `addresses` find (10x, IXSCAN { userId: 1 }) | +| `GET /api/marketplace/purchase-requests/my` | 10 | 353.3ms | 753ms | 753ms | 0 | 30 | `purchaserequests` find (10x, IXSCAN { createdAt: -1 }) | +| `POST /api/auth/login` | 5 | 724.2ms | 1090ms | 1090ms | 0 | 15 | `users` find (5x, IXSCAN { email: 1 }) | + +## Query Groups + +### health + +Path: `GET /api/health` +Status codes: `{"200":{"count":5}}` + +No Mongo operations captured in this endpoint window. + +### categories + +Path: `GET /api/marketplace/categories` +Status codes: `{"200":{"count":10}}` + +No Mongo operations captured in this endpoint window. + +### categories_tree + +Path: `GET /api/marketplace/categories/tree` +Status codes: `{"200":{"count":10}}` + +| Collection | Command | Count | Total ms | Avg ms | P95 ms | Plan | Docs | Keys | Returned | Shape | +|---|---|---:|---:|---:|---:|---|---:|---:|---:|---| +| `categories` | `find` | 10 | 0 | 0 | 0 | `IXSCAN { isActive: 1 }` | 240 | 240 | 240 | `filter={isActive:boolean} sort={name:number,order:number}` | + +### sellers + +Path: `GET /api/marketplace/sellers` +Status codes: `{"200":{"count":10}}` + +| Collection | Command | Count | Total ms | Avg ms | P95 ms | Plan | Docs | Keys | Returned | Shape | +|---|---|---:|---:|---:|---:|---|---:|---:|---:|---| +| `users` | `find` | 10 | 0 | 0 | 0 | `IXSCAN { role: 1 }` | 20 | 20 | 20 | `filter={isEmailVerified:boolean,role:string} projection={_id:number,email:number,firstName:number,lastName:number,profile.avatar:number}` | + +### template_public + +Path: `GET /api/marketplace/request-templates/public/logo-design-template` +Status codes: `{"200":{"count":10}}` + +| Collection | Command | Count | Total ms | Avg ms | P95 ms | Plan | Docs | Keys | Returned | Shape | +|---|---|---:|---:|---:|---:|---|---:|---:|---:|---| +| `requesttemplates` | `find` | 10 | 0 | 0 | 0 | `IXSCAN { shareableLink: 1 }` | 10 | 10 | 10 | `filter={$or:[{expiresAt:null},{expiresAt:{$gt:{}}}],isActive:boolean,shareableLink:string}` | +| `users` | `find` | 10 | 0 | 0 | 0 | `IXSCAN { _id: 1 }` | 10 | 10 | 10 | `filter={_id:{$in:[ObjectId]}} projection={email:number,firstName:number,lastName:number}` | +| `categories` | `find` | 10 | 0 | 0 | 0 | `IXSCAN { _id: 1 }` | 10 | 10 | 10 | `filter={_id:{$in:[ObjectId]}} projection={name:number,nameEn:number}` | + +### payment_options_template + +Path: `GET /api/payment/request-network/options?currency=USD&amount=0.01&sellerId=6a1bfd1400e8b8205e86db9e&templateId=6a1c4512d07eb576c3509690` +Status codes: `{"200":{"count":50}}` + +| Collection | Command | Count | Total ms | Avg ms | P95 ms | Plan | Docs | Keys | Returned | Shape | +|---|---|---:|---:|---:|---:|---|---:|---:|---:|---| +| `requesttemplates` | `find` | 50 | 0 | 0 | 0 | `IDHACK` | 50 | 50 | 50 | `filter={_id:ObjectId} projection={paymentConfig:number}` | +| `shopsettings` | `find` | 50 | 0 | 0 | 0 | `IXSCAN { sellerId: 1 }` | 0 | 0 | 0 | `filter={sellerId:ObjectId} projection={paymentConfig:number}` | + +### addresses_me + +Path: `GET /api/addresses` +Status codes: `{"200":{"count":10}}` + +| Collection | Command | Count | Total ms | Avg ms | P95 ms | Plan | Docs | Keys | Returned | Shape | +|---|---|---:|---:|---:|---:|---|---:|---:|---:|---| +| `addresses` | `find` | 10 | 0 | 0 | 0 | `IXSCAN { userId: 1 }` | 30 | 30 | 30 | `filter={userId:ObjectId} sort={createdAt:number,primary:number}` | + +### purchase_requests_my + +Path: `GET /api/marketplace/purchase-requests/my` +Status codes: `{"200":{"count":10}}` + +| Collection | Command | Count | Total ms | Avg ms | P95 ms | Plan | Docs | Keys | Returned | Shape | +|---|---|---:|---:|---:|---:|---|---:|---:|---:|---| +| `purchaserequests` | `find` | 10 | 1 | 0.1 | 1 | `IXSCAN { createdAt: -1 }` | 0 | 0 | 0 | `filter={buyerId:ObjectId} sort={createdAt:number}` | +| `purchaserequests` | `aggregate` | 10 | 0 | 0 | 0 | `COUNT_SCAN { buyerId: 1 }` | 0 | 10 | 0 | `pipeline=[{$match:{buyerId:ObjectId}},{$group:{_id:number,n:{$sum:number}}}]` | +| `payments` | `find` | 10 | 0 | 0 | 0 | `IXSCAN { status: 1, createdAt: -1 }, IXSCAN { status: 1, createdAt: -1 }, IXSCAN { status: 1, createdAt: -1 }, IXSCAN { status: 1, createdAt: -1 }` | 0 | 0 | 0 | `filter={purchaseRequestId:{$in:[]},status:{$in:[string,string,string,string]}} sort={createdAt:number}` | + +### auth_login + +Path: `POST /api/auth/login` +Status codes: `{"200":{"count":5}}` + +| Collection | Command | Count | Total ms | Avg ms | P95 ms | Plan | Docs | Keys | Returned | Shape | +|---|---|---:|---:|---:|---:|---|---:|---:|---:|---| +| `users` | `find` | 5 | 0 | 0 | 0 | `IXSCAN { email: 1 }` | 5 | 5 | 5 | `filter={email:string,status:string}` | +| `users` | `q` | 5 | 0 | 0 | 0 | `IDHACK` | 5 | 5 | 0 | `-` | +| `users` | `q` | 5 | 0 | 0 | 0 | `IXSCAN { _id: 1 }` | 5 | 5 | 0 | `-` | + +## Block I/O Deltas +- health: amanat-dev-scanner: read 0 B, write 10 KB +- categories: no container block I/O delta +- categories_tree: no container block I/O delta +- sellers: amanat-dev-scanner: read 0 B, write 20 KB +- template_public: no container block I/O delta +- payment_options_template: amanat-dev-postgres: read 100 KB, write 0 B +- addresses_me: no container block I/O delta +- purchase_requests_my: amanat-dev-scanner: read 0 B, write 20 KB +- auth_login: no container block I/O delta + diff --git a/Issues/ISSUE-136-backend-api-profiling-mongo-hot-path-cache-query-fanout.md b/Issues/ISSUE-136-backend-api-profiling-mongo-hot-path-cache-query-fanout.md new file mode 100644 index 0000000..5b0331c --- /dev/null +++ b/Issues/ISSUE-136-backend-api-profiling-mongo-hot-path-cache-query-fanout.md @@ -0,0 +1,128 @@ +--- +issue: 136 +title: "Backend: API profiling shows Mongo hot paths are index-backed but still do avoidable repeated reads" +severity: low +domain: Performance +labels: [performance, backend, mongodb, caching] +status: open +created: 2026-05-31 +source: Live dev performance profiling 2026-05-31 +--- + +# Backend: API profiling shows Mongo hot paths are index-backed but still do avoidable repeated reads + +**Severity:** low +**Domain:** Performance +**Labels:** performance, backend, mongodb, caching + +## Description + +Live profiling on `dev.manwe.qzz.io` after the direct Caddy cutover showed the tested API routes are still Mongo-backed, not Postgres-backed. Postgres `pg_stat_statements` recorded no application SQL for these request windows; the only SQL activity came from profiling probes and database housekeeping. + +The good news: the Mongo queries that did run were using indexes and were effectively sub-millisecond on the tiny dev dataset. The useful optimization work is therefore not "fix a slow scan" yet, but reducing repeated query fan-out and avoiding unnecessary reads before real data volume grows. + +## Profiling Snapshot + +Environment: + +- Backend `2.6.84` +- Public entrypoint: `https://dev.manwe.qzz.io` through Caddy -> local nginx -> backend +- MongoDB `6.0.28`, bounded `system.profile` capture with `slowms: 0` +- PostgreSQL `18.4`, `pg_stat_statements` enabled +- Docker block I/O deltas sampled per endpoint window + +Representative results: + +| Endpoint | Result | Mongo profile | +|---|---:|---| +| `GET /api/marketplace/categories` | 40 req, avg 296ms, p99 721ms | 1 total `categories` query across the window; Redis category cache is working | +| `GET /api/marketplace/categories/tree` | 40 req, avg 319ms, p99 763ms | 40 `categories` queries, 24 docs/request, `IXSCAN { isActive: 1 }` | +| `GET /api/marketplace/sellers` | 20 req, avg 306ms, p99 722ms | 1 `users` query/request, `IXSCAN { role: 1 }` | +| `GET /api/marketplace/request-templates/public/:shareableLink` | 20 req, avg 315ms, p99 781ms | 3 queries/request: template by `shareableLink`, seller by `_id`, category by `_id` | +| `GET /api/payment/request-network/options?sellerId&templateId` | 293 req, avg 270ms, p99 806ms | 2 queries/request: `requesttemplates` by `_id`, `shopsettings` by `sellerId` | +| `GET /api/addresses` | 15 req, avg 315ms, p99 722ms | 1 `addresses` query/request, `IXSCAN { userId: 1 }` | +| `GET /api/marketplace/purchase-requests/my` | 15 req, avg 314ms, p99 702ms | 3 queries/request: request list, request count, payment lookup | +| `POST /api/auth/login` | 5 req, avg 691ms, p99 1058ms | user find + user updates; Mongo time rounded to 0ms, so latency is likely bcrypt/JWT/update path rather than query scan | + +Disk write notes: + +- Mongo profiler itself wrote about 1 MB in some windows; this is measurement overhead, not normal app write load. +- Scanner wrote small background chunks, about 17-24 KB, unrelated to the tested API routes. +- No meaningful app SQL write/read pressure appeared in the API windows. + +## Recommended Work + +1. Cache `GET /api/marketplace/categories/tree`. + - `getAllCategories()` already uses Redis via `cacheService`. + - `getCategoryTree()` re-queries Mongo every request even though it uses the same active category set. + +2. Add a short TTL cache for resolved payment rails. + - Cache by `(sellerId, templateId)` or a normalized key. + - Invalidate on template payment config updates and shop settings updates. + - This removes two Mongo reads from every checkout options poll. + +3. Avoid empty payment lookups in `getPurchaseRequestsByBuyer`. + - The profiled buyer had no matching requests, but the code still performed a `payments` lookup with `purchaseRequestId: { $in: [] }`. + - Short-circuit when the page result is empty. + +4. Add or confirm compound indexes before the dataset grows. + - `PurchaseRequest`: `{ buyerId: 1, createdAt: -1 }` + - `Address`: `{ userId: 1, primary: -1, createdAt: -1 }` + - `User`: consider `{ role: 1, isEmailVerified: 1 }` for public seller lists. + +5. Keep `pg_stat_statements` enabled and add a repeatable profiling script. + - The dev server now has SQL statement profiling available. + - `nick-doc/scripts/profile-mongo-api.mjs` runs the Mongo profiler around the same endpoint matrix and emits JSON/Markdown so future Postgres migrations can be compared cleanly. + +## Affected Files + +- `backend/src/services/marketplace/CategoryService.ts:38` — category tree bypasses the existing category cache. +- `backend/src/services/payment/sellerPaymentConfig.ts:44` — payment rail resolver performs per-call template/shop lookups. +- `backend/src/services/marketplace/RequestTemplateService.ts:465` — public template fetch fans out to template, seller, and category queries. +- `backend/src/services/marketplace/PurchaseRequestService.ts` — buyer list should skip downstream payment lookup when no request IDs exist. +- `backend/src/services/address/addressController.ts:23` — address list is index-backed; compound sort index should be confirmed. + +## Acceptance Criteria + +1. `categories/tree` uses the same cached category source as `categories` and invalidates on category writes. +2. Payment options with `sellerId` + `templateId` returns from cache for repeated calls and invalidates on relevant seller/template settings changes. +3. Empty purchase request result pages do not issue a `payments` query. +4. Required compound indexes are present in model definitions or migration/startup index setup. +5. A repeatable profiling command records per-endpoint latency, Mongo query groups, and block I/O deltas; SQL profiling remains available through the separate `pg_stat_statements` pass. + +## Latest Repeatable Mongo Profile + +Generated report: + +- `09 - Audits/Mongo API Profiles/2026-05-31T14-26-19-969Z/summary.md` +- `09 - Audits/Mongo API Profiles/2026-05-31T14-26-19-969Z/mongo-api-profile.json` + +This pass used the script's `RESET_BACKEND_LIMITER=1` option to restart the dev backend first. That clears the process-local `express-rate-limit` global limiter, so authenticated endpoint windows profile Mongo queries instead of stale 429 responses. + +Clean run highlights: + +| Endpoint | Requests | Non-2xx | Mongo ops | Top Mongo profile | +|---|---:|---:|---:|---| +| `GET /api/marketplace/categories/tree` | 10 | 0 | 10 | `categories` find, `IXSCAN { isActive: 1 }`, 24 docs/request | +| `GET /api/payment/request-network/options?sellerId&templateId` | 50 | 0 | 100 | `requesttemplates` IDHACK + `shopsettings` by `sellerId` every request | +| `GET /api/addresses` | 10 | 0 | 10 | `addresses` find, `IXSCAN { userId: 1 }`, sorted by `primary` and `createdAt` | +| `GET /api/marketplace/purchase-requests/my` | 10 | 0 | 30 | list + count + unnecessary `payments` query with `purchaseRequestId: { $in: [] }` | +| `POST /api/auth/login` | 5 | 0 | 15 | indexed user find plus user update operations; latency is outside Mongo query time | + +## Repeatable Mongo Profiling Command + +```bash +cd nick-doc +RESET_BACKEND_LIMITER=1 \ +BASE_URL=https://dev.manwe.qzz.io \ +SSH_HOST=root@5.78.213.189 \ +SSH_KEY=~/CascadeProjects/wzp \ +node scripts/profile-mongo-api.mjs +``` + +The script enables Mongo `system.profile` only for each bounded endpoint window, groups captured query shapes by namespace/plan/query shape, writes a raw JSON profile and a Markdown summary under `09 - Audits/Mongo API Profiles/`, then disables profiling again. Leave `RESET_BACKEND_LIMITER` unset when you do not want the script to restart the dev backend. + +## References + +- Live profiling on `dev.manwe.qzz.io` — 2026-05-31 +- Caddy direct-path profiling pass after Pangolin/newt removal diff --git a/Issues/Issues Index.md b/Issues/Issues Index.md index 4b9cb67..a59e8eb 100644 --- a/Issues/Issues Index.md +++ b/Issues/Issues Index.md @@ -68,3 +68,4 @@ ## 🟡 Minor +- [[ISSUE-136-backend-api-profiling-mongo-hot-path-cache-query-fanout|Backend: API profiling shows Mongo hot paths are index-backed but still do avoidable repeated reads]] — `Performance` diff --git a/PRD - Direct Address Token Payments via Scanner Balance Watches.md b/PRD - Direct Address Token Payments via Scanner Balance Watches.md index 7bad7aa..053e868 100644 --- a/PRD - Direct Address Token Payments via Scanner Balance Watches.md +++ b/PRD - Direct Address Token Payments via Scanner Balance Watches.md @@ -289,9 +289,9 @@ The smoke can mock scanner responses at first. A live smoke should be added only | Phase | Scope | Status | |---|---|---| | 0 | Scanner primitives and backend low-level adapter/webhook recorder | Done in scanner `0.1.8`, backend `2.8.60` | -| 1 | Backend direct-balance service and payment metadata shape | Not started | -| 2 | Check-on-click API path and tests | Not started | -| 3 | Watch-mode lifecycle, webhook decision, and stop cleanup | Not started | +| 1 | Backend direct-balance service and payment metadata shape | Done — `directBalancePaymentService.ts` | +| 2 | Check-on-click API path and tests | Done — `POST /api/payment/amn-scanner/direct-balance/check/:paymentId`; `POST /api/payment/request-network/intents` with `rail: "direct_balance"` | +| 3 | Watch-mode lifecycle, webhook decision, and stop cleanup | In progress — webhook delegates to `processDirectBalanceWebhook`; watch creation deferred to Phase 3 | | 4 | Frontend checkout controls and "I paid" action | Not started | | 5 | Smoke tests and admin observability | Not started | diff --git a/scripts/profile-mongo-api.mjs b/scripts/profile-mongo-api.mjs new file mode 100755 index 0000000..663fe19 --- /dev/null +++ b/scripts/profile-mongo-api.mjs @@ -0,0 +1,617 @@ +#!/usr/bin/env node + +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdir, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const docRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +const config = { + baseUrl: process.env.BASE_URL || "https://dev.manwe.qzz.io", + sshHost: process.env.SSH_HOST || "root@5.78.213.189", + sshKey: expandHome(process.env.SSH_KEY || "~/CascadeProjects/wzp"), + mongoContainer: process.env.MONGO_CONTAINER || "amanat-dev-mongodb", + mongoDb: process.env.MONGO_DB || "marketplace", + mongoUser: process.env.MONGO_USER || "admin", + mongoPassword: process.env.MONGO_PASSWORD || "password123", + mongoAuthDb: process.env.MONGO_AUTH_DB || "admin", + backendContainer: process.env.BACKEND_CONTAINER || "amanat-dev-backend", + resetBackendLimiter: ["1", "true", "yes"].includes( + String(process.env.RESET_BACKEND_LIMITER || "").toLowerCase(), + ), + npxBin: process.env.NPX_BIN || "npx", + buyerEmail: process.env.BUYER_EMAIL || "buyer@marketplace.com", + buyerPassword: process.env.BUYER_PASSWORD || "Moji6364", + templateShareableLink: process.env.TEMPLATE_SHAREABLE_LINK || "logo-design-template", + outputDir: + process.env.OUT_DIR || + path.join( + docRoot, + "09 - Audits", + "Mongo API Profiles", + new Date().toISOString().replace(/[:.]/g, "-"), + ), +}; + +const containers = (process.env.PROFILE_CONTAINERS || [ + "amanat-dev-nginx", + "amanat-dev-backend", + "amanat-dev-frontend", + "amanat-dev-postgres", + "amanat-dev-mongodb", + "amanat-dev-redis", + "amanat-dev-scanner", +].join(",")) + .split(",") + .map((value) => value.trim()) + .filter(Boolean); + +const endpointMatrix = [ + { name: "health", path: "/api/health", concurrency: 1, amount: 5 }, + { name: "categories", path: "/api/marketplace/categories", concurrency: 2, amount: 10 }, + { name: "categories_tree", path: "/api/marketplace/categories/tree", concurrency: 2, amount: 10 }, + { name: "sellers", path: "/api/marketplace/sellers", concurrency: 2, amount: 10 }, + { + name: "template_public", + path: `/api/marketplace/request-templates/public/${encodeURIComponent(config.templateShareableLink)}`, + concurrency: 2, + amount: 10, + }, + { + name: "payment_options_template", + path: null, + concurrency: 5, + amount: 50, + auth: true, + }, + { name: "addresses_me", path: "/api/addresses", concurrency: 2, amount: 10, auth: true }, + { + name: "purchase_requests_my", + path: "/api/marketplace/purchase-requests/my", + concurrency: 2, + amount: 10, + auth: true, + }, + { + name: "auth_login", + path: "/api/auth/login", + concurrency: 1, + amount: 5, + method: "POST", + headers: ["Content-Type: application/json"], + body: () => ({ email: config.buyerEmail, password: config.buyerPassword }), + }, +]; + +const sshBaseArgs = ["-i", config.sshKey, "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", config.sshHost]; + +if (!existsSync(config.sshKey)) { + throw new Error(`SSH key not found: ${config.sshKey}`); +} + +await mkdir(config.outputDir, { recursive: true }); + +let profilerEnabled = false; + +try { + if (config.resetBackendLimiter) { + console.error(`restarting ${config.backendContainer} to reset process-local rate limits`); + restartBackendContainer(); + await waitForHealth(); + } + + const authToken = await login(); + const template = getTemplateContext(); + + const matrix = endpointMatrix.map((test, index) => { + const pathValue = + test.name === "payment_options_template" + ? `/api/payment/request-network/options?currency=USD&amount=0.01&sellerId=${template.sellerId}&templateId=${template.templateId}` + : test.path; + + return { + ...test, + path: pathValue, + headers: [ + ...(test.headers || []), + // Counts are intentionally low to avoid profiling the in-memory global limiter. + `X-Forwarded-For: 203.0.113.${10 + index}`, + ], + }; + }); + + const results = []; + + for (const test of matrix) { + console.error(`profiling ${test.name} ${test.path}`); + enableProfiler(); + const beforeBlockIo = readDockerBlockIo(); + const bench = runAutocannon(test, authToken); + const afterBlockIo = readDockerBlockIo(); + const mongoProfile = collectMongoProfile(); + + results.push({ + name: test.name, + method: test.method || "GET", + path: test.path, + requestCount: bench.requests.total, + rps: bench.requests.average, + latency: { + averageMs: bench.latency.average, + p50Ms: bench.latency.p50, + p90Ms: bench.latency.p90, + p95Ms: bench.latency.p95 ?? bench.latency.p97_5, + p99Ms: bench.latency.p99, + maxMs: bench.latency.max, + }, + non2xx: bench.non2xx || 0, + statusCodeStats: bench.statusCodeStats || {}, + mongoProfile, + blockIoDelta: diffBlockIo(beforeBlockIo, afterBlockIo), + }); + } + + disableProfiler(); + + const report = { + generatedAt: new Date().toISOString(), + config: { + baseUrl: config.baseUrl, + sshHost: config.sshHost, + mongoContainer: config.mongoContainer, + mongoDb: config.mongoDb, + mongoAuthDb: config.mongoAuthDb, + backendContainer: config.backendContainer, + resetBackendLimiter: config.resetBackendLimiter, + containers, + templateShareableLink: config.templateShareableLink, + outputDir: config.outputDir, + }, + results, + }; + + const jsonPath = path.join(config.outputDir, "mongo-api-profile.json"); + const markdownPath = path.join(config.outputDir, "summary.md"); + await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`); + await writeFile(markdownPath, renderMarkdown(report)); + + console.log(`Wrote ${path.relative(docRoot, jsonPath)}`); + console.log(`Wrote ${path.relative(docRoot, markdownPath)}`); +} catch (error) { + if (profilerEnabled) { + try { + disableProfiler(); + } catch (disableError) { + console.error(`failed to disable profiler: ${disableError.message}`); + } + } + throw error; +} + +function expandHome(value) { + if (!value.startsWith("~/")) return value; + return path.join(os.homedir(), value.slice(2)); +} + +function shellQuote(value) { + return `'${String(value).replace(/'/g, "'\\''")}'`; +} + +function ssh(command, options = {}) { + return execFileSync("ssh", [...sshBaseArgs, command], { + encoding: "utf8", + maxBuffer: options.maxBuffer || 100 * 1024 * 1024, + }); +} + +function mongoEval(js) { + const command = [ + "docker exec", + shellQuote(config.mongoContainer), + "mongosh --quiet", + "-u", + shellQuote(config.mongoUser), + "-p", + shellQuote(config.mongoPassword), + "--authenticationDatabase", + shellQuote(config.mongoAuthDb), + shellQuote(config.mongoDb), + "--eval", + shellQuote(js), + ].join(" "); + return ssh(command); +} + +function restartBackendContainer() { + ssh(`docker restart ${shellQuote(config.backendContainer)}`, { maxBuffer: 1024 * 1024 }); +} + +async function waitForHealth() { + const deadline = Date.now() + 90_000; + let lastError = ""; + + while (Date.now() < deadline) { + try { + const response = await fetch(`${config.baseUrl}/api/health`); + const body = await response.json(); + if (response.ok && body?.status === "ok") return; + lastError = `status=${response.status} body=${JSON.stringify(body)}`; + } catch (error) { + lastError = error.message; + } + await new Promise((resolve) => setTimeout(resolve, 2_000)); + } + + throw new Error(`backend did not become healthy after restart: ${lastError}`); +} + +async function login() { + const response = await fetch(`${config.baseUrl}/api/auth/login`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ email: config.buyerEmail, password: config.buyerPassword }), + }); + const body = await response.json(); + if (!body?.success || !body?.data?.tokens?.accessToken) { + throw new Error(`login failed with status ${response.status}: ${JSON.stringify(body)}`); + } + return body.data.tokens.accessToken; +} + +function getTemplateContext() { + const raw = mongoEval(` + const doc = db.requesttemplates.findOne( + { shareableLink: ${JSON.stringify(config.templateShareableLink)} }, + { _id: 1, sellerId: 1 } + ); + print(JSON.stringify(doc)); + `).trim(); + + if (!raw || raw === "null") { + throw new Error(`template not found: ${config.templateShareableLink}`); + } + + const doc = JSON.parse(raw); + const templateId = doc._id?.$oid || doc._id; + const sellerId = doc.sellerId?.$oid || doc.sellerId; + if (!templateId || !sellerId) { + throw new Error(`template missing _id/sellerId: ${raw}`); + } + return { templateId, sellerId }; +} + +function enableProfiler() { + mongoEval(` + db.setProfilingLevel(0); + try { db.system.profile.drop(); } catch (error) {} + db.setProfilingLevel(2, { slowms: 0, sampleRate: 1 }); + print(JSON.stringify(db.getProfilingStatus())); + `); + profilerEnabled = true; +} + +function disableProfiler() { + mongoEval(`db.setProfilingLevel(0); print(JSON.stringify(db.getProfilingStatus()));`); + profilerEnabled = false; +} + +function collectMongoProfile() { + const output = mongoEval(` + db.setProfilingLevel(0); + const docs = db.system.profile.find({ ns: /^${escapeRegExp(config.mongoDb)}\\./ }).toArray(); + + function commandName(doc) { + const command = doc.command || {}; + for (const key of Object.keys(command)) { + if (!['lsid', '$db', '$clusterTime', 'readConcern', 'writeConcern', 'maxTimeMS'].includes(key)) { + return key; + } + } + return doc.op || 'unknown'; + } + + function collectionName(doc) { + const command = doc.command || {}; + return command.find || + command.aggregate || + command.count || + command.distinct || + command.update || + command.delete || + command.findAndModify || + command.insert || + doc.ns.replace(/^${escapeRegExp(config.mongoDb)}\\./, ''); + } + + function shapeValue(value) { + if (value === null) return 'null'; + if (value === undefined) return 'undefined'; + if (Array.isArray(value)) return '[' + value.map(shapeValue).join(',') + ']'; + if (typeof value === 'object') { + if (value._bsontype) return value._bsontype; + return '{' + Object.keys(value).sort().map((key) => key + ':' + shapeValue(value[key])).join(',') + '}'; + } + return typeof value; + } + + function queryShape(doc) { + const command = doc.command || {}; + const parts = []; + if (command.filter) parts.push('filter=' + shapeValue(command.filter)); + if (command.query) parts.push('query=' + shapeValue(command.query)); + if (command.pipeline) parts.push('pipeline=' + shapeValue(command.pipeline)); + if (command.sort) parts.push('sort=' + shapeValue(command.sort)); + if (command.projection) parts.push('projection=' + shapeValue(command.projection)); + if (command.update) parts.push('update=' + shapeValue(command.update)); + return parts.join(' '); + } + + const groups = new Map(); + for (const doc of docs) { + const key = [ + doc.ns, + doc.op, + commandName(doc), + collectionName(doc), + doc.planSummary || '', + doc.queryHash || '', + doc.planCacheKey || '', + queryShape(doc), + ].join(' | '); + + let group = groups.get(key); + if (!group) { + group = { + namespace: doc.ns, + operation: doc.op, + command: commandName(doc), + collection: collectionName(doc), + planSummary: doc.planSummary || '', + queryHash: doc.queryHash || '', + planCacheKey: doc.planCacheKey || '', + queryShape: queryShape(doc), + count: 0, + millisTotal: 0, + millisMax: 0, + millisValues: [], + docsExamined: 0, + keysExamined: 0, + nreturned: 0, + ninserted: 0, + nMatched: 0, + nModified: 0, + responseLength: 0, + numYield: 0, + }; + groups.set(key, group); + } + + const millis = Number(doc.millis || 0); + group.count += 1; + group.millisTotal += millis; + group.millisMax = Math.max(group.millisMax, millis); + group.millisValues.push(millis); + group.docsExamined += Number(doc.docsExamined || 0); + group.keysExamined += Number(doc.keysExamined || 0); + group.nreturned += Number(doc.nreturned || 0); + group.ninserted += Number(doc.ninserted || 0); + group.nMatched += Number(doc.nMatched || 0); + group.nModified += Number(doc.nModified || 0); + group.responseLength += Number(doc.responseLength || 0); + group.numYield += Number(doc.numYield || 0); + } + + function percentile(values, p) { + if (!values.length) return 0; + values.sort((a, b) => a - b); + return values[Math.min(values.length - 1, Math.floor((p / 100) * values.length))]; + } + + const groupsOut = Array.from(groups.values()) + .map((group) => ({ + namespace: group.namespace, + operation: group.operation, + command: group.command, + collection: group.collection, + planSummary: group.planSummary, + queryHash: group.queryHash, + planCacheKey: group.planCacheKey, + queryShape: group.queryShape, + count: group.count, + millisTotal: group.millisTotal, + millisAverage: group.count ? group.millisTotal / group.count : 0, + millisP50: percentile(group.millisValues, 50), + millisP95: percentile(group.millisValues, 95), + millisMax: group.millisMax, + docsExamined: group.docsExamined, + keysExamined: group.keysExamined, + nreturned: group.nreturned, + ninserted: group.ninserted, + nMatched: group.nMatched, + nModified: group.nModified, + responseLength: group.responseLength, + numYield: group.numYield, + })) + .sort((a, b) => b.millisTotal - a.millisTotal || b.count - a.count); + + print(JSON.stringify({ + totalOperations: docs.length, + totalMillis: groupsOut.reduce((sum, group) => sum + group.millisTotal, 0), + groups: groupsOut, + })); + `); + + return JSON.parse(output.trim().split(/\n/).pop()); +} + +function runAutocannon(test, authToken) { + const args = ["-y", "autocannon@8.0.0"]; + if (test.amount) args.push("-a", String(test.amount)); + if (test.duration) args.push("-d", String(test.duration)); + args.push("-c", String(test.concurrency || 1), "--json"); + for (const header of test.headers || []) args.push("-H", header); + if (test.auth) args.push("-H", `Authorization: Bearer ${authToken}`); + if (test.method) args.push("-m", test.method); + const body = typeof test.body === "function" ? test.body() : test.body; + if (body) args.push("-b", JSON.stringify(body)); + args.push(`${config.baseUrl}${test.path}`); + + const result = spawnSync(config.npxBin, args, { + encoding: "utf8", + maxBuffer: 100 * 1024 * 1024, + }); + + if (result.status !== 0) { + throw new Error(`autocannon failed for ${test.name}\n${result.stderr}\n${result.stdout}`); + } + + return JSON.parse(result.stdout); +} + +function readDockerBlockIo() { + const output = ssh( + `docker stats --no-stream --format '{{json .}}' ${containers.map(shellQuote).join(" ")}`, + ); + const rows = output.trim().split(/\n/).filter(Boolean).map((line) => JSON.parse(line)); + const map = {}; + for (const row of rows) { + const [readRaw, writeRaw] = String(row.BlockIO || "0B / 0B").split("/").map((item) => item.trim()); + map[row.Name] = { + readBytes: parseBytes(readRaw), + writeBytes: parseBytes(writeRaw), + raw: row.BlockIO, + }; + } + return map; +} + +function diffBlockIo(before, after) { + const diff = {}; + for (const [name, value] of Object.entries(after)) { + diff[name] = { + readBytes: Math.max(0, value.readBytes - (before[name]?.readBytes || 0)), + writeBytes: Math.max(0, value.writeBytes - (before[name]?.writeBytes || 0)), + }; + } + return diff; +} + +function parseBytes(value) { + const match = String(value || "").trim().match(/^([0-9.]+)\s*([KMGT]?i?B|B)$/i); + if (!match) return 0; + const number = Number(match[1]); + const unit = match[2].toLowerCase(); + const multiplier = { + b: 1, + kb: 1_000, + mb: 1_000_000, + gb: 1_000_000_000, + tb: 1_000_000_000_000, + kib: 1024, + mib: 1024 ** 2, + gib: 1024 ** 3, + tib: 1024 ** 4, + }[unit] || 1; + return number * multiplier; +} + +function formatBytes(value) { + if (!value) return "0 B"; + const units = ["B", "KB", "MB", "GB"]; + let next = value; + let index = 0; + while (next >= 1000 && index < units.length - 1) { + next /= 1000; + index += 1; + } + return `${next.toFixed(next >= 10 || index === 0 ? 0 : 1)} ${units[index]}`; +} + +function renderMarkdown(report) { + const lines = []; + lines.push("# Mongo API Query Profile"); + lines.push(""); + lines.push(`Generated: ${report.generatedAt}`); + lines.push(`Base URL: \`${report.config.baseUrl}\``); + lines.push(`Mongo: \`${report.config.mongoContainer}/${report.config.mongoDb}\``); + lines.push(""); + lines.push("This is a query-shape profile, not a max-throughput test. Request counts are intentionally small so the backend rate limiter does not dominate the profile."); + lines.push(""); + lines.push("## Endpoint Summary"); + lines.push(""); + lines.push("| Endpoint | Requests | Avg | P95 | P99 | Non-2xx | Mongo ops | Top Mongo query |"); + lines.push("|---|---:|---:|---:|---:|---:|---:|---|"); + for (const result of report.results) { + const top = result.mongoProfile.groups[0]; + lines.push( + [ + `\`${result.method} ${result.path}\``, + result.requestCount, + `${result.latency.averageMs}ms`, + `${result.latency.p95Ms}ms`, + `${result.latency.p99Ms}ms`, + result.non2xx, + result.mongoProfile.totalOperations, + top ? `\`${top.collection}\` ${top.command} (${top.count}x, ${top.planSummary || "no plan"})` : "-", + ].join(" | ").replace(/^/, "| ").replace(/$/, " |"), + ); + } + lines.push(""); + lines.push("## Query Groups"); + for (const result of report.results) { + lines.push(""); + lines.push(`### ${result.name}`); + lines.push(""); + lines.push(`Path: \`${result.method} ${result.path}\``); + lines.push(`Status codes: \`${JSON.stringify(result.statusCodeStats)}\``); + lines.push(""); + if (!result.mongoProfile.groups.length) { + lines.push("No Mongo operations captured in this endpoint window."); + continue; + } + lines.push("| Collection | Command | Count | Total ms | Avg ms | P95 ms | Plan | Docs | Keys | Returned | Shape |"); + lines.push("|---|---|---:|---:|---:|---:|---|---:|---:|---:|---|"); + for (const group of result.mongoProfile.groups.slice(0, 12)) { + lines.push( + [ + `\`${group.collection}\``, + `\`${group.command}\``, + group.count, + group.millisTotal, + round(group.millisAverage), + group.millisP95, + `\`${group.planSummary || "-"}\``, + group.docsExamined, + group.keysExamined, + group.nreturned, + `\`${truncate(group.queryShape || "-", 140)}\``, + ].join(" | ").replace(/^/, "| ").replace(/$/, " |"), + ); + } + } + lines.push(""); + lines.push("## Block I/O Deltas"); + for (const result of report.results) { + const active = Object.entries(result.blockIoDelta) + .filter(([, value]) => value.readBytes || value.writeBytes) + .map(([name, value]) => `${name}: read ${formatBytes(value.readBytes)}, write ${formatBytes(value.writeBytes)}`); + lines.push(`- ${result.name}: ${active.length ? active.join("; ") : "no container block I/O delta"}`); + } + lines.push(""); + return `${lines.join("\n")}\n`; +} + +function truncate(value, max) { + return value.length > max ? `${value.slice(0, max - 3)}...` : value; +} + +function round(value) { + return Math.round(value * 1000) / 1000; +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +}