Compare commits
3 Commits
7ded5b7837
...
2570144112
| Author | SHA1 | Date | |
|---|---|---|---|
| 2570144112 | |||
| 2fb5629a89 | |||
| c243ba4f35 |
23
Memory.md
23
Memory.md
@ -1,5 +1,28 @@
|
||||
# Omniledger 架构与开发记忆 (Memory)
|
||||
|
||||
## 修复腾讯行情接口 URL 拼接逻辑,剔除导致数据残缺的 s_ (简易版) 前缀,确保所有市场强制获取包含时间戳的全量报文 (Task 67)
|
||||
- 在 `app/api/cron/fetch-prices/route.ts` 的 `fetchStockPrice()` 函数与 `src/actions/market.ts` 的 `getTencentSymbol()` 函数中,将美股资产的前缀映射从 `'s_us'` 强制重构为 `'us'`。
|
||||
- **根因分析:** 腾讯财经 gtimg 接口使用 `s_us` 前缀时返回的是"简易版"报文(仅 ~10 个字段),缺失 Index 30 的日期时间字段;使用 `us` 前缀时返回"全量版"报文(60+ 个字段),包含完整的交易时间戳 `2026-05-01 16:00:06`。
|
||||
- 修改前的错误拼接:`https://sqt.gtimg.cn/q=s_usGOOG` → 10 字段,无日期 → 触发 `[Date Parse Fatal Error]`。
|
||||
- 修改后的正确拼接:`https://sqt.gtimg.cn/q=usGOOG` → 60+ 字段,Index 30 含日期 → `parseMarketDate()` 成功解析 `2026-05-01`。
|
||||
- 其他市场前缀保持不变:港股 `hk` (如 `hk01810`)、A股沪市 `sh` (如 `sh600009`)、A股深市 `sz` (如 `sz002594`)。
|
||||
- **验证:** `curl` 对比 `s_usGOOG` (10字段) vs `usGOOG` (60+字段),Cron 接口成功同步 21 条记录,0 失败,控制台无任何 `[Date Parse Fatal Error]` 报错,美股日期正确解析为 `2026-05-01`。
|
||||
|
||||
|
||||
## 部署防弹级 parseMarketDate 解析引擎,增加 payload 脏前缀清洗逻辑,彻底解决美股日期解析崩溃触发 fallback 的幽灵 Bug (Task 66)
|
||||
- 在 `app/api/cron/fetch-prices/route.ts` 中一字不差地替换 `parseMarketDate(rawString: string)` 为工业级防污染版本。
|
||||
- **核心修复:** 新增 `payload.includes('="')` 脏前缀清洗逻辑,从腾讯 gtimg 原始响应中提取 `="` 之后的纯净数据段,剔除结尾 `";`,从根本上消除 `v_usGOOG="...` 前缀导致的数组偏移与数据污染风险。
|
||||
- 日期匹配顺序调整为:美股 (`-`) → 港股 (`/`) → A股 (`/^\d{8}/`),与 payload 清洗逻辑配合,确保跨市场日期提取零误判。
|
||||
- 错误日志升级为 `console.error("[Date Parse Fatal Error]")`,致命错误时完整打印原始字符串便于调试;兜底逻辑保持不变。
|
||||
- **验证:** `curl` 触发 Cron 接口成功,21 条记录全部 upsert,0 失败,控制台无任何 `[Date Parse Fatal Error]` 红字报错,美股日期字段正确解析为 `2026-05-01`。
|
||||
|
||||
## 建立 asset_id 与 date 的联合唯一索引,重构三套跨市场日期解析正则,实现基于 onConflictDoUpdate 的价格历史幂等覆盖逻辑 (Task 65)
|
||||
- 在 `src/db/schema.ts` 的 `assetPricesHistory` 表中新增 `updateTime` 字段 (`timestamp('update_time').defaultNow()`),并将联合索引从 `uniqueIndex` 升级为 `unique()` 约束 (`unique().on(table.assetId, table.date)`),确保 `(assetId, date)` 在物理层严格唯一。
|
||||
- 在 `app/api/cron/fetch-prices/route.ts` 中一字不差地注入 `parseMarketDate(rawString: string)` 傻瓜式日期解析引擎:从腾讯财经 gtimg 原始响应的 Index 30 提取日期,支持 A股 (14位数字→YYYY-MM-DD)、港股 (斜杠分隔→YYYY-MM-DD)、美股 (横杠分隔→YYYY-MM-DD) 三种格式,含 try/catch 极端兜底。
|
||||
- 废弃原有的 `SELECT → UPDATE/INSERT` 双步查询逻辑,全面替换为 Drizzle 的 `onConflictDoUpdate` UPSERT 语法:基于联合唯一约束,冲突时只更新 `price` 与 `updateTime`,实现完全幂等的价格覆盖。
|
||||
- 移除不再使用的 `skippedCount` 计数器与 `eq` 导入,简化响应结构。
|
||||
- 执行 `drizzle-kit push` 完成物理迁移,`curl` 测试确认:美股存入 `2026-05-01`,港A股存入 `2026-04-30`;重复调用不产生新行,`update_time` 正确刷新。
|
||||
|
||||
## 升级时光机历史快照生成逻辑,引入就近汇率匹配策略 (Closest Rate Matching),消除因使用单一日结汇率导致的历史资产估值失真 (Task 64)
|
||||
- 在 `src/actions/snapshots.ts` 的 `reconstructPortfolioHistory()` 中,废弃从静态 `exchangeRates` 表获取当前汇率的旧逻辑,全面接入 `exchangeRatesHistory` 历史汇率时间序列表。
|
||||
- **架构调整**:在 `dayLoop` 循环之前,一次性加载全部 `exchangeRatesHistory` 记录到内存,按 `(fromCurrency, toCurrency)` 键分组构建 `ratesCache`(`Map<string, RateRecord[]>`),每条记录已按 `fetchTime` 升序排列。
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import { db } from '@/db';
|
||||
import { assets, assetPricesHistory } from '@/db/schema';
|
||||
import { eq, inArray } from 'drizzle-orm';
|
||||
import { inArray } from 'drizzle-orm';
|
||||
import { ProxyAgent, setGlobalDispatcher } from 'undici';
|
||||
|
||||
export const dynamic = 'force-dynamic';
|
||||
@ -15,19 +15,44 @@ function formatDateStr(date: Date): string {
|
||||
}
|
||||
|
||||
function parseMarketDate(rawString: string): string {
|
||||
const parts = rawString.split('~');
|
||||
const rawDate = parts[30];
|
||||
if (!rawDate) return new Date().toISOString().split('T')[0];
|
||||
|
||||
if (/^\d{14}$/.test(rawDate)) {
|
||||
return `${rawDate.slice(0, 4)}-${rawDate.slice(4, 6)}-${rawDate.slice(6, 8)}`;
|
||||
try {
|
||||
// 1. 强制清洗脏前缀 (消除 v_usGOOG=" 导致的数组偏移与污染风险)
|
||||
let payload = rawString;
|
||||
if (payload.includes('="')) {
|
||||
// 提取 =" 之后的内容,并剔除结尾可能存在的 ";
|
||||
payload = payload.split('="')[1].replace(/";/g, '');
|
||||
}
|
||||
|
||||
const parts = payload.split('~');
|
||||
// 腾讯接口核心数据长度校验
|
||||
if (parts.length < 31) throw new Error("Payload length insufficient");
|
||||
|
||||
const rawDate = parts[30];
|
||||
if (!rawDate) throw new Error("Index 30 is undefined or empty");
|
||||
|
||||
// 2. 美股匹配 (2026-05-01 16:00:06)
|
||||
if (rawDate.includes('-')) {
|
||||
return rawDate.split(' ')[0];
|
||||
}
|
||||
|
||||
// 3. 港股匹配 (2026/04/30 16:08:24)
|
||||
if (rawDate.includes('/')) {
|
||||
return rawDate.split(' ')[0].replace(/\//g, '-');
|
||||
}
|
||||
|
||||
return rawDate.split(' ')[0];
|
||||
// 4. A股匹配 (20260430161416)
|
||||
if (/^\d{8}/.test(rawDate)) {
|
||||
return `${rawDate.slice(0, 4)}-${rawDate.slice(4, 6)}-${rawDate.slice(6, 8)}`;
|
||||
}
|
||||
|
||||
throw new Error(`Unrecognized date format: ${rawDate}`);
|
||||
} catch (e) {
|
||||
// 致命错误暴露:必须打印出导致崩溃的原始字符串
|
||||
console.error("[Date Parse Fatal Error] String:", rawString, "Error:", e);
|
||||
// 最终兜底
|
||||
const today = new Date();
|
||||
return `${today.getFullYear()}-${String(today.getMonth() + 1).padStart(2, '0')}-${String(today.getDate()).padStart(2, '0')}`;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchStockPrice(asset: { symbol: string; exchange: string | null }): Promise<{ price: string | null; rawResponse: string | null }> {
|
||||
@ -46,7 +71,7 @@ async function fetchStockPrice(asset: { symbol: string; exchange: string | null
|
||||
break;
|
||||
case 'US':
|
||||
default:
|
||||
tCode = 's_us' + cleanSymbol;
|
||||
tCode = 'us' + cleanSymbol;
|
||||
break;
|
||||
}
|
||||
|
||||
@ -116,13 +141,11 @@ export async function GET(req: Request) {
|
||||
message: 'No active assets to sync',
|
||||
date: dateStr,
|
||||
synced: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
});
|
||||
}
|
||||
|
||||
let syncedCount = 0;
|
||||
let skippedCount = 0;
|
||||
let failedCount = 0;
|
||||
const results: Array<{ symbol: string; price: string | null; status: string }> = [];
|
||||
|
||||
@ -146,38 +169,25 @@ export async function GET(req: Request) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const entryDate = asset.type === 'STOCK' && rawResponse ? parseMarketDate(rawResponse) : dateStr;
|
||||
const parsedDate = asset.type === 'STOCK' && rawResponse ? parseMarketDate(rawResponse) : dateStr;
|
||||
|
||||
const existing = await db
|
||||
.select()
|
||||
.from(assetPricesHistory)
|
||||
.where(
|
||||
eq(assetPricesHistory.assetId, asset.id)
|
||||
)
|
||||
.then((rows) =>
|
||||
rows.filter((row) => row.date === entryDate)
|
||||
);
|
||||
|
||||
if (existing.length > 0) {
|
||||
await db
|
||||
.update(assetPricesHistory)
|
||||
.set({ price })
|
||||
.where(
|
||||
eq(assetPricesHistory.id, existing[0].id)
|
||||
);
|
||||
skippedCount++;
|
||||
results.push({ symbol: asset.symbol, price, status: 'updated' });
|
||||
} else {
|
||||
await db
|
||||
.insert(assetPricesHistory)
|
||||
await db.insert(assetPricesHistory)
|
||||
.values({
|
||||
assetId: asset.id,
|
||||
price,
|
||||
date: entryDate,
|
||||
});
|
||||
syncedCount++;
|
||||
results.push({ symbol: asset.symbol, price, status: 'inserted' });
|
||||
date: parsedDate,
|
||||
price: price.toString(),
|
||||
updateTime: new Date()
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [assetPricesHistory.assetId, assetPricesHistory.date],
|
||||
set: {
|
||||
price: price.toString(),
|
||||
updateTime: new Date()
|
||||
}
|
||||
});
|
||||
|
||||
syncedCount++;
|
||||
results.push({ symbol: asset.symbol, price, status: 'upserted' });
|
||||
} catch (error) {
|
||||
failedCount++;
|
||||
results.push({ symbol: asset.symbol, price: null, status: 'error' });
|
||||
@ -189,7 +199,6 @@ export async function GET(req: Request) {
|
||||
success: true,
|
||||
date: dateStr,
|
||||
synced: syncedCount,
|
||||
skipped: skippedCount,
|
||||
failed: failedCount,
|
||||
details: results,
|
||||
});
|
||||
|
||||
@ -15,7 +15,7 @@ function getTencentSymbol(asset: { symbol: string; exchange: string | null }): s
|
||||
case 'SZSE': return 'sz' + cleanSymbol;
|
||||
case 'HKEX': return 'hk' + cleanSymbol;
|
||||
case 'US':
|
||||
default: return 's_us' + cleanSymbol;
|
||||
default: return 'us' + cleanSymbol;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { pgTable, uuid, varchar, timestamp, pgEnum, numeric, uniqueIndex, date } from "drizzle-orm/pg-core";
|
||||
import { pgTable, uuid, varchar, timestamp, pgEnum, numeric, uniqueIndex, unique, date } from "drizzle-orm/pg-core";
|
||||
|
||||
export const users = pgTable("users", {
|
||||
id: uuid("id").primaryKey().$defaultFn(() => crypto.randomUUID()),
|
||||
@ -86,11 +86,13 @@ export const assetPricesHistory = pgTable("asset_prices_history", {
|
||||
.references(() => assets.id),
|
||||
price: numeric("price", { precision: 36, scale: 18 }).notNull(),
|
||||
date: date("date", { mode: "string" }).notNull(),
|
||||
updateTime: timestamp("update_time", { withTimezone: true, mode: "date" })
|
||||
.defaultNow(),
|
||||
createdAt: timestamp("created_at", { withTimezone: true, mode: "date" })
|
||||
.defaultNow()
|
||||
.notNull(),
|
||||
}, (table) => [
|
||||
uniqueIndex("asset_price_date_idx").on(table.assetId, table.date),
|
||||
unique().on(table.assetId, table.date),
|
||||
]);
|
||||
|
||||
export const exchangeRatesHistory = pgTable("exchange_rates_history", {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user