/** * 历史汇率数据播种脚本 * 运行方式: npx tsx scripts/seed-historical-rates.ts * * 功能:解析 scripts/rates.csv,将历史汇率数据分批 upsert 到 exchange_rates_history 表。 * 特性: * - 自动剔除 BOM 头 (\uFEFF) * - 按 500 条/批次批量写入 * - 联合唯一约束 (fromCurrency, toCurrency, fetchTime) 确保幂等性 */ import * as fs from 'fs'; import * as path from 'path'; import { db } from '@/db'; import { exchangeRatesHistory } from '@/db/schema'; import { sql } from 'drizzle-orm'; const BATCH_SIZE = 500; function parseCsv(filePath: string): typeof exchangeRatesHistory.$inferInsert[] { const raw = fs.readFileSync(filePath, 'utf-8'); const content = raw.replace(/^\uFEFF/, ''); const lines = content.split('\n').filter(line => line.trim().length > 0); // 剔除表头 const header = lines[0]; if (!header) { throw new Error('CSV file is empty'); } const records: typeof exchangeRatesHistory.$inferInsert[] = []; for (let i = 1; i < lines.length; i++) { const line = lines[i].replace(/^\uFEFF/, '').trim(); if (!line) continue; const parts = line.split(','); if (parts.length < 4) { console.warn(`Skipping malformed line ${i + 1}: ${line}`); continue; } const [fromCurrency, toCurrency, rateStr, fetchTimeStr] = parts; const rate = parseFloat(rateStr.trim()); if (isNaN(rate)) { console.warn(`Skipping line ${i + 1} with invalid rate: ${rateStr}`); continue; } const fetchTime = new Date(fetchTimeStr.trim()); if (isNaN(fetchTime.getTime())) { console.warn(`Skipping line ${i + 1} with invalid date: ${fetchTimeStr}`); continue; } records.push({ fromCurrency: fromCurrency.trim(), toCurrency: toCurrency.trim(), rate: rate.toString(), fetchTime, }); } return records; } async function seed() { const csvPath = path.join(__dirname, 'rates.csv'); if (!fs.existsSync(csvPath)) { console.error(`CSV file not found: ${csvPath}`); process.exit(1); } console.log(`Reading CSV from: ${csvPath}`); const records = parseCsv(csvPath); console.log(`Parsed ${records.length} valid records.`); const batches: typeof exchangeRatesHistory.$inferInsert[][] = []; for (let i = 0; i < records.length; i += BATCH_SIZE) { batches.push(records.slice(i, i + BATCH_SIZE)); } let totalUpserted = 0; for (let i = 0; i < batches.length; i++) { const batch = batches[i]; await db .insert(exchangeRatesHistory) .values(batch) .onConflictDoUpdate({ target: [exchangeRatesHistory.fromCurrency, exchangeRatesHistory.toCurrency, exchangeRatesHistory.fetchTime], set: { rate: sql`EXCLUDED.rate` }, }); totalUpserted += batch.length; console.log(`Batch ${i + 1}/${batches.length}: ${batch.length} records processed (${totalUpserted}/${records.length} total).`); } console.log(`Seed complete. Total records processed: ${totalUpserted}`); } seed().catch((err) => { console.error('Seed failed:', err); process.exit(1); });