105 lines
3.0 KiB
TypeScript
105 lines
3.0 KiB
TypeScript
/**
|
||
* 历史汇率数据播种脚本
|
||
* 运行方式: npx tsx scripts/seed-historical-rates.ts
|
||
*
|
||
* 功能：解析 scripts/rates.csv，将历史汇率数据分批 upsert 到 exchange_rates_history 表。
|
||
* 特性：
|
||
* - 自动剔除 BOM 头 (\uFEFF)
|
||
* - 按 500 条/批次批量写入
|
||
* - 联合唯一约束 (fromCurrency, toCurrency, fetchTime) 确保幂等性
|
||
*/
|
||
|
||
import * as fs from 'fs';
|
||
import * as path from 'path';
|
||
import { db } from '@/db';
|
||
import { exchangeRatesHistory } from '@/db/schema';
|
||
import { sql } from 'drizzle-orm';
|
||
|
||
const BATCH_SIZE = 500;
|
||
|
||
function parseCsv(filePath: string): typeof exchangeRatesHistory.$inferInsert[] {
|
||
const raw = fs.readFileSync(filePath, 'utf-8');
|
||
const content = raw.replace(/^\uFEFF/, '');
|
||
const lines = content.split('\n').filter(line => line.trim().length > 0);
|
||
|
||
// 剔除表头
|
||
const header = lines[0];
|
||
if (!header) {
|
||
throw new Error('CSV file is empty');
|
||
}
|
||
|
||
const records: typeof exchangeRatesHistory.$inferInsert[] = [];
|
||
for (let i = 1; i < lines.length; i++) {
|
||
const line = lines[i].replace(/^\uFEFF/, '').trim();
|
||
if (!line) continue;
|
||
|
||
const parts = line.split(',');
|
||
if (parts.length < 4) {
|
||
console.warn(`Skipping malformed line ${i + 1}: ${line}`);
|
||
continue;
|
||
}
|
||
|
||
const [fromCurrency, toCurrency, rateStr, fetchTimeStr] = parts;
|
||
|
||
const rate = parseFloat(rateStr.trim());
|
||
if (isNaN(rate)) {
|
||
console.warn(`Skipping line ${i + 1} with invalid rate: ${rateStr}`);
|
||
continue;
|
||
}
|
||
|
||
const fetchTime = new Date(fetchTimeStr.trim());
|
||
if (isNaN(fetchTime.getTime())) {
|
||
console.warn(`Skipping line ${i + 1} with invalid date: ${fetchTimeStr}`);
|
||
continue;
|
||
}
|
||
|
||
records.push({
|
||
fromCurrency: fromCurrency.trim(),
|
||
toCurrency: toCurrency.trim(),
|
||
rate: rate.toString(),
|
||
fetchTime,
|
||
});
|
||
}
|
||
|
||
return records;
|
||
}
|
||
|
||
async function seed() {
|
||
const csvPath = path.join(__dirname, 'rates.csv');
|
||
|
||
if (!fs.existsSync(csvPath)) {
|
||
console.error(`CSV file not found: ${csvPath}`);
|
||
process.exit(1);
|
||
}
|
||
|
||
console.log(`Reading CSV from: ${csvPath}`);
|
||
const records = parseCsv(csvPath);
|
||
console.log(`Parsed ${records.length} valid records.`);
|
||
|
||
const batches: typeof exchangeRatesHistory.$inferInsert[][] = [];
|
||
for (let i = 0; i < records.length; i += BATCH_SIZE) {
|
||
batches.push(records.slice(i, i + BATCH_SIZE));
|
||
}
|
||
|
||
let totalUpserted = 0;
|
||
for (let i = 0; i < batches.length; i++) {
|
||
const batch = batches[i];
|
||
await db
|
||
.insert(exchangeRatesHistory)
|
||
.values(batch)
|
||
.onConflictDoUpdate({
|
||
target: [exchangeRatesHistory.fromCurrency, exchangeRatesHistory.toCurrency, exchangeRatesHistory.fetchTime],
|
||
set: { rate: sql`EXCLUDED.rate` },
|
||
});
|
||
totalUpserted += batch.length;
|
||
console.log(`Batch ${i + 1}/${batches.length}: ${batch.length} records processed (${totalUpserted}/${records.length} total).`);
|
||
}
|
||
|
||
console.log(`Seed complete. Total records processed: ${totalUpserted}`);
|
||
}
|
||
|
||
seed().catch((err) => {
|
||
console.error('Seed failed:', err);
|
||
process.exit(1);
|
||
});
|