first commit
This commit is contained in:
Bivekich
@ -1,363 +1,46 @@
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-nocheck
|
||||
import { Pool } from 'pg'
|
||||
|
||||
class PartsDatabase {
|
||||
private pool: Pool
|
||||
|
||||
// Temporary no-op implementation to avoid requiring 'pg' when parts index is disabled.
|
||||
class NoopPartsDatabase {
|
||||
constructor() {
|
||||
const connectionString = process.env.DATABASE_URL
|
||||
if (!connectionString) {
|
||||
throw new Error('DATABASE_URL environment variable is not set')
|
||||
}
|
||||
|
||||
this.pool = new Pool({
|
||||
connectionString,
|
||||
max: 10, // Reduce concurrent connections to avoid overwhelming the DB
|
||||
idleTimeoutMillis: 60000, // 1 minute
|
||||
connectionTimeoutMillis: 10000, // 10 seconds
|
||||
keepAlive: true,
|
||||
keepAliveInitialDelayMillis: 10000,
|
||||
})
|
||||
|
||||
console.log('🔌 Parts Database connection initialized (using main DATABASE_URL)')
|
||||
console.warn('Parts DB disabled: using no-op implementation')
|
||||
}
|
||||
|
||||
// Create table for a specific category
|
||||
async createCategoryTable(categoryId: string, categoryName: string, categoryType: 'partsindex' | 'partsapi'): Promise<void> {
|
||||
const tableName = this.getCategoryTableName(categoryId, categoryType)
|
||||
|
||||
try {
|
||||
const query = `
|
||||
CREATE TABLE IF NOT EXISTS "${tableName}" (
|
||||
id SERIAL PRIMARY KEY,
|
||||
external_id VARCHAR(255) UNIQUE NOT NULL,
|
||||
name VARCHAR(500) NOT NULL,
|
||||
brand VARCHAR(255),
|
||||
article VARCHAR(255),
|
||||
description TEXT,
|
||||
image_url VARCHAR(500),
|
||||
price DECIMAL(10,2),
|
||||
category_id VARCHAR(255) NOT NULL,
|
||||
category_name VARCHAR(500) NOT NULL,
|
||||
category_type VARCHAR(20) NOT NULL,
|
||||
group_id VARCHAR(255),
|
||||
group_name VARCHAR(500),
|
||||
raw_data JSONB,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Create indexes for better performance
|
||||
CREATE INDEX IF NOT EXISTS "idx_${tableName}_external_id" ON "${tableName}" (external_id);
|
||||
CREATE INDEX IF NOT EXISTS "idx_${tableName}_category_id" ON "${tableName}" (category_id);
|
||||
CREATE INDEX IF NOT EXISTS "idx_${tableName}_brand" ON "${tableName}" (brand);
|
||||
CREATE INDEX IF NOT EXISTS "idx_${tableName}_article" ON "${tableName}" (article);
|
||||
CREATE INDEX IF NOT EXISTS "idx_${tableName}_created_at" ON "${tableName}" (created_at);
|
||||
|
||||
-- Create trigger to update updated_at timestamp
|
||||
CREATE OR REPLACE FUNCTION update_${tableName}_timestamp()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
DROP TRIGGER IF EXISTS trigger_update_${tableName}_timestamp ON "${tableName}";
|
||||
CREATE TRIGGER trigger_update_${tableName}_timestamp
|
||||
BEFORE UPDATE ON "${tableName}"
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_${tableName}_timestamp();
|
||||
`
|
||||
|
||||
await this.pool.query(query)
|
||||
console.log(`✅ Created table ${tableName} for category: ${categoryName}`)
|
||||
} catch (error) {
|
||||
console.error(`❌ Error creating table ${tableName}:`, error)
|
||||
throw error
|
||||
}
|
||||
async createCategoryTable() {
|
||||
return
|
||||
}
|
||||
|
||||
// Insert or update products in category table
|
||||
async insertProducts(
|
||||
categoryId: string,
|
||||
categoryName: string,
|
||||
categoryType: 'partsindex' | 'partsapi',
|
||||
products: any[],
|
||||
groupId?: string,
|
||||
groupName?: string
|
||||
): Promise<number> {
|
||||
const tableName = this.getCategoryTableName(categoryId, categoryType)
|
||||
|
||||
console.log(`🔄 Starting to insert ${products.length} products into ${tableName}`)
|
||||
|
||||
// First ensure table exists
|
||||
await this.createCategoryTable(categoryId, categoryName, categoryType)
|
||||
|
||||
let insertedCount = 0
|
||||
let errorCount = 0
|
||||
|
||||
// Process in smaller batches to reduce connection pressure
|
||||
const batchSize = 50
|
||||
const batches: any[][] = []
|
||||
for (let i = 0; i < products.length; i += batchSize) {
|
||||
batches.push(products.slice(i, i + batchSize))
|
||||
}
|
||||
|
||||
console.log(`📦 Processing ${products.length} products in ${batches.length} batches of ${batchSize}`)
|
||||
|
||||
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
||||
const batch = batches[batchIndex]
|
||||
console.log(`🔄 Processing batch ${batchIndex + 1}/${batches.length} (${batch.length} products)`)
|
||||
|
||||
for (let i = 0; i < batch.length; i++) {
|
||||
const globalIndex = batchIndex * batchSize + i
|
||||
const product = batch[i]
|
||||
|
||||
// Retry logic with exponential backoff
|
||||
let retryAttempts = 0
|
||||
const maxRetries = 3
|
||||
let success = false
|
||||
|
||||
while (retryAttempts < maxRetries && !success) {
|
||||
try {
|
||||
const values = this.prepareProductData(product, categoryId, categoryName, categoryType, groupId, groupName)
|
||||
|
||||
const query = `
|
||||
INSERT INTO "${tableName}" (
|
||||
external_id, name, brand, article, description, image_url, price,
|
||||
category_id, category_name, category_type, group_id, group_name, raw_data
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
ON CONFLICT (external_id)
|
||||
DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
brand = EXCLUDED.brand,
|
||||
article = EXCLUDED.article,
|
||||
description = EXCLUDED.description,
|
||||
image_url = EXCLUDED.image_url,
|
||||
price = EXCLUDED.price,
|
||||
group_id = EXCLUDED.group_id,
|
||||
group_name = EXCLUDED.group_name,
|
||||
raw_data = EXCLUDED.raw_data,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`
|
||||
|
||||
await this.pool.query(query, values)
|
||||
insertedCount++
|
||||
success = true
|
||||
|
||||
// Log progress every 100 insertions
|
||||
if (insertedCount % 100 === 0) {
|
||||
console.log(`📊 Progress: ${insertedCount}/${products.length} products inserted into ${tableName}`)
|
||||
}
|
||||
} catch (error: any) {
|
||||
retryAttempts++
|
||||
|
||||
// Check if it's a network/connection error that might be retryable
|
||||
if (error.code === 'ENOTFOUND' || error.code === 'ECONNRESET' || error.code === 'ETIMEDOUT') {
|
||||
if (retryAttempts < maxRetries) {
|
||||
const delayMs = Math.pow(2, retryAttempts) * 1000 // Exponential backoff: 2s, 4s, 8s
|
||||
console.log(`🔄 Retry ${retryAttempts}/${maxRetries} for product ${globalIndex + 1} after ${delayMs}ms delay`)
|
||||
await new Promise(resolve => setTimeout(resolve, delayMs))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// If max retries exceeded or non-retryable error
|
||||
errorCount++
|
||||
console.error(`❌ Error inserting product ${globalIndex + 1}/${products.length} into ${tableName} (after ${retryAttempts} retries):`, error)
|
||||
console.error(`❌ Product data:`, product)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Small delay between batches to allow DB to recover
|
||||
if (batchIndex < batches.length - 1) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100))
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`✅ Insertion complete for ${tableName}:`)
|
||||
console.log(` - Successfully inserted/updated: ${insertedCount} products`)
|
||||
console.log(` - Errors: ${errorCount} products`)
|
||||
console.log(` - Total processed: ${products.length} products`)
|
||||
|
||||
return insertedCount
|
||||
async insertProducts(_categoryId, _categoryName, _categoryType, products) {
|
||||
console.warn(`Parts DB noop: insertProducts called for ${products?.length || 0} items`)
|
||||
return 0
|
||||
}
|
||||
|
||||
// Get products from category table
|
||||
async getProducts(
|
||||
categoryId: string,
|
||||
categoryType: 'partsindex' | 'partsapi',
|
||||
options: {
|
||||
limit?: number
|
||||
offset?: number
|
||||
search?: string
|
||||
} = {}
|
||||
): Promise<{ products: any[], total: number }> {
|
||||
const tableName = this.getCategoryTableName(categoryId, categoryType)
|
||||
const { limit = 50, offset = 0, search } = options
|
||||
|
||||
try {
|
||||
let whereClause = ''
|
||||
let searchParams: any[] = []
|
||||
|
||||
if (search) {
|
||||
whereClause = 'WHERE (name ILIKE $1 OR brand ILIKE $1 OR article ILIKE $1 OR description ILIKE $1)'
|
||||
searchParams = [`%${search}%`]
|
||||
}
|
||||
|
||||
// Count total
|
||||
const countQuery = `SELECT COUNT(*) FROM "${tableName}" ${whereClause}`
|
||||
const countResult = await this.pool.query(countQuery, searchParams)
|
||||
const total = parseInt(countResult.rows[0].count)
|
||||
|
||||
// Get products
|
||||
const dataQuery = `
|
||||
SELECT * FROM "${tableName}"
|
||||
${whereClause}
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $${searchParams.length + 1} OFFSET $${searchParams.length + 2}
|
||||
`
|
||||
const dataResult = await this.pool.query(dataQuery, [...searchParams, limit, offset])
|
||||
|
||||
return {
|
||||
products: dataResult.rows,
|
||||
total
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`❌ Error getting products from ${tableName}:`, error)
|
||||
return { products: [], total: 0 }
|
||||
}
|
||||
async getProducts(_categoryId, _categoryType, _options = {}) {
|
||||
return { products: [], total: 0 }
|
||||
}
|
||||
|
||||
// Get all category tables
|
||||
async getCategoryTables(): Promise<{ tableName: string, categoryId: string, categoryType: string, recordCount: number }[]> {
|
||||
try {
|
||||
const query = `
|
||||
SELECT
|
||||
tablename,
|
||||
schemaname
|
||||
FROM pg_tables
|
||||
WHERE schemaname = 'public'
|
||||
AND (tablename LIKE 'category_partsindex_%' OR tablename LIKE 'category_partsapi_%')
|
||||
ORDER BY tablename
|
||||
`
|
||||
|
||||
const result = await this.pool.query(query)
|
||||
const tables: { tableName: string, categoryId: string, categoryType: string, recordCount: number }[] = []
|
||||
|
||||
for (const row of result.rows) {
|
||||
const tableName = row.tablename
|
||||
|
||||
// Parse category info from table name
|
||||
const [, categoryType, categoryId] = tableName.split('_')
|
||||
|
||||
// Get record count
|
||||
const countQuery = `SELECT COUNT(*) FROM "${tableName}"`
|
||||
const countResult = await this.pool.query(countQuery)
|
||||
const recordCount = parseInt(countResult.rows[0].count)
|
||||
|
||||
tables.push({
|
||||
tableName,
|
||||
categoryId,
|
||||
categoryType,
|
||||
recordCount
|
||||
})
|
||||
}
|
||||
|
||||
return tables
|
||||
} catch (error) {
|
||||
console.error('❌ Error getting category tables:', error)
|
||||
return []
|
||||
}
|
||||
async getCategoryTables() {
|
||||
return []
|
||||
}
|
||||
|
||||
// Delete category table
|
||||
async deleteCategoryTable(categoryId: string, categoryType: 'partsindex' | 'partsapi'): Promise<void> {
|
||||
const tableName = this.getCategoryTableName(categoryId, categoryType)
|
||||
|
||||
try {
|
||||
await this.pool.query(`DROP TABLE IF EXISTS "${tableName}" CASCADE`)
|
||||
console.log(`✅ Deleted table ${tableName}`)
|
||||
} catch (error) {
|
||||
console.error(`❌ Error deleting table ${tableName}:`, error)
|
||||
throw error
|
||||
}
|
||||
async deleteCategoryTable() {
|
||||
return
|
||||
}
|
||||
|
||||
// Helper method to generate table name
|
||||
private getCategoryTableName(categoryId: string, categoryType: 'partsindex' | 'partsapi'): string {
|
||||
// Sanitize category ID for use in table name
|
||||
const sanitizedId = categoryId.replace(/[^a-zA-Z0-9_]/g, '_').toLowerCase()
|
||||
async testConnection() {
|
||||
return true
|
||||
}
|
||||
|
||||
async close() {
|
||||
return
|
||||
}
|
||||
|
||||
// Keep signature compatibility with previous helper
|
||||
getCategoryTableName(categoryId, categoryType) {
|
||||
const sanitizedId = String(categoryId || '').replace(/[^a-zA-Z0-9_]/g, '_').toLowerCase()
|
||||
return `category_${categoryType}_${sanitizedId}`
|
||||
}
|
||||
|
||||
// Helper method to prepare product data
|
||||
private prepareProductData(
|
||||
product: any,
|
||||
categoryId: string,
|
||||
categoryName: string,
|
||||
categoryType: 'partsindex' | 'partsapi',
|
||||
groupId?: string,
|
||||
groupName?: string
|
||||
): any[] {
|
||||
if (categoryType === 'partsindex') {
|
||||
return [
|
||||
product.id || product.external_id || `${Date.now()}_${Math.random()}`,
|
||||
product.name || '',
|
||||
product.brand || '',
|
||||
product.article || '',
|
||||
product.description || '',
|
||||
product.image || '',
|
||||
product.price ? parseFloat(product.price) : null,
|
||||
categoryId,
|
||||
categoryName,
|
||||
categoryType,
|
||||
groupId || null,
|
||||
groupName || null,
|
||||
JSON.stringify(product)
|
||||
]
|
||||
} else {
|
||||
// PartsAPI
|
||||
return [
|
||||
product.artId || `${Date.now()}_${Math.random()}`,
|
||||
product.artArticleNr || '',
|
||||
product.artSupBrand || '',
|
||||
product.artArticleNr || '',
|
||||
product.productGroup || '',
|
||||
'', // no image for PartsAPI
|
||||
null, // no price for PartsAPI
|
||||
categoryId,
|
||||
categoryName,
|
||||
categoryType,
|
||||
groupId || null,
|
||||
groupName || null,
|
||||
JSON.stringify(product)
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
// Test database connection
|
||||
async testConnection(): Promise<boolean> {
|
||||
try {
|
||||
await this.pool.query('SELECT 1')
|
||||
console.log('✅ Parts database connection test successful')
|
||||
return true
|
||||
} catch (error) {
|
||||
console.error('❌ Parts database connection test failed:', error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Close connection pool
|
||||
async close(): Promise<void> {
|
||||
await this.pool.end()
|
||||
console.log('🔌 Parts database connection closed')
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
export const partsDb = new PartsDatabase()
|
||||
export const partsDb = new NoopPartsDatabase()
|
||||
|
||||
Reference in New Issue
Block a user