const mysql = require('mysql2/promise');
require('dotenv').config();
const fs = require('fs');
const path = require('path');

const NODE_ENV = process.env.NODE_ENV || 'development';

const getSSLConfig = () => {
    const useSSL = String(process.env.DB_SSL || '').toLowerCase() === 'true';
    const caPath = process.env.DB_SSL_CA_PATH;
    if (!useSSL) return undefined;

    try {
        if (caPath) {
            const resolved = path.resolve(caPath);
            if (fs.existsSync(resolved)) {
                console.log('🔐 (createDb) Loading DB CA certificate from:', resolved);
                return {
                    ca: fs.readFileSync(resolved),
                    rejectUnauthorized: false
                };
            } else {
                console.warn('⚠️  (createDb) CA file not found at:', resolved, '- proceeding with rejectUnauthorized:false');
            }
        } else {
            console.warn('⚠️  (createDb) DB_SSL_CA_PATH not set - proceeding with rejectUnauthorized:false');
        }
    } catch (e) {
        console.warn('⚠️  (createDb) Failed to load CA file:', e.message, '- proceeding with rejectUnauthorized:false');
    }
    return { rejectUnauthorized: false };
};

let dbConfig;
if (NODE_ENV === 'development') {
    dbConfig = {
        host: process.env.DEV_DB_HOST || 'localhost',
        port: Number(process.env.DEV_DB_PORT) || 3306,
        user: process.env.DEV_DB_USER || 'root',
        password: process.env.DEV_DB_PASSWORD || '', // XAMPP default: no password
        database: process.env.DEV_DB_NAME || 'profitplanet_centralserver',
        ssl: undefined
    };
} else {
    dbConfig = {
        host: process.env.DB_HOST,
        port: Number(process.env.DB_PORT) || 3306,
        user: process.env.DB_USER,
        password: process.env.DB_PASSWORD,
        database: process.env.DB_NAME,
        ssl: getSSLConfig()
    };
}

const allowCreateDb = String(process.env.DB_ALLOW_CREATE_DB || 'false').toLowerCase() === 'true';

const SYSTEM_POOLS = [
    { pool_name: 'ABO 60', description: 'System pool for ABO 60 capsule distribution', price: 0.01, pool_type: 'coffee', price_per_capsule_gross: 0.01 },
    { pool_name: 'ABO 120', description: 'System pool for ABO 120 capsule distribution', price: 0.01, pool_type: 'coffee', price_per_capsule_gross: 0.01 },
    { pool_name: 'Business', description: 'System pool for Business capsule distribution', price: 0.02, pool_type: 'other', price_per_capsule_gross: 0.02 },
    { pool_name: 'Gigantea', description: 'System pool for Gigantea capsule distribution', price: 0.02, pool_type: 'other', price_per_capsule_gross: 0.02 },
    { pool_name: 'Core', description: 'Every member receives 1 cent per capsule sold — the amount multiplies with each member, not divided.', price: 0.01, pool_type: 'other', price_per_capsule_gross: 0.01 },
];

// --- Performance Helpers ---
async function ensureIndex(conn, table, indexName, indexDDL) {
    const [rows] = await conn.query(`SHOW INDEX FROM \`${table}\` WHERE Key_name = ?`, [indexName]);
    if (!rows.length) {
        await conn.query(`CREATE INDEX \`${indexName}\` ON \`${table}\` (${indexDDL})`);
        console.log(`🆕 Created index ${indexName} ON ${table}`);
    } else {
        console.log(`ℹ️  Index ${indexName} already exists on ${table}`);
    }
}

// --- Schema pre-check helpers (Approach A) ---
async function columnExists(conn, table, column) {
    const [rows] = await conn.query(
        `SELECT 1
         FROM INFORMATION_SCHEMA.COLUMNS
         WHERE TABLE_SCHEMA = DATABASE()
           AND TABLE_NAME = ?
           AND COLUMN_NAME = ?
         LIMIT 1`,
        [table, column]
    );
    return rows.length > 0;
}

async function addColumnIfMissing(conn, table, column, ddlFragment /* includes type + nullability + AFTER/etc */) {
    if (await columnExists(conn, table, column)) return false;
    await conn.query(`ALTER TABLE \`${table}\` ADD COLUMN \`${column}\` ${ddlFragment}`);
    console.log(`🆕 Added column ${table}.${column}`);
    return true;
}

async function constraintExists(conn, table, constraintName) {
    const [rows] = await conn.query(
        `SELECT 1
         FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS
         WHERE TABLE_SCHEMA = DATABASE()
           AND TABLE_NAME = ?
           AND CONSTRAINT_NAME = ?
         LIMIT 1`,
        [table, constraintName]
    );
    return rows.length > 0;
}

async function addForeignKeyIfMissing(conn, table, constraintName, ddl /* full ALTER TABLE ... ADD CONSTRAINT ... */) {
    if (await constraintExists(conn, table, constraintName)) return false;
    await conn.query(ddl);
    console.log(`🆕 Added constraint ${constraintName} on ${table}`);
    return true;
}

async function getPrimaryKeyColumns(conn, table) {
    const [rows] = await conn.query(
        `SELECT k.COLUMN_NAME
         FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS t
         JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE k
           ON t.CONSTRAINT_NAME = k.CONSTRAINT_NAME
          AND t.TABLE_SCHEMA = k.TABLE_SCHEMA
          AND t.TABLE_NAME = k.TABLE_NAME
         WHERE t.TABLE_SCHEMA = DATABASE()
           AND t.TABLE_NAME = ?
           AND t.CONSTRAINT_TYPE = 'PRIMARY KEY'
         ORDER BY k.ORDINAL_POSITION`,
        [table]
    );
    return rows.map(r => r.COLUMN_NAME);
}

const createDatabase = async () => {
    console.log('🚀 Starting MySQL database initialization...');
    console.log('📍 Database host:', process.env.DB_HOST);
    console.log('📍 Database name:', process.env.DB_NAME);
    
    let connection;
    try {
        if (allowCreateDb) {
            // Connect without specifying a database to create it if it doesn't exist
            connection = await mysql.createConnection({
                host: dbConfig.host,
                port: dbConfig.port,
                user: dbConfig.user,
                password: dbConfig.password,
                ssl: dbConfig.ssl
            });

            await connection.query(`CREATE DATABASE IF NOT EXISTS \`${dbConfig.database}\`;`);
            console.log(`✅ Database "${dbConfig.database}" created/verified`);
            await connection.end();
        } else {
            console.log('ℹ️  Skipping database creation (DB_ALLOW_CREATE_DB=false)');
        }

        // Reconnect with the database specified
        connection = await mysql.createConnection(dbConfig);
        console.log('✅ Connected to MySQL database');

        // --- Core Tables ---

        // 1. users table: Central user authentication and common attributes
        await connection.query(`
            CREATE TABLE IF NOT EXISTS users (
                id INT AUTO_INCREMENT PRIMARY KEY,
                email VARCHAR(255) UNIQUE NOT NULL,
                password VARCHAR(255) NOT NULL,
                user_type ENUM('personal', 'company', 'guest') NOT NULL,
                role ENUM('user', 'admin', 'super_admin', 'guest') DEFAULT 'user',
                iban VARCHAR(34),
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                last_login_at TIMESTAMP NULL,
                INDEX idx_email (email),
                INDEX idx_user_type (user_type),
                INDEX idx_role (role)
            );
        `);
        console.log('✅ Users table created/verified');

        // Migrate existing role ENUM to include 'guest'
        try {
            await connection.query(`
                ALTER TABLE users
                MODIFY COLUMN role ENUM('user', 'admin', 'super_admin', 'guest') DEFAULT 'user'
            `);
            console.log('✅ Updated users.role column to include guest');
        } catch (err) {
            console.warn('⚠️  Could not modify users.role column:', err.message);
        }

        // 2. personal_profiles table: Details specific to personal users
        await connection.query(`
            CREATE TABLE IF NOT EXISTS personal_profiles (
                id INT AUTO_INCREMENT PRIMARY KEY,
                user_id INT NOT NULL,
                first_name VARCHAR(255) NOT NULL,
                last_name VARCHAR(255) NOT NULL,
                phone VARCHAR(255) NULL,
                date_of_birth DATE,
                nationality VARCHAR(255),
                address TEXT,
                zip_code VARCHAR(20),
                city VARCHAR(100),
                country VARCHAR(100),
                phone_secondary VARCHAR(255),
                emergency_contact_name VARCHAR(255),
                emergency_contact_phone VARCHAR(255),
                account_holder_name VARCHAR(255),
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                UNIQUE KEY unique_user_profile (user_id)
            );
        `);
        console.log('✅ Personal profiles table created/verified');

        // 3. company_profiles table: Details specific to company users
        await connection.query(`
            CREATE TABLE IF NOT EXISTS company_profiles (
                id INT AUTO_INCREMENT PRIMARY KEY,
                user_id INT NOT NULL,
                company_name VARCHAR(255) NOT NULL,
                registration_number VARCHAR(255) UNIQUE,
                phone VARCHAR(255) NULL,
                address TEXT,
                zip_code VARCHAR(20),
                city VARCHAR(100),
                country VARCHAR(100),
                branch VARCHAR(255),
                number_of_employees INT,
                business_type VARCHAR(255),
                contact_person_name VARCHAR(255),
                contact_person_phone VARCHAR(255),
                account_holder_name VARCHAR(255),
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                UNIQUE KEY unique_user_profile (user_id),
                UNIQUE KEY unique_registration_number (registration_number)
            );
        `);
        console.log('✅ Company profiles table created/verified');

        // Ensure registration_number allows NULL if table already existed
        try {
          await connection.query(`ALTER TABLE company_profiles MODIFY COLUMN registration_number VARCHAR(255) UNIQUE NULL`);
          console.log('🔧 Ensured registration_number allows NULL');
        } catch (e) {
          console.log('ℹ️  registration_number column already allows NULL or ALTER not required');
        }

        // Ensure phone columns are nullable if tables already existed
        try {
          await connection.query(`ALTER TABLE personal_profiles MODIFY COLUMN phone VARCHAR(255) NULL`);
          await connection.query(`ALTER TABLE company_profiles MODIFY COLUMN phone VARCHAR(255) NULL`);
          console.log('🔧 Ensured phone columns are nullable');
        } catch (e) {
          console.log('ℹ️  Phone columns already nullable or ALTER not required');
        }

        // ATU number for company profiles
        await addColumnIfMissing(connection, 'company_profiles', 'atu_number', `VARCHAR(50) NULL AFTER registration_number`);

        // 4. user_status table: Comprehensive tracking of user verification and completion steps
        await connection.query(`
            CREATE TABLE IF NOT EXISTS user_status (
                user_id INT PRIMARY KEY,
                status ENUM('inactive', 'pending', 'active', 'suspended', 'archived') DEFAULT 'pending',
                previous_status ENUM('inactive', 'pending', 'active', 'suspended', 'archived') NULL,
                email_verified BOOLEAN DEFAULT FALSE,
                email_verified_at TIMESTAMP NULL,
                profile_completed BOOLEAN DEFAULT FALSE,
                profile_completed_at TIMESTAMP NULL,
                documents_uploaded BOOLEAN DEFAULT FALSE,
                documents_uploaded_at TIMESTAMP NULL,
                contract_signed BOOLEAN DEFAULT FALSE,
                contract_signed_at TIMESTAMP NULL,
                is_admin_verified BOOLEAN DEFAULT FALSE,
                admin_verified_at TIMESTAMP NULL,
                registration_completed BOOLEAN DEFAULT FALSE,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
            );
        `);
        console.log('✅ User status table created/verified');
        
        // Modify existing ENUM columns to add 'archived' status (for existing databases)
        try {
            await connection.query(`
                ALTER TABLE user_status 
                MODIFY COLUMN status ENUM('inactive', 'pending', 'active', 'suspended', 'archived') DEFAULT 'pending'
            `);
            console.log('✅ Updated status column to include archived');
        } catch (err) {
            console.warn('⚠️  Could not modify status column:', err.message);
        }
        
        try {
            await connection.query(`
                ALTER TABLE user_status 
                MODIFY COLUMN previous_status ENUM('inactive', 'pending', 'active', 'suspended', 'archived') NULL
            `);
            console.log('✅ Updated previous_status column to include archived');
        } catch (err) {
            console.warn('⚠️  Could not modify previous_status column:', err.message);
        }

        // --- Authentication & Verification Tables ---

        // 5. refresh_tokens table: For refresh token authentication
        await connection.query(`
            CREATE TABLE IF NOT EXISTS refresh_tokens (
                id INT AUTO_INCREMENT PRIMARY KEY,
                user_id INT NOT NULL,
                token VARCHAR(255) UNIQUE NOT NULL,
                expires_at DATETIME NOT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                revoked_at TIMESTAMP NULL,
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                INDEX idx_user_id (user_id),
                INDEX idx_expires_at (expires_at)
            );
        `);
        console.log('✅ Refresh tokens table created/verified');

        // Ensure refresh token column length supports longer tokens (JWT/base64/etc.)
        try {
            await connection.query(`
              ALTER TABLE refresh_tokens
              MODIFY COLUMN token VARCHAR(512) NOT NULL;
            `);
            console.log('🔧 Ensured refresh_tokens.token is VARCHAR(512)');
        } catch (e) {
            console.log('ℹ️  refresh_tokens.token length ALTER skipped:', e.message);
        }

        // 6. email_verifications table: For email verification codes
        await connection.query(`
            CREATE TABLE IF NOT EXISTS email_verifications (
                id INT AUTO_INCREMENT PRIMARY KEY,
                user_id INT NOT NULL,
                verification_code VARCHAR(6) NOT NULL,
                expires_at DATETIME NOT NULL,
                verified_at TIMESTAMP NULL,
                attempts INT DEFAULT 0,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                INDEX idx_user_code (user_id, verification_code),
                INDEX idx_expires_at (expires_at)
            );
        `);
        console.log('✅ Email verifications table created/verified');

        // 7. password_resets table: For password reset tokens
        await connection.query(`
            CREATE TABLE IF NOT EXISTS password_resets (
                id INT AUTO_INCREMENT PRIMARY KEY,
                user_id INT NOT NULL,
                token VARCHAR(255) UNIQUE NOT NULL,
                expires_at DATETIME NOT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                used_at TIMESTAMP NULL,
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                INDEX idx_user_token (user_id, token),
                INDEX idx_expires_at (expires_at)
            );
        `);
        console.log('✅ Password resets table created/verified');

        // --- Document & Logging Tables ---

        // 8. user_documents table: Stores object storage IDs
        await connection.query(`
            CREATE TABLE IF NOT EXISTS user_documents (
              id INT AUTO_INCREMENT PRIMARY KEY,
              user_id INT NOT NULL,
              document_type ENUM('personal_id', 'company_id', 'signature', 'contract', 'other') NOT NULL,
                            contract_type ENUM('contract','gdpr','abo') NOT NULL DEFAULT 'contract',
              object_storage_id VARCHAR(255) UNIQUE NULL,
              signatureBase64 LONGTEXT NULL,
              original_filename VARCHAR(255),
              file_size INT,
              mime_type VARCHAR(100),
              upload_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
              verified_by_admin BOOLEAN DEFAULT FALSE,
              admin_verified_at TIMESTAMP NULL,
              FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
              INDEX idx_user_document_type (user_id, document_type),
              INDEX idx_user_contract_type (user_id, contract_type),
              INDEX idx_object_storage_id (object_storage_id)
            );
        `);

        // Backfill/migrate for older schemas (NO IF NOT EXISTS; pre-check instead)
        await addColumnIfMissing(
            connection,
            'user_documents',
            'contract_type',
            `ENUM('contract','gdpr','abo') NOT NULL DEFAULT 'contract' AFTER document_type`
        );
        await ensureIndex(connection, 'user_documents', 'idx_user_contract_type', '`user_id`, `contract_type`');

        // Ensure enum includes 'abo' on existing schemas
        try {
            await connection.query(`
              ALTER TABLE user_documents
              MODIFY COLUMN contract_type ENUM('contract','gdpr','abo') NOT NULL DEFAULT 'contract';
            `);
            console.log("🔧 Ensured user_documents.contract_type includes 'abo'");
        } catch (e) {
            console.log("ℹ️  user_documents.contract_type ENUM ALTER skipped:", e.message);
        }

        await connection.query(`
          ALTER TABLE user_documents
          MODIFY COLUMN object_storage_id VARCHAR(255) NULL;
        `);

        await addColumnIfMissing(
            connection,
            'user_documents',
            'signatureBase64',
            `LONGTEXT NULL AFTER object_storage_id`
        );

        console.log('✅ User documents table created/verified');

        // 8c. document_templates table: Stores template metadata and object storage keys
        await connection.query(`
            CREATE TABLE IF NOT EXISTS document_templates (
                id INT AUTO_INCREMENT PRIMARY KEY,
                name VARCHAR(255) NOT NULL,
                type VARCHAR(100) NOT NULL,
                contract_type ENUM('contract','gdpr','abo') NULL DEFAULT NULL,
                storageKey VARCHAR(255) NOT NULL,
                description TEXT,
                lang VARCHAR(10) NOT NULL,
                user_type ENUM('personal','company','both') DEFAULT 'both',
                version INT DEFAULT 1,
                state ENUM('active','inactive') DEFAULT 'inactive',
                createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CHECK (
                  (type <> 'contract' AND contract_type IS NULL)
                  OR (type = 'contract' AND contract_type IN ('contract','gdpr','abo'))
                )
            );
        `);
        console.log('✅ Document templates table created/verified');

        // Ensure enum includes 'abo' on existing schemas
        try {
            await connection.query(`
              ALTER TABLE document_templates
              MODIFY COLUMN contract_type ENUM('contract','gdpr','abo') NULL DEFAULT NULL;
            `);
            console.log("🔧 Ensured document_templates.contract_type includes 'abo'");
        } catch (e) {
            console.log("ℹ️  document_templates.contract_type ENUM ALTER skipped:", e.message);
        }

        // Ensure CHECK constraint includes 'abo' (best-effort; some MySQL/MariaDB versions ignore/limit CHECK)
        try {
            const [checks] = await connection.query(`
              SELECT tc.CONSTRAINT_NAME AS name, cc.CHECK_CLAUSE AS clause
              FROM information_schema.TABLE_CONSTRAINTS tc
              JOIN information_schema.CHECK_CONSTRAINTS cc
                ON tc.CONSTRAINT_SCHEMA = cc.CONSTRAINT_SCHEMA
               AND tc.CONSTRAINT_NAME = cc.CONSTRAINT_NAME
              WHERE tc.CONSTRAINT_SCHEMA = DATABASE()
                AND tc.TABLE_NAME = 'document_templates'
                AND tc.CONSTRAINT_TYPE = 'CHECK'
            `);

            const checkRows = Array.isArray(checks) ? checks : [];
            const toDrop = checkRows.filter(r => {
                const clause = (r && r.clause) ? String(r.clause) : '';
                const looksLikeOldContractTypeCheck = clause.includes('contract_type')
                  && clause.includes("'contract'")
                  && clause.includes("'gdpr'")
                  && !clause.includes("'abo'");
                return looksLikeOldContractTypeCheck;
            });

            if (toDrop.length) {
                for (const r of toDrop) {
                    const name = r && r.name ? String(r.name) : '';
                    if (!name) continue;
                    try {
                        // MariaDB: DROP CONSTRAINT; MySQL: DROP CHECK
                        try {
                            await connection.query(`ALTER TABLE document_templates DROP CONSTRAINT \`${name}\`;`);
                        } catch (e1) {
                            await connection.query(`ALTER TABLE document_templates DROP CHECK \`${name}\`;`);
                        }
                    } catch (e2) {
                        console.log('ℹ️  document_templates CHECK drop skipped:', e2.message);
                    }
                }
            }

            // Add (or re-add) the desired check constraint
            const hasDesired = checkRows.some(r => {
                const clause = (r && r.clause) ? String(r.clause) : '';
                return clause.includes("contract_type IN ('contract','gdpr','abo')");
            });
            if (!hasDesired) {
                try {
                    await connection.query(`
                      ALTER TABLE document_templates
                      ADD CONSTRAINT chk_document_templates_contract_type
                      CHECK (
                        (type <> 'contract' AND contract_type IS NULL)
                        OR (type = 'contract' AND contract_type IN ('contract','gdpr','abo'))
                      );
                    `);
                    console.log("🔧 Ensured document_templates CHECK allows 'abo'");
                } catch (e) {
                    console.log('ℹ️  document_templates CHECK add skipped:', e.message);
                }
            }
        } catch (e) {
            console.log('ℹ️  document_templates CHECK migration skipped:', e.message);
        }

        await connection.query(`
            CREATE TABLE IF NOT EXISTS no_user_abo_mails (
                id INT AUTO_INCREMENT PRIMARY KEY,
                email VARCHAR(255) NOT NULL,
                abonement_id INT NOT NULL,
                status ENUM('pending','linked') DEFAULT 'pending',
                source VARCHAR(50) DEFAULT 'subscribe',
                created_by_user_id INT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                UNIQUE KEY uq_no_user_abo_email_abonement (email, abonement_id),
                INDEX idx_no_user_abo_email_status (email, status)
            );
        `);
        console.log('✅ no_user_abo_mails table created/verified');

        // 8b. user_id_documents table: Stores ID-specific metadata (front/back object storage IDs)
        await connection.query(`
            CREATE TABLE IF NOT EXISTS user_id_documents (
                id INT AUTO_INCREMENT PRIMARY KEY,
                user_id INT NOT NULL,
                document_type ENUM('personal_id', 'company_id') NOT NULL,
                front_object_storage_id VARCHAR(255) NOT NULL,
                back_object_storage_id VARCHAR(255) NULL,
                original_filename_front VARCHAR(255),
                original_filename_back VARCHAR(255),
                id_type VARCHAR(50),
                id_number VARCHAR(100),
                expiry_date DATE,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE
            );
        `);
        console.log('✅ User ID documents table created/verified');

        // 9. user_action_logs table: For detailed user activity logging
        await connection.query(`
            CREATE TABLE IF NOT EXISTS user_action_logs (
                id INT AUTO_INCREMENT PRIMARY KEY,
                user_id INT NULL,
                affected_user_id INT NULL,
                action VARCHAR(100) NOT NULL,
                performed_by_user_id INT NULL,
                details JSON,
                ip_address VARCHAR(45),
                user_agent TEXT,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,               -- NEW FK
                FOREIGN KEY (affected_user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                FOREIGN KEY (performed_by_user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                INDEX idx_user_id (user_id),                                                                 -- NEW index
                INDEX idx_affected_user (affected_user_id),
                INDEX idx_performed_by (performed_by_user_id),
                INDEX idx_action (action),
                INDEX idx_created_at (created_at)
            );
        `);
        console.log('✅ User action logs table created/verified');

        // --- Add missing user_id column for existing databases + backfill ---
        // (convert from "try/catch duplicate" to pre-check)
        if (await addColumnIfMissing(connection, 'user_action_logs', 'user_id', `INT NULL`)) {
            await addForeignKeyIfMissing(
                connection,
                'user_action_logs',
                'fk_user_action_logs_user',
                `
                ALTER TABLE user_action_logs
                ADD CONSTRAINT fk_user_action_logs_user
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE
                `
            );
            await ensureIndex(connection, 'user_action_logs', 'idx_user_id', '`user_id`');
        }

        // Backfill: prefer performed_by_user_id, else affected_user_id
        try {
            await connection.query(`
              UPDATE user_action_logs
              SET user_id = performed_by_user_id
              WHERE user_id IS NULL AND performed_by_user_id IS NOT NULL
            `);
            await connection.query(`
              UPDATE user_action_logs
              SET user_id = affected_user_id
              WHERE user_id IS NULL AND affected_user_id IS NOT NULL
            `);
            console.log('🧹 Backfilled user_action_logs.user_id from performed_by_user_id/affected_user_id');
        } catch (e) {
            console.warn('⚠️  Could not backfill user_action_logs.user_id:', e.message);
        }

        // --- Email & Registration Flow Tables ---

        // 10. email_attempts table: For tracking email sending attempts
        await connection.query(`
            CREATE TABLE IF NOT EXISTS email_attempts (
                id INT AUTO_INCREMENT PRIMARY KEY,
                user_id INT NOT NULL,
                attempt_type ENUM('verification', 'password_reset', 'registration_completion', 'notification', 'other') NOT NULL,
                email_address VARCHAR(255) NOT NULL,
                success BOOLEAN DEFAULT FALSE,
                error_message TEXT,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                INDEX idx_user_attempts (user_id, attempt_type),
                INDEX idx_created_at (created_at)
            );
        `);
        console.log('✅ Email attempts table created/verified');

        // --- Referral Tables ---

        // 12. referral_tokens table: Manages referral codes
        await connection.query(`
            CREATE TABLE IF NOT EXISTS referral_tokens (
                id INT AUTO_INCREMENT PRIMARY KEY,
                token VARCHAR(64) UNIQUE NOT NULL,
                created_by_user_id INT NOT NULL,
                expires_at DATETIME NOT NULL,
                max_uses INT DEFAULT -1,
                uses_remaining INT DEFAULT -1,
                status ENUM('active', 'inactive', 'expired', 'exhausted') DEFAULT 'active',
                deactivation_reason VARCHAR(50),
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                FOREIGN KEY (created_by_user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                INDEX idx_token (token),
                INDEX idx_status_expires (status, expires_at),
                INDEX idx_created_by (created_by_user_id)
            );
        `);
        
          // News table for News Manager
          await connection.query(`
            CREATE TABLE IF NOT EXISTS news (
              id INT AUTO_INCREMENT PRIMARY KEY,
              title VARCHAR(255) NOT NULL,
              summary TEXT NULL,
              content MEDIUMTEXT NULL,
              slug VARCHAR(255) UNIQUE NOT NULL,
              category VARCHAR(128) NULL,
              object_storage_id VARCHAR(255) NULL,
              original_filename VARCHAR(255) NULL,
              is_active TINYINT(1) NOT NULL DEFAULT 1,
              published_at DATETIME NULL,
              created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
              updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
            );
          `);
          console.log('✅ News table created/verified');
        console.log('✅ Referral tokens table created/verified');

        // Generated label columns (virtual) to display 'unlimited' instead of -1 in UI tools
        try {
          await connection.query(`
            ALTER TABLE referral_tokens 
              ADD COLUMN max_uses_label VARCHAR(20) 
                GENERATED ALWAYS AS (CASE WHEN max_uses = -1 THEN 'unlimited' ELSE CAST(max_uses AS CHAR) END) VIRTUAL
          `);
          console.log('🆕 Added virtual column referral_tokens.max_uses_label');
        } catch (e) {
          console.log('ℹ️  max_uses_label already exists or cannot add:', e.message);
        }
        try {
          await connection.query(`
            ALTER TABLE referral_tokens 
              ADD COLUMN uses_remaining_label VARCHAR(20) 
                GENERATED ALWAYS AS (CASE WHEN uses_remaining = -1 THEN 'unlimited' ELSE CAST(uses_remaining AS CHAR) END) VIRTUAL
          `);
          console.log('🆕 Added virtual column referral_tokens.uses_remaining_label');
        } catch (e) {
          console.log('ℹ️  uses_remaining_label already exists or cannot add:', e.message);
        }

        // Normalized view now sources the generated label columns (still handy for joins)
        try {
          await connection.query(`
            CREATE OR REPLACE VIEW referral_tokens_normalized AS
            SELECT
              rt.*,
              rt.max_uses_label AS max_uses_display,
              rt.uses_remaining_label AS uses_remaining_display
            FROM referral_tokens rt;
          `);
          console.log('🆕 referral_tokens_normalized view created/updated');
        } catch (e) {
          console.warn('⚠️  Could not create referral_tokens_normalized view:', e.message);
        }

        // 13. referral_token_usage table: Tracks each use of a referral token
        await connection.query(`
            CREATE TABLE IF NOT EXISTS referral_token_usage (
                id INT AUTO_INCREMENT PRIMARY KEY,
                referral_token_id INT NOT NULL,
                used_by_user_id INT NOT NULL,
                used_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                FOREIGN KEY (referral_token_id) REFERENCES referral_tokens(id) ON DELETE CASCADE ON UPDATE CASCADE,
                FOREIGN KEY (used_by_user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                UNIQUE KEY unique_token_user_usage (referral_token_id, used_by_user_id),
                INDEX idx_token_usage (referral_token_id),
                INDEX idx_user_usage (used_by_user_id)
            );
        `);
        console.log('✅ Referral token usage table created/verified');

        // --- Affiliates table (for Affiliate Manager) ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS affiliates (
              id INT AUTO_INCREMENT PRIMARY KEY,
              name VARCHAR(255) NOT NULL,
              description TEXT NULL,
              url VARCHAR(1024) NOT NULL,
              object_storage_id VARCHAR(255) NULL,
              original_filename VARCHAR(255) NULL,
              category VARCHAR(128) NOT NULL,
              is_active TINYINT(1) NOT NULL DEFAULT 1,
              commission_rate DECIMAL(5,2) NULL,
              created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
              updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
              INDEX idx_affiliates_created (created_at),
              INDEX idx_affiliates_active (is_active),
              INDEX idx_affiliates_category (category)
            );
        `);
        console.log('✅ Affiliates table created/verified');

        // --- Authorization Tables ---

        // 14. permissions table: Defines granular permissions
        await connection.query(`
            CREATE TABLE IF NOT EXISTS permissions (
                id INT AUTO_INCREMENT PRIMARY KEY,
                name VARCHAR(100) UNIQUE NOT NULL,
                description VARCHAR(255),
                is_active BOOLEAN DEFAULT TRUE,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- Added
                created_by INT NULL, -- Added
                FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE
            );
        `);
        console.log('✅ Permissions table created/verified');

        // 15. user_permissions join table: Assigns specific permissions to users
        await connection.query(`
            CREATE TABLE IF NOT EXISTS user_permissions (
                id INT AUTO_INCREMENT PRIMARY KEY,
                user_id INT NOT NULL,
                permission_id INT NOT NULL,
                granted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                granted_by INT NULL, -- Added column
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                FOREIGN KEY (permission_id) REFERENCES permissions(id) ON DELETE CASCADE ON UPDATE CASCADE,
                FOREIGN KEY (granted_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE, -- FK constraint
                UNIQUE KEY unique_user_permission (user_id, permission_id)
            );
        `);
        console.log('✅ User permissions table created/verified');

        // --- User Settings Table ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS user_settings (
                user_id INT PRIMARY KEY,
                theme ENUM('light', 'dark') DEFAULT 'light',
                font_size ENUM('normal', 'large') DEFAULT 'normal',
                high_contrast_mode BOOLEAN DEFAULT FALSE,
                two_factor_auth_enabled BOOLEAN DEFAULT FALSE,
                account_visibility ENUM('public', 'private') DEFAULT 'public',
                show_email BOOLEAN DEFAULT TRUE,
                show_phone BOOLEAN DEFAULT TRUE,
                data_export_requested BOOLEAN DEFAULT FALSE,
                last_data_export_at TIMESTAMP NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE
            );
        `);
        console.log('✅ User settings table created/verified');

        // --- Company Settings (single-row, global invoice / company info) ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS company_settings (
                id INT PRIMARY KEY DEFAULT 1,
                company_name VARCHAR(200) NOT NULL DEFAULT 'ProfitPlanet GmbH',
                company_street VARCHAR(255) NOT NULL DEFAULT '',
                company_postal_city VARCHAR(255) NOT NULL DEFAULT '',
                company_country VARCHAR(100) NOT NULL DEFAULT 'Germany',
                qr_code_60_base64 LONGTEXT NULL,
                qr_code_120_base64 LONGTEXT NULL,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CHECK (id = 1)
            );
        `);
        // Seed default row if missing
        await connection.query(`
            INSERT IGNORE INTO company_settings (id) VALUES (1);
        `);
        console.log('✅ Company settings table created/verified');

        // Backward-compatible: add QR code columns if missing
        await addColumnIfMissing(connection, 'company_settings', 'qr_code_60_base64', 'LONGTEXT NULL');
        await addColumnIfMissing(connection, 'company_settings', 'qr_code_120_base64', 'LONGTEXT NULL');

        // --- I18n Preferences (single-row, admin language-management settings) ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS i18n_preferences (
                id TINYINT PRIMARY KEY DEFAULT 1,
                categories_json JSON NULL,
                global_keys_json JSON NULL,
                updated_by_user_id INT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CONSTRAINT chk_i18n_preferences_singleton CHECK (id = 1),
                FOREIGN KEY (updated_by_user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE
            );
        `);
        await connection.query(`
            INSERT IGNORE INTO i18n_preferences (id, categories_json, global_keys_json, updated_by_user_id)
            VALUES (1, JSON_ARRAY(), JSON_ARRAY(), NULL);
        `);
        console.log('✅ i18n preferences table created/verified');

        // Backward-compatible for older schemas
        await addColumnIfMissing(connection, 'i18n_preferences', 'categories_json', 'JSON NULL');
        await addColumnIfMissing(connection, 'i18n_preferences', 'global_keys_json', 'JSON NULL');
        await addColumnIfMissing(connection, 'i18n_preferences', 'updated_by_user_id', 'INT NULL');

        await addForeignKeyIfMissing(
            connection,
            'i18n_preferences',
            'i18n_preferences_ibfk_1',
            `ALTER TABLE \`i18n_preferences\`
             ADD CONSTRAINT \`i18n_preferences_ibfk_1\` FOREIGN KEY (\`updated_by_user_id\`) REFERENCES \`users\`(\`id\`) ON DELETE SET NULL ON UPDATE CASCADE`
        );

        // Language metadata used by language-management UI
        await connection.query(`
            CREATE TABLE IF NOT EXISTS i18n_languages (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                language_code VARCHAR(16) NOT NULL,
                label VARCHAR(100) NULL,
                is_enabled BOOLEAN NOT NULL DEFAULT TRUE,
                is_custom BOOLEAN NOT NULL DEFAULT FALSE,
                created_by_user_id INT NULL,
                updated_by_user_id INT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CONSTRAINT uq_i18n_languages_code UNIQUE (language_code),
                FOREIGN KEY (created_by_user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                FOREIGN KEY (updated_by_user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE
            );
        `);
        console.log('✅ i18n languages table created/verified');

        // Translation overrides/custom values per language
        await connection.query(`
            CREATE TABLE IF NOT EXISTS i18n_translation_overrides (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                language_code VARCHAR(16) NOT NULL,
                namespace VARCHAR(128) NOT NULL,
                t_key VARCHAR(255) NOT NULL,
                t_value LONGTEXT NOT NULL,
                is_custom BOOLEAN NOT NULL DEFAULT TRUE,
                created_by_user_id INT NULL,
                updated_by_user_id INT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CONSTRAINT uq_i18n_translation_overrides UNIQUE (language_code, namespace, t_key),
                FOREIGN KEY (language_code) REFERENCES i18n_languages(language_code) ON DELETE CASCADE ON UPDATE CASCADE,
                FOREIGN KEY (created_by_user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                FOREIGN KEY (updated_by_user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE
            );
        `);
        console.log('✅ i18n translation overrides table created/verified');

        await ensureIndex(connection, 'i18n_translation_overrides', 'idx_i18n_overrides_lang', '`language_code`');
        await ensureIndex(connection, 'i18n_translation_overrides', 'idx_i18n_overrides_namespace', '`namespace`');
        await ensureIndex(connection, 'i18n_translation_overrides', 'idx_i18n_overrides_lang_namespace', '`language_code`, `namespace`');

        // --- Dashboard Platforms (admin managed dashboard cards) ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS dashboard_plattforms (
                id VARCHAR(36) PRIMARY KEY,
                title VARCHAR(255) NOT NULL,
                description TEXT NULL,
                href VARCHAR(1024) NOT NULL,
                icon VARCHAR(255) NULL,
                color VARCHAR(255) NULL,
                state BOOLEAN NOT NULL DEFAULT TRUE,
                disabled BOOLEAN NOT NULL DEFAULT FALSE,
                disabled_text VARCHAR(255) NULL,
                sort_order INT NOT NULL DEFAULT 0,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
            );
        `);
        console.log('✅ Dashboard plattforms table created/verified');

        await ensureIndex(connection, 'dashboard_plattforms', 'idx_dashboard_plattforms_state', '`state`');
        await ensureIndex(connection, 'dashboard_plattforms', 'idx_dashboard_plattforms_sort_order', '`sort_order`');
        await ensureIndex(connection, 'dashboard_plattforms', 'idx_dashboard_plattforms_title', '`title`');

        // Optional one-time migration from legacy company_settings.dashboard_plattforms (JSON column)
        try {
            const hasLegacyColumn = await columnExists(connection, 'company_settings', 'dashboard_plattforms');
            if (hasLegacyColumn) {
                const [countRows] = await connection.query(`SELECT COUNT(*) AS c FROM dashboard_plattforms`);
                const count = Number(countRows?.[0]?.c ?? 0);
                if (count === 0) {
                    const [rows] = await connection.query(`SELECT dashboard_plattforms FROM company_settings WHERE id = 1`);
                    const raw = rows?.[0]?.dashboard_plattforms;

                    let parsed = [];
                    try {
                        if (raw == null) parsed = [];
                        else if (typeof raw === 'string') parsed = JSON.parse(raw);
                        else parsed = raw;
                    } catch (_) {
                        parsed = [];
                    }

                    const list = Array.isArray(parsed) ? parsed : [];
                    if (list.length) {
                        for (const p of list) {
                            const id = typeof p?.id === 'string' ? p.id.trim() : '';
                            if (!id) continue;

                            const title = typeof p?.title === 'string' ? p.title : '';
                            const href = typeof p?.href === 'string' ? p.href : '';

                            await connection.query(
                                `INSERT INTO dashboard_plattforms
                                   (id, title, description, href, icon, color, state, disabled, disabled_text, sort_order)
                                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                                 ON DUPLICATE KEY UPDATE
                                   title = VALUES(title),
                                   description = VALUES(description),
                                   href = VALUES(href),
                                   icon = VALUES(icon),
                                   color = VALUES(color),
                                   state = VALUES(state),
                                   disabled = VALUES(disabled),
                                   disabled_text = VALUES(disabled_text),
                                   sort_order = VALUES(sort_order)`,
                                [
                                    id,
                                    title,
                                    typeof p?.description === 'string' ? p.description : (p?.description == null ? '' : String(p.description)),
                                    href,
                                    typeof p?.icon === 'string' ? p.icon : (p?.icon == null ? '' : String(p.icon)),
                                    typeof p?.color === 'string' ? p.color : (p?.color == null ? '' : String(p.color)),
                                    p?.state === false ? 0 : 1,
                                    p?.disabled === true ? 1 : 0,
                                    p?.disabledText == null ? null : String(p.disabledText),
                                    Number.isFinite(Number(p?.sortOrder)) ? Number(p.sortOrder) : 0,
                                ]
                            );
                        }
                        console.log('🔁 Migrated legacy dashboard_plattforms JSON into dashboard_plattforms table');
                    }
                }
            }
        } catch (e) {
            console.warn('⚠️  Dashboard platforms migration skipped/failed:', e?.message || e);
        }

        // --- Rate Limiting Table ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS rate_limit (
                id INT AUTO_INCREMENT PRIMARY KEY,
                rate_key VARCHAR(255) NOT NULL,
                window_start DATETIME NOT NULL,
                count INT DEFAULT 0,
                window_seconds INT NOT NULL,
                max INT NOT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                UNIQUE KEY unique_key_window (rate_key, window_start)
            );
        `);
        console.log('✅ Rate limit table created/verified');

        // --- Tax: countries and VAT rates ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS countries (
                id INT AUTO_INCREMENT PRIMARY KEY,
                country_code VARCHAR(3) NOT NULL,
                country_name VARCHAR(255) NOT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                created_by INT NULL,
                updated_by INT NULL,
                CONSTRAINT uq_country_code UNIQUE (country_code),
                INDEX idx_country_name (country_name),
                FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE
            );
        `);
        console.log('✅ Countries table created/verified');

        await connection.query(`
            CREATE TABLE IF NOT EXISTS vat_rates (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                country_id INT NOT NULL,
                standard_rate DECIMAL(6,3) NULL,
                reduced_rate DECIMAL(6,3) NULL,
                super_reduced_rate DECIMAL(6,3) NULL,
                parking_rate DECIMAL(6,3) NULL,
                effective_from DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
                effective_to DATETIME NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                created_by INT NULL,
                updated_by INT NULL,
                FOREIGN KEY (country_id) REFERENCES countries(id) ON DELETE CASCADE ON UPDATE CASCADE,
                FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                INDEX idx_vat_country_eff_to (country_id, effective_to),
                INDEX idx_vat_standard (standard_rate)
            );
        `);
        console.log('✅ VAT rates table created/verified');

        await connection.query(`
            CREATE TABLE IF NOT EXISTS vat_rate_history (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                country_id INT NOT NULL,
                standard_rate DECIMAL(6,3) NULL,
                reduced_rate DECIMAL(6,3) NULL,
                super_reduced_rate DECIMAL(6,3) NULL,
                parking_rate DECIMAL(6,3) NULL,
                effective_from DATETIME NOT NULL,
                effective_to DATETIME NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                created_by INT NULL,
                updated_by INT NULL,
                FOREIGN KEY (country_id) REFERENCES countries(id) ON DELETE CASCADE ON UPDATE CASCADE,
                FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                INDEX idx_vat_hist_country_from (country_id, effective_from),
                INDEX idx_vat_hist_country_to (country_id, effective_to)
            );
        `);
        console.log('✅ VAT rate history table created/verified');

        // --- NEW: company_stamps table (for company/admin managed stamps) ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS company_stamps (
                id INT AUTO_INCREMENT PRIMARY KEY,
                company_id INT NOT NULL,
                label VARCHAR(100) NULL,
                mime_type VARCHAR(50) NOT NULL,
                image_base64 LONGTEXT NOT NULL,
                is_active BOOLEAN DEFAULT FALSE,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                FOREIGN KEY (company_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                UNIQUE KEY unique_company_label (company_id, label),
                INDEX idx_company_active (company_id, is_active)
            );
        `);
        console.log('✅ Company stamps table created/verified');

        // --- Coffee / Subscriptions Table (simplified) ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS coffee_table (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                title VARCHAR(200) NOT NULL,
                description TEXT NOT NULL,
                price DECIMAL(10,2) NOT NULL DEFAULT 0.00,
                currency CHAR(3) NOT NULL DEFAULT 'EUR',
                is_featured BOOLEAN NOT NULL DEFAULT FALSE,
                billing_interval ENUM('day','week','month','year') NULL,
                interval_count INT UNSIGNED NULL,
                object_storage_id VARCHAR(255) NULL,
                original_filename VARCHAR(255) NULL,
                state BOOLEAN NOT NULL DEFAULT TRUE,
                pack_group VARCHAR(100) NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                INDEX idx_pack_group (pack_group)
            );
        `);
        console.log('✅ Coffee table (simplified) created/verified');

                // --- Coffee shipping fees (fixed package sizes) ---
                await connection.query(`
                        CREATE TABLE IF NOT EXISTS coffee_shipping_fees (
                                id INT AUTO_INCREMENT PRIMARY KEY,
                                piece_count INT NOT NULL,
                                price DECIMAL(10,2) NOT NULL DEFAULT 0.00,
                                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                                CONSTRAINT uq_coffee_shipping_fees_piece_count UNIQUE (piece_count),
                                CONSTRAINT chk_coffee_shipping_fees_piece_count CHECK (piece_count IN (60, 120))
                        );
                `);
                await connection.query(`
                        INSERT INTO coffee_shipping_fees (piece_count, price)
                        VALUES
                            (60, 0.00),
                            (120, 0.00)
                        ON DUPLICATE KEY UPDATE
                            piece_count = VALUES(piece_count)
                `);
                console.log('✅ Coffee shipping fees table created/verified and seeded (60/120)');

        // --- Coffee Abonements (subscriptions) ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS coffee_abonements (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                pack_group VARCHAR(100) NOT NULL DEFAULT '',
                status ENUM('active','paused','canceled','expired') NOT NULL DEFAULT 'active',
                started_at DATETIME NOT NULL,
                ended_at DATETIME NULL,
                next_billing_at DATETIME NULL,
                billing_interval ENUM('day','week','month','year') NOT NULL,
                interval_count INT UNSIGNED NOT NULL DEFAULT 1,
                price DECIMAL(10,2) NOT NULL,
                currency CHAR(3) NOT NULL,
                is_auto_renew TINYINT(1) NOT NULL DEFAULT 1,
                notes VARCHAR(255) NULL,
                pack_breakdown JSON NULL,
                first_name VARCHAR(100) NULL,
                last_name VARCHAR(100) NULL,
                email VARCHAR(255) NULL,
                street VARCHAR(255) NULL,
                postal_code VARCHAR(20) NULL,
                city VARCHAR(100) NULL,
                country VARCHAR(100) NULL,
                frequency VARCHAR(50) NULL,
                referred_by INT NULL, -- NEW: user_id of the logged-in user who referred the subscription
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CONSTRAINT fk_abon_referred_by FOREIGN KEY (referred_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                INDEX idx_pack_group (pack_group),
                INDEX idx_abon_status (status),
                INDEX idx_abon_billing (next_billing_at),
                INDEX idx_abon_created (created_at)
            );
        `);
        console.log('✅ Coffee abonements table updated');

        // Ownership columns for "self" and "gift" subscriptions
        await addColumnIfMissing(
            connection,
            'coffee_abonements',
            'user_id',
            `INT NULL AFTER referred_by`
        );
        await addColumnIfMissing(
            connection,
            'coffee_abonements',
            'purchaser_user_id',
            `INT NULL AFTER user_id`
        );

        await ensureIndex(connection, 'coffee_abonements', 'idx_abon_user_id', '`user_id`');
        await ensureIndex(connection, 'coffee_abonements', 'idx_abon_purchaser_user_id', '`purchaser_user_id`');

        await addForeignKeyIfMissing(
            connection,
            'coffee_abonements',
            'fk_abon_user',
            `ALTER TABLE \`coffee_abonements\`
             ADD CONSTRAINT \`fk_abon_user\` FOREIGN KEY (\`user_id\`) REFERENCES \`users\`(\`id\`) ON DELETE SET NULL ON UPDATE CASCADE`
        );
        await addForeignKeyIfMissing(
            connection,
            'coffee_abonements',
            'fk_abon_purchaser_user',
            `ALTER TABLE \`coffee_abonements\`
             ADD CONSTRAINT \`fk_abon_purchaser_user\` FOREIGN KEY (\`purchaser_user_id\`) REFERENCES \`users\`(\`id\`) ON DELETE SET NULL ON UPDATE CASCADE`
        );

        // Contract fields
        await addColumnIfMissing(connection, 'coffee_abonements', 'contract_number', `VARCHAR(50) NULL AFTER purchaser_user_id`);
        await addColumnIfMissing(connection, 'coffee_abonements', 'contract_storage_key', `VARCHAR(255) NULL AFTER contract_number`);

        // Additional shipping / contact fields
        await addColumnIfMissing(connection, 'coffee_abonements', 'phone', `VARCHAR(50) NULL AFTER country`);

        // Contract recipient
        await addColumnIfMissing(connection, 'coffee_abonements', 'recipient_name', `VARCHAR(255) NULL AFTER phone`);
        await addColumnIfMissing(connection, 'coffee_abonements', 'recipient_address', `TEXT NULL AFTER recipient_name`);

        // Payment
        await addColumnIfMissing(connection, 'coffee_abonements', 'payment_method', `VARCHAR(30) NULL AFTER frequency`);
        await addColumnIfMissing(connection, 'coffee_abonements', 'invoice_by_email', `TINYINT(1) NOT NULL DEFAULT 0 AFTER payment_method`);

        // Invoice address
        await addColumnIfMissing(connection, 'coffee_abonements', 'invoice_same_as_shipping', `TINYINT(1) NOT NULL DEFAULT 1 AFTER invoice_by_email`);
        await addColumnIfMissing(connection, 'coffee_abonements', 'invoice_full_name', `VARCHAR(200) NULL AFTER invoice_same_as_shipping`);
        await addColumnIfMissing(connection, 'coffee_abonements', 'invoice_street', `VARCHAR(255) NULL AFTER invoice_full_name`);
        await addColumnIfMissing(connection, 'coffee_abonements', 'invoice_postal_code', `VARCHAR(20) NULL AFTER invoice_street`);
        await addColumnIfMissing(connection, 'coffee_abonements', 'invoice_city', `VARCHAR(100) NULL AFTER invoice_postal_code`);
        await addColumnIfMissing(connection, 'coffee_abonements', 'invoice_phone', `VARCHAR(50) NULL AFTER invoice_city`);
        await addColumnIfMissing(connection, 'coffee_abonements', 'invoice_email', `VARCHAR(255) NULL AFTER invoice_phone`);

        // --- Coffee Abonement History ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS coffee_abonement_history (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                abonement_id BIGINT NOT NULL,
                event_type VARCHAR(50) NOT NULL,
                event_at DATETIME NOT NULL,
                actor_user_id INT NULL,
                details JSON NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                CONSTRAINT fk_hist_abon FOREIGN KEY (abonement_id) REFERENCES coffee_abonements(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_hist_actor FOREIGN KEY (actor_user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                INDEX idx_hist_abon (abonement_id),
                INDEX idx_hist_event_at (event_at),
                INDEX idx_hist_event_type (event_type)
            );
        `);
        console.log('✅ Coffee abonement history table created/verified');

        // --- Invoices: unified for subscriptions and shop orders ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS invoices (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                invoice_number VARCHAR(64) NOT NULL UNIQUE,
                user_id INT NULL,
                source_type ENUM('subscription','shop') NOT NULL,
                source_id BIGINT NOT NULL,
                buyer_name VARCHAR(255) NULL,
                buyer_email VARCHAR(255) NULL,
                buyer_street VARCHAR(255) NULL,
                buyer_postal_code VARCHAR(20) NULL,
                buyer_city VARCHAR(100) NULL,
                buyer_country VARCHAR(100) NULL,
                currency CHAR(3) NOT NULL,
                total_net DECIMAL(12,2) NOT NULL DEFAULT 0.00,
                total_tax DECIMAL(12,2) NOT NULL DEFAULT 0.00,
                total_gross DECIMAL(12,2) NOT NULL DEFAULT 0.00,
                vat_rate DECIMAL(6,3) NULL,
                status ENUM('draft','issued','paid','canceled') NOT NULL DEFAULT 'draft',
                issued_at DATETIME NULL,
                due_at DATETIME NULL,
                pdf_storage_key VARCHAR(255) NULL,
                context JSON NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CONSTRAINT fk_invoice_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                INDEX idx_invoice_source (source_type, source_id),
                INDEX idx_invoice_issued (issued_at),
                INDEX idx_invoice_status (status)
            );
        `);
        console.log('✅ Invoices table created/verified');

        await connection.query(`
            CREATE TABLE IF NOT EXISTS invoice_items (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                invoice_id BIGINT NOT NULL,
                product_id BIGINT NULL,
                sku VARCHAR(128) NULL,
                description VARCHAR(512) NOT NULL,
                quantity DECIMAL(12,3) NOT NULL DEFAULT 1.000,
                unit_price DECIMAL(12,2) NOT NULL DEFAULT 0.00,
                tax_rate DECIMAL(6,3) NULL,
                line_net DECIMAL(12,2) NOT NULL DEFAULT 0.00,
                line_tax DECIMAL(12,2) NOT NULL DEFAULT 0.00,
                line_gross DECIMAL(12,2) NOT NULL DEFAULT 0.00,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                CONSTRAINT fk_item_invoice FOREIGN KEY (invoice_id) REFERENCES invoices(id) ON DELETE CASCADE ON UPDATE CASCADE,
                INDEX idx_item_invoice (invoice_id)
            );
        `);
        console.log('✅ Invoice items table created/verified');

        await connection.query(`
            CREATE TABLE IF NOT EXISTS invoice_payments (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                invoice_id BIGINT NOT NULL,
                payment_method VARCHAR(64) NOT NULL,
                transaction_id VARCHAR(128) NULL,
                amount DECIMAL(12,2) NOT NULL,
                paid_at DATETIME NULL,
                status ENUM('succeeded','pending','failed','refunded') NOT NULL DEFAULT 'pending',
                details JSON NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                CONSTRAINT fk_payment_invoice FOREIGN KEY (invoice_id) REFERENCES invoices(id) ON DELETE CASCADE ON UPDATE CASCADE,
                INDEX idx_payment_invoice (invoice_id),
                INDEX idx_payment_status (status),
                INDEX idx_payment_paid_at (paid_at)
            );
        `);
        console.log('✅ Invoice payments table created/verified');

        // Extend coffee_abonement_history with optional related_invoice_id
        await addColumnIfMissing(connection, 'coffee_abonement_history', 'related_invoice_id', `BIGINT NULL`);
        await addForeignKeyIfMissing(
            connection,
            'coffee_abonement_history',
            'fk_hist_invoice',
            `
            ALTER TABLE coffee_abonement_history
            ADD CONSTRAINT fk_hist_invoice FOREIGN KEY (related_invoice_id)
            REFERENCES invoices(id) ON DELETE SET NULL ON UPDATE CASCADE
            `
        );

        // --- Matrix: Global 5-ary tree config and relations ---
        await connection.query(`
            CREATE TABLE IF NOT EXISTS matrix_config (
                id TINYINT PRIMARY KEY DEFAULT 1,
                master_top_user_id INT NOT NULL,
                name VARCHAR(255) NULL,                -- ADDED (was missing, caused Unknown column 'name')
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CONSTRAINT fk_matrix_config_master FOREIGN KEY (master_top_user_id) REFERENCES users(id) ON DELETE RESTRICT ON UPDATE CASCADE,
                CONSTRAINT chk_matrix_singleton CHECK (id = 1)
            );
        `);
        // Safeguard: if table pre-existed without name column, add it (pre-check)
        await addColumnIfMissing(connection, 'matrix_config', 'name', `VARCHAR(255) NULL`);

        console.log('✅ Matrix config table created/verified');

        await connection.query(`
            CREATE TABLE IF NOT EXISTS user_tree_edges (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                parent_user_id INT NOT NULL,
                child_user_id INT NOT NULL,
                position INT NOT NULL,                                -- CHANGED: allow unlimited positions
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                CONSTRAINT fk_edges_parent FOREIGN KEY (parent_user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_edges_child FOREIGN KEY (child_user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT uq_child UNIQUE (child_user_id),
                CONSTRAINT uq_parent_position UNIQUE (parent_user_id, position)
                -- REMOVED: chk_position CHECK (position BETWEEN 1 AND 5)
            );
        `);
        console.log('✅ User tree edges table created/verified');

        // --- Migration: relax position constraints on existing schemas ---
        try {
          await connection.query(`ALTER TABLE user_tree_edges MODIFY COLUMN position INT NOT NULL`);
          console.log('🛠️  user_tree_edges.position changed to INT');
        } catch (e) {
          console.log('ℹ️  position type change skipped:', e.message);
        }
        try {
                    try {
                        await connection.query('ALTER TABLE user_tree_edges DROP CONSTRAINT chk_position');
                    } catch (e1) {
                        await connection.query('ALTER TABLE user_tree_edges DROP CHECK chk_position');
                    }
          console.log('🧹 Dropped CHECK constraint chk_position on user_tree_edges');
        } catch (e) {
          // MySQL versions or engines may report different messages if CHECK is not enforced or named differently
          console.log('ℹ️  DROP CHECK chk_position skipped:', e.message);
        }

        await connection.query(`
            CREATE TABLE IF NOT EXISTS matrix_instances (
              id INT AUTO_INCREMENT PRIMARY KEY,
              root_user_id INT NOT NULL,
              name VARCHAR(255) NULL,
              is_active BOOLEAN DEFAULT TRUE,
              max_depth INT NULL,
              ego_activated_at TIMESTAMP NULL,
              immediate_children_count INT DEFAULT 0,
              first_free_position TINYINT NULL,
              created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
              updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
              CONSTRAINT fk_matrix_instances_root FOREIGN KEY (root_user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
              CONSTRAINT chk_first_free_pos CHECK (first_free_position IS NULL OR (first_free_position BETWEEN 1 AND 5))
            );
        `);
        console.log('✅ matrix_instances table created/verified');

        await connection.query(`
            CREATE TABLE IF NOT EXISTS pools (
                id INT AUTO_INCREMENT PRIMARY KEY,
                pool_name VARCHAR(255) NOT NULL,
                description TEXT NULL,
                price DECIMAL(10,2) NOT NULL DEFAULT 0.00,
                subscription_coffee_id BIGINT NULL,
                pool_type ENUM('coffee','other') NOT NULL DEFAULT 'other',
                is_active TINYINT(1) NOT NULL DEFAULT 1,
                created_by INT NULL,
                updated_by INT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CONSTRAINT uq_pools_name UNIQUE (pool_name),
                CONSTRAINT fk_pools_subscription_coffee FOREIGN KEY (subscription_coffee_id) REFERENCES coffee_table(id) ON DELETE SET NULL ON UPDATE CASCADE,
                CONSTRAINT fk_pools_created_by FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                CONSTRAINT fk_pools_updated_by FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                INDEX idx_pools_active (is_active),
                INDEX idx_pools_type (pool_type),
                INDEX idx_pools_subscription_coffee (subscription_coffee_id)
            );
        `);
        console.log('✅ Pools table created/verified');

        // Backward-compatible migration for existing pools table
        await addColumnIfMissing(connection, 'pools', 'subscription_coffee_id', `BIGINT NULL`);
        await ensureIndex(connection, 'pools', 'idx_pools_subscription_coffee', '`subscription_coffee_id`');
        await addForeignKeyIfMissing(
            connection,
            'pools',
            'fk_pools_subscription_coffee',
            `
            ALTER TABLE pools
            ADD CONSTRAINT fk_pools_subscription_coffee FOREIGN KEY (subscription_coffee_id)
            REFERENCES coffee_table(id) ON DELETE SET NULL ON UPDATE CASCADE
            `
        );

        await connection.query(`
            CREATE TABLE IF NOT EXISTS pool_capsule_rules (
                id INT AUTO_INCREMENT PRIMARY KEY,
                pool_id INT NOT NULL,
                price_per_capsule_gross DECIMAL(10,4) NOT NULL,
                applies_to_all_capsules TINYINT(1) NOT NULL DEFAULT 1,
                is_active TINYINT(1) NOT NULL DEFAULT 1,
                created_by INT NULL,
                updated_by INT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CONSTRAINT uq_pool_capsule_rules_pool UNIQUE (pool_id),
                CONSTRAINT fk_pool_capsule_rules_pool FOREIGN KEY (pool_id) REFERENCES pools(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_pool_capsule_rules_created_by FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                CONSTRAINT fk_pool_capsule_rules_updated_by FOREIGN KEY (updated_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                INDEX idx_pool_capsule_rules_active (is_active)
            );
        `);
        console.log('✅ pool_capsule_rules table created/verified');

        for (const cfg of SYSTEM_POOLS) {
            await connection.query(
                `INSERT INTO pools (pool_name, description, price, subscription_coffee_id, pool_type, is_active, created_by, updated_by)
                 VALUES (?, ?, ?, NULL, ?, 1, NULL, NULL)
                 ON DUPLICATE KEY UPDATE
                   description = VALUES(description),
                   price = VALUES(price),
                   subscription_coffee_id = NULL,
                   pool_type = VALUES(pool_type),
                   is_active = 1,
                   updated_by = NULL,
                   updated_at = NOW()`,
                [cfg.pool_name, cfg.description, cfg.price, cfg.pool_type]
            );

            const [poolRows] = await connection.query(
                `SELECT id FROM pools WHERE pool_name = ? LIMIT 1`,
                [cfg.pool_name]
            );
            const poolId = poolRows?.[0]?.id;
            if (!poolId) continue;

            await connection.query(
                `INSERT INTO pool_capsule_rules (pool_id, price_per_capsule_gross, applies_to_all_capsules, is_active, created_by, updated_by)
                 VALUES (?, ?, 1, 1, NULL, NULL)
                 ON DUPLICATE KEY UPDATE
                   price_per_capsule_gross = VALUES(price_per_capsule_gross),
                   applies_to_all_capsules = 1,
                   is_active = 1,
                   updated_by = NULL,
                   updated_at = NOW()`,
                [poolId, cfg.price_per_capsule_gross]
            );
        }

        const systemPoolNames = SYSTEM_POOLS.map((x) => x.pool_name);
        const systemPoolPlaceholders = systemPoolNames.map(() => '?').join(',');
        await connection.query(
            `UPDATE pools
             SET is_active = 0,
                 updated_at = NOW()
             WHERE pool_name NOT IN (${systemPoolPlaceholders})`,
            systemPoolNames
        );
        console.log('✅ System pools synchronized (ABO 60, ABO 120, Business, Gigantea, Core)');

        await connection.query(`
            CREATE TABLE IF NOT EXISTS pool_members (
                id INT AUTO_INCREMENT PRIMARY KEY,
                pool_id INT NOT NULL,
                user_id INT NOT NULL,
                created_by INT NULL,
                joined_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CONSTRAINT fk_pool_members_pool FOREIGN KEY (pool_id) REFERENCES pools(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_pool_members_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_pool_members_created_by FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                CONSTRAINT uq_pool_members UNIQUE (pool_id, user_id),
                INDEX idx_pool_members_pool (pool_id),
                INDEX idx_pool_members_user (user_id)
            );
        `);
        console.log('✅ pool_members table created/verified');

        // Track sold capsules per paid invoice as calculation basis for pools
        await connection.query(`
            CREATE TABLE IF NOT EXISTS capsule_sales (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                invoice_id BIGINT NOT NULL,
                abonement_id BIGINT NOT NULL,
                coffee_table_id BIGINT NOT NULL,
                capsules_count INT NOT NULL,
                sold_at DATETIME NOT NULL,
                currency CHAR(3) NOT NULL DEFAULT 'EUR',
                details JSON NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
                CONSTRAINT fk_capsule_sales_invoice FOREIGN KEY (invoice_id) REFERENCES invoices(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_capsule_sales_abon FOREIGN KEY (abonement_id) REFERENCES coffee_abonements(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_capsule_sales_coffee FOREIGN KEY (coffee_table_id) REFERENCES coffee_table(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT uq_capsule_sales_invoice_coffee UNIQUE (invoice_id, coffee_table_id),
                INDEX idx_capsule_sales_invoice (invoice_id),
                INDEX idx_capsule_sales_abon (abonement_id),
                INDEX idx_capsule_sales_sold_at (sold_at)
            );
        `);
        console.log('✅ capsule_sales table created/verified');

        // Track money inflow into pools from subscriptions/invoices
        await connection.query(`
            CREATE TABLE IF NOT EXISTS pool_inflows (
                id BIGINT AUTO_INCREMENT PRIMARY KEY,
                pool_id INT NOT NULL,
                invoice_id BIGINT NULL,
                abonement_id BIGINT NOT NULL,
                coffee_table_id BIGINT NOT NULL,
                event_type ENUM('invoice_paid','subscription_created','renewal_paid','manual_adjustment') NOT NULL DEFAULT 'invoice_paid',
                capsules_count INT NOT NULL,
                price_per_capsule_net DECIMAL(10,4) NOT NULL,
                amount_net DECIMAL(12,2) NOT NULL,
                currency CHAR(3) NOT NULL DEFAULT 'EUR',
                details JSON NULL,
                created_by_user_id INT NULL,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                CONSTRAINT fk_pool_inflows_pool FOREIGN KEY (pool_id) REFERENCES pools(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_pool_inflows_invoice FOREIGN KEY (invoice_id) REFERENCES invoices(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_pool_inflows_abon FOREIGN KEY (abonement_id) REFERENCES coffee_abonements(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_pool_inflows_coffee FOREIGN KEY (coffee_table_id) REFERENCES coffee_table(id) ON DELETE CASCADE ON UPDATE CASCADE,
                CONSTRAINT fk_pool_inflows_created_by FOREIGN KEY (created_by_user_id) REFERENCES users(id) ON DELETE SET NULL ON UPDATE CASCADE,
                CONSTRAINT uq_pool_inflow_invoice_event UNIQUE (pool_id, invoice_id, coffee_table_id, event_type),
                INDEX idx_pool_inflows_pool_created (pool_id, created_at),
                INDEX idx_pool_inflows_abon (abonement_id),
                INDEX idx_pool_inflows_invoice (invoice_id)
            );
        `);
        console.log('✅ pool_inflows table created/verified');

        // Backward-compatible migration for existing pool_inflows table
        await addColumnIfMissing(connection, 'pool_inflows', 'invoice_id', `BIGINT NULL`);
        await ensureIndex(connection, 'pool_inflows', 'idx_pool_inflows_invoice', '`invoice_id`');
        await addForeignKeyIfMissing(
            connection,
            'pool_inflows',
            'fk_pool_inflows_invoice',
            `
            ALTER TABLE pool_inflows
            ADD CONSTRAINT fk_pool_inflows_invoice FOREIGN KEY (invoice_id)
            REFERENCES invoices(id) ON DELETE CASCADE ON UPDATE CASCADE
            `
        );
        try {
            await connection.query(`
                ALTER TABLE pool_inflows
                MODIFY COLUMN event_type ENUM('invoice_paid','subscription_created','renewal_paid','manual_adjustment') NOT NULL DEFAULT 'invoice_paid'
            `);
        } catch (e) {
            console.log('ℹ️  pool_inflows.event_type enum alignment skipped:', e.message);
        }
        try {
            await connection.query(`ALTER TABLE pool_inflows DROP INDEX uq_pool_inflow_event`);
        } catch (e) {
            console.log('ℹ️  old pool inflow unique index drop skipped:', e.message);
        }
        try {
            await connection.query(`
                ALTER TABLE pool_inflows
                ADD CONSTRAINT uq_pool_inflow_invoice_event UNIQUE (pool_id, invoice_id, coffee_table_id, event_type)
            `);
        } catch (e) {
            console.log('ℹ️  new pool inflow unique index creation skipped:', e.message);
        }

        // --- user_matrix_metadata: add matrix_instance_id + alter PK ---
        await connection.query(`
          CREATE TABLE IF NOT EXISTS user_matrix_metadata (
            matrix_instance_id INT PRIMARY KEY,
            root_user_id INT NOT NULL,
            ego_activated_at TIMESTAMP NULL,
            last_bfs_fill_at TIMESTAMP NULL,
            immediate_children_count INT DEFAULT 0,
            first_free_position TINYINT NULL,
            name VARCHAR(255) NULL,
            is_active BOOLEAN DEFAULT TRUE,
            max_depth INT NULL,
            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
            CONSTRAINT fk_meta_instance FOREIGN KEY (matrix_instance_id) REFERENCES matrix_instances(id) ON DELETE CASCADE ON UPDATE CASCADE,
            CONSTRAINT fk_meta_root FOREIGN KEY (root_user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
            CONSTRAINT chk_meta_first_free CHECK (first_free_position IS NULL OR (first_free_position BETWEEN 1 AND 5)),
            INDEX idx_meta_root (root_user_id)
          );
        `);
        console.log('✅ user_matrix_metadata (multi) created/verified');

        // Migration: if legacy data without matrix_instances rows
        const [instCheck] = await connection.query(`SELECT COUNT(*) AS cnt FROM matrix_instances`);
        if (Number(instCheck[0].cnt) === 0) {
          const [legacyCfg] = await connection.query(`SELECT master_top_user_id, name FROM matrix_config WHERE id=1`);
            if (legacyCfg.length) {
              const legacyRoot = legacyCfg[0].master_top_user_id;
              const legacyName = legacyCfg[0].name || null;
              // Determine existing root children stats
              const [legacyEdges] = await connection.query(`SELECT position FROM user_tree_edges WHERE parent_user_id = ?`, [legacyRoot]);
              const usedRootPos = new Set(legacyEdges.map(r => Number(r.position)));
              let firstFree = null;
              for (let i = 1; i <= 5; i++) { if (!usedRootPos.has(i)) { firstFree = i; break; } }
              // Create initial instance
              const [instRes] = await connection.query(`
                INSERT INTO matrix_instances
                  (root_user_id, name, is_active, max_depth, ego_activated_at, immediate_children_count, first_free_position)
                VALUES (?, ?, TRUE, NULL, NOW(), ?, NULL)         -- CHANGED: first_free_position NULL for root
              `, [legacyRoot, legacyName, legacyEdges.length, /* firstFree removed */ null]);
              const firstInstanceId = instRes.insertId;
              // Backfill metadata
              await connection.query(`
                INSERT INTO user_matrix_metadata
                  (matrix_instance_id, root_user_id, ego_activated_at, immediate_children_count, first_free_position, name, is_active, max_depth)
                VALUES (?, ?, NOW(), ?, ?, ?, TRUE, NULL)
              `, [firstInstanceId, legacyRoot, legacyEdges.length, firstFree, legacyName]);
              console.log('🧩 Migration: created first matrix_instance id=', firstInstanceId);
            }
        }

        // --- Legacy cleanup: remove old matrix_id / fk_edges_matrix referencing obsolete `matrices` table ---
        try {
          const [legacyCols] = await connection.query(`SHOW COLUMNS FROM user_tree_edges LIKE 'matrix_id'`);
          if (legacyCols.length) {
            console.log('🧹 Found legacy user_tree_edges.matrix_id; dropping old FK & column');
            try { await connection.query(`ALTER TABLE user_tree_edges DROP FOREIGN KEY fk_edges_matrix`); } catch (e) {
              console.log('ℹ️  fk_edges_matrix drop skipped:', e.message);
            }
            try { await connection.query(`ALTER TABLE user_tree_edges DROP COLUMN matrix_id`); } catch (e) {
              console.log('ℹ️  matrix_id column drop skipped:', e.message);
            }
          }
        } catch (e) {
          console.log('ℹ️  Legacy matrix_id cleanup check failed:', e.message);
        }

        // --- Ensure multi-instance columns (idempotent) ---
        await addColumnIfMissing(connection, 'user_tree_edges', 'matrix_instance_id', `INT NULL`);
        await addColumnIfMissing(connection, 'user_tree_edges', 'rogue_user', `BOOLEAN DEFAULT FALSE`);

        await addForeignKeyIfMissing(
            connection,
            'user_tree_edges',
            'fk_edges_instance',
            `
            ALTER TABLE user_tree_edges
            ADD CONSTRAINT fk_edges_instance FOREIGN KEY (matrix_instance_id)
            REFERENCES matrix_instances(id) ON DELETE CASCADE ON UPDATE CASCADE
            `
        );

        // Backfill matrix_instance_id for existing edges
        const [firstInstRow] = await connection.query(`SELECT id FROM matrix_instances ORDER BY id ASC LIMIT 1`);
        const firstInstanceId = firstInstRow[0]?.id;
        if (firstInstanceId) {
            await connection.query(
                `UPDATE user_tree_edges SET matrix_instance_id = ? WHERE matrix_instance_id IS NULL`,
                [firstInstanceId]
            );
        }

        // Indexes (pre-check)
        await ensureIndex(connection, 'user_tree_edges', 'idx_edges_instance_parent', '`matrix_instance_id`, `parent_user_id`');
        await ensureIndex(connection, 'user_tree_edges', 'idx_edges_instance_child', '`matrix_instance_id`, `child_user_id`');
        await ensureIndex(connection, 'user_tree_edges', 'idx_edges_rogue', '`matrix_instance_id`, `rogue_user`');

        // --- Alter user_tree_closure: add matrix_instance_id ---
        await connection.query(`
          CREATE TABLE IF NOT EXISTS user_tree_closure (
            matrix_instance_id INT NOT NULL,
            ancestor_user_id INT NOT NULL,
            descendant_user_id INT NOT NULL,
            depth INT NOT NULL,
            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
            CONSTRAINT pk_closure PRIMARY KEY (matrix_instance_id, ancestor_user_id, descendant_user_id),
            CONSTRAINT fk_closure_instance FOREIGN KEY (matrix_instance_id) REFERENCES matrix_instances(id) ON DELETE CASCADE ON UPDATE CASCADE,
            CONSTRAINT fk_closure_ancestor FOREIGN KEY (ancestor_user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
            CONSTRAINT fk_closure_descendant FOREIGN KEY (descendant_user_id) REFERENCES users(id) ON DELETE CASCADE ON UPDATE CASCADE,
            CONSTRAINT chk_depth_nonneg CHECK (depth >= 0)
          );
        `);
        console.log('✅ user_tree_closure (multi) created/verified');

        // If legacy closure table existed without matrix_instance_id, migrate it safely (pre-check)
        if (!(await columnExists(connection, 'user_tree_closure', 'matrix_instance_id'))) {
            await connection.query(`ALTER TABLE user_tree_closure ADD COLUMN matrix_instance_id INT NULL`);
            if (firstInstanceId) {
                await connection.query(
                    `UPDATE user_tree_closure SET matrix_instance_id = ? WHERE matrix_instance_id IS NULL`,
                    [firstInstanceId]
                );
            }
            // Best-effort: enforce NOT NULL after backfill
            try {
                await connection.query(`ALTER TABLE user_tree_closure MODIFY COLUMN matrix_instance_id INT NOT NULL`);
            } catch (e) {
                console.log('ℹ️  user_tree_closure.matrix_instance_id NOT NULL enforcement skipped:', e.message);
            }

            // Best-effort: align PK to (matrix_instance_id, ancestor_user_id, descendant_user_id)
            try {
                const pkCols = await getPrimaryKeyColumns(connection, 'user_tree_closure');
                const wants = ['matrix_instance_id', 'ancestor_user_id', 'descendant_user_id'];
                const matches =
                    pkCols.length === wants.length && pkCols.every((c, i) => c === wants[i]);

                if (!matches) {
                    await connection.query(`ALTER TABLE user_tree_closure DROP PRIMARY KEY`);
                    await connection.query(
                        `ALTER TABLE user_tree_closure ADD PRIMARY KEY (matrix_instance_id, ancestor_user_id, descendant_user_id)`
                    );
                }
            } catch (e) {
                console.log('ℹ️  user_tree_closure PK alignment skipped:', e.message);
            }
        }

        await ensureIndex(connection, 'user_tree_closure', 'idx_closure_instance_depth', '`matrix_instance_id`, `depth`');
        await ensureIndex(connection, 'user_tree_closure', 'idx_closure_instance_ancestor', '`matrix_instance_id`, `ancestor_user_id`');

        // Remove singleton constraint if present (best effort)
        try {
                    try {
                        await connection.query('ALTER TABLE matrix_config DROP CONSTRAINT chk_matrix_singleton');
                    } catch (e1) {
                        await connection.query('ALTER TABLE matrix_config DROP CHECK chk_matrix_singleton');
                    }
          console.log('🧹 Dropped chk_matrix_singleton');
        } catch (e) {
          console.log('ℹ️  chk_matrix_singleton drop skipped:', e.message);
        }

        // --- Added Index Optimization Section ---
        try {
            // Core / status
            await ensureIndex(connection, 'users', 'idx_users_created_at', 'created_at');
            await ensureIndex(connection, 'user_status', 'idx_user_status_status', 'status');
            await ensureIndex(connection, 'user_status', 'idx_user_status_registration_completed', 'registration_completed');

            // Tokens & auth
            await ensureIndex(connection, 'refresh_tokens', 'idx_refresh_user_expires', 'user_id, expires_at');
            await ensureIndex(connection, 'email_verifications', 'idx_email_verifications_user_expires', 'user_id, expires_at');
            await ensureIndex(connection, 'password_resets', 'idx_password_resets_user_expires', 'user_id, expires_at');

            // Documents
            await ensureIndex(connection, 'user_documents', 'idx_user_documents_upload_at', 'upload_at');
            await ensureIndex(connection, 'user_documents', 'idx_user_documents_verified', 'verified_by_admin');
            await ensureIndex(connection, 'user_id_documents', 'idx_user_id_docs_user_type', 'user_id, document_type');

            // Activity logs (composite for common filtered ordering)
            await ensureIndex(connection, 'user_action_logs', 'idx_user_action_logs_action_created', 'action, created_at');

            // Referrals
            await ensureIndex(connection, 'referral_token_usage', 'idx_referral_token_usage_used_at', 'used_at');

            // Permissions
            await ensureIndex(connection, 'permissions', 'idx_permissions_is_active', 'is_active');
            await ensureIndex(connection, 'user_permissions', 'idx_user_permissions_granted_by', 'granted_by');

            // Settings
            await ensureIndex(connection, 'user_settings', 'idx_user_settings_theme', 'theme');

            // Rate limit (for queries only on rate_key)
            await ensureIndex(connection, 'rate_limit', 'idx_rate_limit_rate_key', 'rate_key');
            await ensureIndex(connection, 'document_templates', 'idx_document_templates_user_type', 'user_type');
            await ensureIndex(connection, 'document_templates', 'idx_document_templates_state_user_type', 'state, user_type');
            await ensureIndex(connection, 'company_stamps', 'idx_company_stamps_company', 'company_id');
            await ensureIndex(connection, 'company_stamps', 'idx_company_stamps_active', 'is_active');
            console.log('🚀 Performance indexes created/verified');
        } catch (e) {
            console.warn('⚠️  Index optimization phase encountered an issue:', e.message);
        }

        console.log('🎉 Normalized database schema created/updated successfully!');
        
        await connection.end();
        return true;
        
    } catch (error) {
        console.error('💥 Error during database initialization:', error.message);
        if (connection) {
            await connection.end();
        }
        throw error;
    }
}

module.exports = { createDatabase };