const fs = require("node:fs"); const path = require("node:path"); const sql = require("./db"); const config = require("./dbconfig"); const { Readable } = require('stream'); const os = require("os"); const { S3Client, GetObjectCommand, DeleteObjectCommand } = require("@aws-sdk/client-s3"); const archiver = require("archiver"); const { Upload } = require("@aws-sdk/lib-storage"); const s3Client = new S3Client({ region: "auto", endpoint: process.env.R2_ENDPOINT, credentials: { accessKeyId: process.env.R2_ACCESS_KEY_ID, secretAccessKey: process.env.R2_SECRET_ACCESS_KEY, }, }); const dirPath = path.join(__dirname, "/Exports"); // Cleaned up the redundancy: Merged into one single focused operational query method async function exportMemoriesCheck(userID) { try { const pool = await sql.connect(config); const myproc = new sql.Request(pool); myproc.input("userID", sql.Int, parseInt(userID, 10)); const result = await myproc.execute("checkExportDate"); // Unpack database table row results safely from your custom database layer const rawRows = result.recordset || result.rows || result; // Fixed: Multi-case fallback lookup grabs the binary permission flag regardless of uppercase mutations! let isAllowed = 0; if (Array.isArray(rawRows) && rawRows.length > 0) { const row = rawRows[0]; isAllowed = row.ExportAllowed !== undefined ? row.ExportAllowed : (row.exportallowed || row.EXPORTALLOWED || 0); } else if (rawRows) { isAllowed = rawRows.ExportAllowed !== undefined ? rawRows.ExportAllowed : (rawRows.exportallowed || rawRows.EXPORTALLOWED || 0); } console.log(`๐Ÿ“Š Export Initialization check: User ${userID} permission status flag resolves to: ${isAllowed}`); return Number(isAllowed); // Returns a clean, guaranteed primitive number 1 or 0 straight to your API route! } catch (error) { console.error("๐Ÿ”ด Error inside exportMemoriesCheck operation handler:", error); throw error; } } async function getMediaExportList(userID) { try { // 1. Lock the active background processing flag await setExportStarted(userID); const pool = await sql.connect(config); const myproc = new sql.Request(pool); const cleanUserID = parseInt(userID, 10); myproc.input("userID", sql.Int, cleanUserID); console.log(`๐Ÿ“ก Sending "getExportMedia" link query to Supabase for User ID: ${cleanUserID}...`); const media = await myproc.execute("getExportMedia"); // 2. Safely extract media row arrays regardless of nested properties const rawRows = media.recordset || media.rows || media; const mediaList = Array.isArray(rawRows) ? rawRows : (rawRows.recordset ? rawRows.recordset : []); console.log(`๐Ÿ“ฆ Found ${mediaList.length} attachment links. Initializing "createMediaZip" file compressor loop...`); // 3. Pass clean rows to your automated archive bundling microservice await createMediaZip(mediaList, cleanUserID); console.log(`โš™๏ธ "createMediaZip" compression pipeline completed for User ${cleanUserID}!`); } catch (error) { console.error("๐Ÿ”ด Fatal exception inside getMediaExportList operations engine:", error); } } async function resetReady(userID) { const pool = await sql.connect(config); const myproc = new sql.Request(pool); myproc.input("userID", sql.Int, userID); await myproc.execute("setExportDone"); } async function setExportStarted(userID) { try { const pool = await sql.connect(config); const myproc = new sql.Request(pool); // Explicitly update your active status using lowercase database column conventions // Note: If you already have a custom PL/pgSQL function or direct query here, ensure it sets exportstarted to 1 myproc.input("userID", sql.Int, parseInt(userID, 10)); await myproc.query("UPDATE usertbl SET exportstarted = 1 WHERE userid = @userID"); console.log(`๐Ÿ”’ Database State Lock: ExportStarted flag turned ON for User ${userID}`); } catch (error) { console.error("๐Ÿ”ด Error inside setExportStarted background state locker:", error); } } async function createMediaZip(media, userID) { if (!media || media.length === 0) { console.log("โš ๏ธ createMediaZip: No media attachments discovered for compression pass."); return; } const zipFilePath = dirPath + "/" + userID + "_media.zip"; const streamDict = {}; console.log(`โšก Initializing stable sequential cloud download for ${media.length} Cloudflare R2 assets...`); let counter = 1; for (const item of media) { const blobName = item.MediaLoc || item.medialoc || item.Media_Loc; if (!blobName) continue; try { if (counter % 25 === 0 || counter === media.length) { console.log(`๐Ÿ“ฅ Downloader Progress: Processing asset connection [${counter} / ${media.length}] -> ${blobName}`); } counter++; const response = await s3Client.send( new GetObjectCommand({ Bucket: process.env.R2_BUCKET_NAME, Key: blobName, }) ); // Read the incoming cloud network data chunks into an array natively const chunks = []; for await (const chunk of response.Body) { chunks.push(chunk); } // Flatten the chunks into one clean, finished file Buffer const fileBuffer = Buffer.concat(chunks); // Convert the completed buffer back into a stream that your zip utility can process instantly! streamDict[blobName] = Readable.from(fileBuffer); } catch (err) { console.error(`๐Ÿ”ด Failed to securely retrieve storage blob ${blobName} from Cloudflare R2:`, err.message); } } console.log(`๐Ÿ“ฆ All cloud transfers complete! Passing file streams into ZIP compression matrix...`); // Hand the collected streams over to your automated archiving compressor stream utilities await streamsToCompressed(streamDict, zipFilePath); console.log(`๐Ÿ’พ Media Archive ZIP Package Written Successfully to local disk: ${zipFilePath}`); } async function streamsToCompressed(streamDict, outputFilePath) { return new Promise((resolve, reject) => { const fs = require("fs"); const archiver = require("archiver"); // create a file to stream archive data to. // In case you want to directly stream output in http response of express, just grab 'res' in that case instead of creating file stream const output = fs.createWriteStream(outputFilePath); const archive = archiver("zip", { zlib: { level: 9 }, // Sets the compression level. }); // listen for all archive data to be written // 'close' event is fired only when a file descriptor is involved output.on("close", () => {}); // good practice to catch warnings (ie stat failures and other non-blocking errors) archive.on("warning", (err) => { if (err.code === "ENOENT") { // log warning } else { // throw error throw err; } }); // good practice to catch this error explicitly archive.on("error", (err) => { throw err; }); // pipe archive data to the file archive.pipe(output); for (const blobName in streamDict) { const readableStream = streamDict[blobName]; // finalize the archive (ie we are done appending files but streams have to finish yet) archive.append(readableStream, { name: blobName }); readableStream.on("error", reject); } archive.finalize(); resolve(); }); } async function makeHtml(userID) { try { const pool = await sql.connect(config); const myproc = new sql.Request(pool); // 1. Defensive parameter casting prevents parsing freezes const cleanUserID = parseInt(userID, 10); myproc.input("userID", sql.Int, cleanUserID); console.log(`๐Ÿ“ก Sending "getExport" data request to Supabase for User ID: ${cleanUserID}...`); // 2. Wrap the execution in an explicit performance tracker const stories = await myproc.execute("getExport"); console.log("๐Ÿ“ Raw response received from getExport execution pass:", stories); const storyList = (stories && (stories.recordset || stories.rows)) ? (stories.recordset || stories.rows) : (Array.isArray(stories) ? stories : []); console.log(`๐Ÿ“š Found ${storyList.length} stories to compile for User ${cleanUserID}. Starting cloud-optimized loop...`); let htmlContent = ""; // 3. High-efficiency sequential assembly loop for (const story of storyList) { if (!story) continue; const newStory = await buildStorySection(story); if (newStory) { htmlContent = htmlContent + newStory; } } // 4. Document Construction and Local Disk Stream Flush const htmlCreateContent = "

Memories

" + htmlContent + ""; const htmlName = dirPath + "/" + cleanUserID + "_memories.html"; await writeExportHtml(htmlName, htmlCreateContent); console.log(`๐Ÿ’พ HTML Export File Written Successfully: ${htmlName}`); } catch (error) { console.error("๐Ÿ”ด Fatal exception inside sequential makeHtml generator:", error); } return userID; } async function writeExportHtml(uniqueFile, htmlCreateContent) { fs.writeFile(uniqueFile, htmlCreateContent, (error) => { console.log("Error on html page: ", error); }); } // async function exportMedia(storyID) { // let mediaContent = ""; // try { // const pool = await sql.connect(config); // const myproc2 = new sql.Request(pool); // myproc2.input("storyID", sql.Int, storyID); // await myproc2.execute("getMedia").then((media) => { // media.recordset.forEach((img) => { // mediaContent = // mediaContent + // '
media' + // "
"; // }); // }); // } catch (error) { // console.log("error is: ", error); // } // return mediaContent; // } async function getExportDate(userID) { try { const pool = await sql.connect(config); const myproc = new sql.Request(pool); myproc.input("userID", sql.Int, userID); const exportDate = await myproc.execute("getExportDate"); return exportDate.recordset[0]; } catch (error) { console.log(error); } } async function checkExportStarted(userID) { try { const pool = await sql.connect(config); const myproc = new sql.Request(pool); myproc.input("userID", sql.Int, userID); const startedReturn = await myproc.execute("checkExportStarted"); return startedReturn.recordset[0]; } catch (error) { console.log(error); } } async function checkExportCompleted(userID) { const startedCheck = await checkExportStarted(userID); if (startedCheck.ExportStarted === 0) { return "no files"; } try { const completedPathZip = dirPath + "/" + userID + "_media.zip"; const completedPathHtml = dirPath + "/" + userID + "_memories.html"; if (fs.existsSync(completedPathZip) && fs.existsSync(completedPathHtml)) { return "files ready"; } else { return "no files"; } } catch (error) { console.log(error); } } async function setExportDone(userID) { const pool = await sql.connect(config); const myproc = new sql.Request(pool); myproc.input("userID", sql.Int, userID); await myproc.execute("setExportDone"); } async function cleanUpExports(verify) { const today = new Date(); const dom = today.getDate(); const dow = today.getDay(); const yr = today.getFullYear(); const verifyTest = dom * dow * yr; if (Number(verify) === Number(verifyTest)) { try { //Will delete all export files older than 7 days fs.readdirSync(dirPath).forEach((file) => { const isOlder = fs.statSync(dirPath + "/" + file).ctime < Date.now() - 604800000; // 604800000 = 7 * 24 * 60 * 60 * 1000 if (isOlder) { fs.unlinkSync(dirPath + "/" + file); } }); } catch (err) { console.error(err); } } } // BACKGROUND PROCESSING OF EXPORT let isExportWorkerRunning = false; async function processOfflineExport(userID) { const cleanUserID = parseInt(userID, 10); if (isExportWorkerRunning) { console.log(`๐Ÿ›ก๏ธ Concurrency Guard: An export is already running for User ${cleanUserID}. Dropping duplicate.`); return; } isExportWorkerRunning = true; console.log(`โš™๏ธ Background Worker Activated: Starting stable disk-cached export for User ${cleanUserID}...`); // โœ… REDIRECTED TO EXTERNAL STORAGE: Bypasses your internal full Mac HD drive entirely! // Points straight to your spacious Crucial X9 external drive partition cache space. const externalDriveCachePath = "/Volumes/Crucial X9/code/M4U_BackEnd/server/temp_scratch"; // Establish the dedicated scratch folders safely on the external drive layout if (!fs.existsSync(externalDriveCachePath)) { fs.mkdirSync(externalDriveCachePath, { recursive: true }); } const localZipPath = path.join(externalDriveCachePath, `m4u_export_${cleanUserID}_temp.zip`); const tempStagingDir = path.join(externalDriveCachePath, `m4u_staging_${cleanUserID}`); const exportKey = `temp-exports/${cleanUserID}_archive_${Date.now()}.zip`; try { const pool = await sql.connect(config); // 1. PRE-CLEANUP GUARD: Securely unpack table rows using case-insensitive fallbacks console.log(new Date().toLocaleTimeString(), "step 1"); const checkFileReq = new sql.Request(pool); checkFileReq.input("userID", sql.Int, cleanUserID); const checkFileProc = await checkFileReq.query("SELECT export_r2_key FROM usertbl WHERE userid = @userID"); const fileRows = checkFileProc.recordset || checkFileProc.rows || checkFileProc || []; const firstRow = Array.isArray(fileRows) && fileRows.length > 0 ? fileRows[0] : (fileRows || {}); const oldKey = firstRow.export_r2_key || firstRow.ExportR2Key || null; if (oldKey) { console.log(`๐Ÿงน Pre-Cleanup Guard: Purging old backup archive "${oldKey}" from R2...`); try { await s3Client.send(new DeleteObjectCommand({ Bucket: process.env.R2_BUCKET_NAME, Key: oldKey })); } catch (purgeErr) { console.warn(`โš ๏ธ Pre-Cleanup Warning: ${purgeErr.message}`); } } // 2. Establish a fresh local temporary staging directory folder on your disk console.log(new Date().toLocaleTimeString(), "step 2"); if (!fs.existsSync(tempStagingDir)) { fs.mkdirSync(tempStagingDir, { recursive: true }); } // 3. Mark status as 'processing' in the database immediately using your custom request syntax console.log(new Date().toLocaleTimeString(), "step 3"); const statusReq = new sql.Request(pool); statusReq.input("userID", sql.Int, cleanUserID); await statusReq.query("UPDATE usertbl SET export_status = 'processing', exportstarted = 1 WHERE userid = @userID"); // 4. Extract attachment metadata from your working database procedure console.log(new Date().toLocaleTimeString(), "step 4"); const mediaProc = new sql.Request(pool); mediaProc.input("userID", sql.Int, cleanUserID); const mediaResult = await mediaProc.execute("getExportMedia"); let mediaList = []; if (mediaResult) { if (Array.isArray(mediaResult.recordset)) mediaList = mediaResult.recordset; else if (Array.isArray(mediaResult.rows)) mediaList = mediaResult.rows; else if (Array.isArray(mediaResult)) mediaList = mediaResult; } console.log(`๐Ÿ“ฅ Step 1: Streaming ${mediaList.length} media assets from Cloudflare R2 straight to local disk cache...`); let mediaCounter = 1; for (const item of mediaList) { if (!item) continue; const blobName = item.MediaLoc || item.medialoc || item.Media_Loc; if (!blobName || typeof blobName !== 'string' || blobName.trim() === '') { mediaCounter++; continue; } try { const response = await s3Client.send( new GetObjectCommand({ Bucket: process.env.R2_BUCKET_NAME, Key: blobName.trim() }) ); // Transform the cloud stream natively into a clean Byte Array instantly const fileUint8Array = await response.Body.transformToByteArray(); const fileBuffer = Buffer.from(fileUint8Array); const destinationPath = path.join(tempStagingDir, path.basename(blobName.trim())); // Write the file directly to your disk cache folder (Bypasses RAM overhead entirely!) fs.writeFileSync(destinationPath, fileBuffer); if (mediaCounter % 50 === 0 || mediaCounter === mediaList.length) { console.log(`.. Disk Cache Progress: [${mediaCounter} / ${mediaList.length}] assets written to disk successfully.`); } mediaCounter++; } catch (mediaErr) { console.error(`๐Ÿ”ด Failed to cache asset "${blobName}" to disk:`, mediaErr.message); mediaCounter++; } } // 5. Gather family story text files via your active getExport function console.log(new Date().toLocaleTimeString(), "step 5"); console.log("๐Ÿ“ Step 2: Compiling family narrative memories HTML booklet..."); const storyProc = new sql.Request(pool); storyProc.input("userID", sql.Int, cleanUserID); const storyResult = await storyProc.execute("getExport"); const storyList = storyResult.recordset || storyResult.rows || []; let htmlContent = ""; for (const story of storyList) { if (!story) continue; const newStory = await buildStorySection(story); if (newStory) { htmlContent = htmlContent + newStory; } } // RE-DESIGNED BRANDED HTML MATRIX FRAME const htmlCreateContent = ` Memories For Us - Family Archive

Memories For Us

Your Permanent Personal Narrative Archive

${htmlContent}
`; // 6. Setup the High-Performance Local Zip Write Stream console.log(new Date().toLocaleTimeString(), "step 6"); console.log("๐Ÿ“ฆ Step 3: Packing disk cache folder into a single consolidated archive zip file..."); const outputStream = fs.createWriteStream(localZipPath); const archive = archiver("zip", { zlib: { level: 4 } }); archive.pipe(outputStream); archive.directory(tempStagingDir, "media"); archive.append(htmlCreateContent, { name: "memories.html" }); const streamClosePromise = new Promise((resolve, reject) => { outputStream.on("close", resolve); outputStream.on("error", reject); }); await archive.finalize(); await streamClosePromise; console.log(`๐Ÿš€ Step 4: Local compilation successful! Uploading single file up to Cloudflare R2...`); console.log(new Date().toLocaleTimeString(), "step 7"); // 7. Stream the finished consolidated zip file up to R2 in one clean operation const fileStream = fs.createReadStream(localZipPath); const parallelUpload = new Upload({ client: s3Client, params: { Bucket: process.env.R2_BUCKET_NAME, Key: exportKey, Body: fileStream, ContentType: "application/zip", }, }); try { await parallelUpload.done(); console.log("โ˜๏ธ Cloud storage write confirmation verified by Cloudflare R2!"); } catch (r2UploadErr) { throw new Error(`Cloudflare R2 Upload Stream Failed: \${r2UploadErr.message}`); } // 8. Success Path: Set up a 7-day download expiration window (7 * 24h) and update statuses const expiresAt = new Date(Date.now() + (7 * 24 * 60 * 60 * 1000)); const expiresIsoString = expiresAt.toISOString(); console.log(new Date().toLocaleTimeString(), "step 8 - Bypassing client shim translator loop..."); // Execute direct raw SQL update query to bypass parameter translation errors inside db.js! const rawSuccessPool = await sql.connect(config); const directQueryReq = new sql.Request(rawSuccessPool); await directQueryReq.query(` UPDATE usertbl SET export_status = 'ready', export_r2_key = '${exportKey.trim()}', export_expires_at = '${expiresIsoString}'::timestamp, exportstarted = 0 WHERE userid = ${cleanUserID} `); console.log(`๐ŸŽ‰ Background Export Complete for User ${cleanUserID}! Key saved: ${exportKey}`); } catch (workerError) { console.error(`๐Ÿ”ด Fatal Error inside background export worker loop for User ${cleanUserID}:`, workerError.message); try { const pool = await sql.connect(config); const failReq = new sql.Request(pool); await failReq.query(`UPDATE usertbl SET export_status = 'failed', exportstarted = 0 WHERE userid = ${cleanUserID}`); console.log(`๐Ÿ”’ Recovery complete: Status rolled back to 'failed' for User ${cleanUserID}`); } catch (dbErr) { console.error("๐Ÿ”ด Failed to write recovery status to database:", dbErr.message); } } finally { // 9. HOUSEKEEPING CLEANUP: Purge temporary scratch folder and archive file from local disk space instantly! console.log("Core Housekeeping: Initializing local disk cleanup..."); try { if (fs.existsSync(tempStagingDir)) { fs.rmSync(tempStagingDir, { recursive: true, force: true }); } if (fs.existsSync(localZipPath)) { fs.unlinkSync(localZipPath); } console.log("โœจ Housekeeping cleanup completed successfully. Disk space cleared."); } catch (cleanErr) { console.warn("โš ๏ธ Minor local cleanup warning:", cleanErr.message); } isExportWorkerRunning = false; } } // ========================================== // ๐Ÿ–ผ๏ธ COMPILER HELPER: BUILD INDIVIDUAL MEMORY CARD MARKUP WITH MEDIA EMBEDS // ========================================== async function buildStorySection(story) { if (!story) return ""; // Unpack columns cleanly supporting case-insensitive database fields const title = story.StoryTitle || story.storytitle || story.Story_Title || "Untitled Memory"; const content = story.StoryContent || story.storycontent || story.StoryText || ""; const dateRecord = story.MemoryDate || story.memorydate || ""; const collectionName = story.CollectionName || story.collectionname || "General"; const mediaFile = story.MediaLoc || story.medialoc || story.Media_Loc || ""; // Format a reader-friendly calendar date text line let displayDate = ""; if (dateRecord) { const cleanDate = new Date(dateRecord); if (!isNaN(cleanDate.getTime())) { displayDate = cleanDate.toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' }); } } // Compile the top metadata tracking bar row let metaMarkup = `
`; if (displayDate) metaMarkup += `${displayDate} • `; metaMarkup += `Collection: ${collectionName}`; metaMarkup += `
`; // Media Attachment Rendering Logic Grid let mediaDisplayMarkup = ""; if (mediaFile && typeof mediaFile === 'string' && mediaFile.trim() !== '') { const cleanFileName = path.basename(mediaFile.trim()); const fileExtension = path.extname(cleanFileName).toLowerCase(); // Route directly into the unzipped local 'media/' subdirectory folder const localPath = `media/${cleanFileName}`; if (['.mp4', '.mov', '.webm', '.ogg'].includes(fileExtension)) { // Sleek HTML5 Offline Local Video Player Layout mediaDisplayMarkup = `
๐Ÿ“Ž Open Video File Directly (${cleanFileName})
`; } else if (['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'].includes(fileExtension)) { // Embedded Photo Element with Full-Screen Direct Access fallback badges mediaDisplayMarkup = `
${title}
๐Ÿ“Ž Open Image File Directly (${cleanFileName})
`; } else { // Standard fallback action badge links for audio files or generic document items mediaDisplayMarkup = `
๐Ÿ“Ž View Attached Asset File (${cleanFileName})
`; } } // Compile individual story nodes into a clean card markup skeleton return `
${metaMarkup}

${title}

${content}
${mediaDisplayMarkup}
`; } // โœ… COMPLETE EXPLICIT MODULE EXPORTS REGISTRY module.exports = { exportMemoriesCheck, createMediaZip, getMediaExportList, makeHtml, getExportDate, setExportStarted, setExportDone, resetReady, cleanUpExports, checkExportCompleted, checkExportStarted, processOfflineExport };