const fs = require("node:fs");
const path = require("node:path");
const sql = require("./db");
const config = require("./dbconfig");
const { Readable } = require('stream');
const os = require("os");
const { S3Client, GetObjectCommand, DeleteObjectCommand } = require("@aws-sdk/client-s3");
const archiver = require("archiver");
const { Upload } = require("@aws-sdk/lib-storage");
const s3Client = new S3Client({
region: "auto",
endpoint: process.env.R2_ENDPOINT,
credentials: {
accessKeyId: process.env.R2_ACCESS_KEY_ID,
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY,
},
});
const dirPath = path.join(__dirname, "/Exports");
// Cleaned up the redundancy: Merged into one single focused operational query method
async function exportMemoriesCheck(userID) {
try {
const pool = await sql.connect(config);
const myproc = new sql.Request(pool);
myproc.input("userID", sql.Int, parseInt(userID, 10));
const result = await myproc.execute("checkExportDate");
// Unpack database table row results safely from your custom database layer
const rawRows = result.recordset || result.rows || result;
// Fixed: Multi-case fallback lookup grabs the binary permission flag regardless of uppercase mutations!
let isAllowed = 0;
if (Array.isArray(rawRows) && rawRows.length > 0) {
const row = rawRows[0];
isAllowed = row.ExportAllowed !== undefined ? row.ExportAllowed : (row.exportallowed || row.EXPORTALLOWED || 0);
} else if (rawRows) {
isAllowed = rawRows.ExportAllowed !== undefined ? rawRows.ExportAllowed : (rawRows.exportallowed || rawRows.EXPORTALLOWED || 0);
}
console.log(`๐ Export Initialization check: User ${userID} permission status flag resolves to: ${isAllowed}`);
return Number(isAllowed); // Returns a clean, guaranteed primitive number 1 or 0 straight to your API route!
} catch (error) {
console.error("๐ด Error inside exportMemoriesCheck operation handler:", error);
throw error;
}
}
async function getMediaExportList(userID) {
try {
// 1. Lock the active background processing flag
await setExportStarted(userID);
const pool = await sql.connect(config);
const myproc = new sql.Request(pool);
const cleanUserID = parseInt(userID, 10);
myproc.input("userID", sql.Int, cleanUserID);
console.log(`๐ก Sending "getExportMedia" link query to Supabase for User ID: ${cleanUserID}...`);
const media = await myproc.execute("getExportMedia");
// 2. Safely extract media row arrays regardless of nested properties
const rawRows = media.recordset || media.rows || media;
const mediaList = Array.isArray(rawRows) ? rawRows : (rawRows.recordset ? rawRows.recordset : []);
console.log(`๐ฆ Found ${mediaList.length} attachment links. Initializing "createMediaZip" file compressor loop...`);
// 3. Pass clean rows to your automated archive bundling microservice
await createMediaZip(mediaList, cleanUserID);
console.log(`โ๏ธ "createMediaZip" compression pipeline completed for User ${cleanUserID}!`);
} catch (error) {
console.error("๐ด Fatal exception inside getMediaExportList operations engine:", error);
}
}
async function resetReady(userID) {
const pool = await sql.connect(config);
const myproc = new sql.Request(pool);
myproc.input("userID", sql.Int, userID);
await myproc.execute("setExportDone");
}
async function setExportStarted(userID) {
try {
const pool = await sql.connect(config);
const myproc = new sql.Request(pool);
// Explicitly update your active status using lowercase database column conventions
// Note: If you already have a custom PL/pgSQL function or direct query here, ensure it sets exportstarted to 1
myproc.input("userID", sql.Int, parseInt(userID, 10));
await myproc.query("UPDATE usertbl SET exportstarted = 1 WHERE userid = @userID");
console.log(`๐ Database State Lock: ExportStarted flag turned ON for User ${userID}`);
} catch (error) {
console.error("๐ด Error inside setExportStarted background state locker:", error);
}
}
async function createMediaZip(media, userID) {
if (!media || media.length === 0) {
console.log("โ ๏ธ createMediaZip: No media attachments discovered for compression pass.");
return;
}
const zipFilePath = dirPath + "/" + userID + "_media.zip";
const streamDict = {};
console.log(`โก Initializing stable sequential cloud download for ${media.length} Cloudflare R2 assets...`);
let counter = 1;
for (const item of media) {
const blobName = item.MediaLoc || item.medialoc || item.Media_Loc;
if (!blobName) continue;
try {
if (counter % 25 === 0 || counter === media.length) {
console.log(`๐ฅ Downloader Progress: Processing asset connection [${counter} / ${media.length}] -> ${blobName}`);
}
counter++;
const response = await s3Client.send(
new GetObjectCommand({
Bucket: process.env.R2_BUCKET_NAME,
Key: blobName,
})
);
// Read the incoming cloud network data chunks into an array natively
const chunks = [];
for await (const chunk of response.Body) {
chunks.push(chunk);
}
// Flatten the chunks into one clean, finished file Buffer
const fileBuffer = Buffer.concat(chunks);
// Convert the completed buffer back into a stream that your zip utility can process instantly!
streamDict[blobName] = Readable.from(fileBuffer);
} catch (err) {
console.error(`๐ด Failed to securely retrieve storage blob ${blobName} from Cloudflare R2:`, err.message);
}
}
console.log(`๐ฆ All cloud transfers complete! Passing file streams into ZIP compression matrix...`);
// Hand the collected streams over to your automated archiving compressor stream utilities
await streamsToCompressed(streamDict, zipFilePath);
console.log(`๐พ Media Archive ZIP Package Written Successfully to local disk: ${zipFilePath}`);
}
async function streamsToCompressed(streamDict, outputFilePath) {
return new Promise((resolve, reject) => {
const fs = require("fs");
const archiver = require("archiver");
// create a file to stream archive data to.
// In case you want to directly stream output in http response of express, just grab 'res' in that case instead of creating file stream
const output = fs.createWriteStream(outputFilePath);
const archive = archiver("zip", {
zlib: { level: 9 }, // Sets the compression level.
});
// listen for all archive data to be written
// 'close' event is fired only when a file descriptor is involved
output.on("close", () => {});
// good practice to catch warnings (ie stat failures and other non-blocking errors)
archive.on("warning", (err) => {
if (err.code === "ENOENT") {
// log warning
} else {
// throw error
throw err;
}
});
// good practice to catch this error explicitly
archive.on("error", (err) => {
throw err;
});
// pipe archive data to the file
archive.pipe(output);
for (const blobName in streamDict) {
const readableStream = streamDict[blobName];
// finalize the archive (ie we are done appending files but streams have to finish yet)
archive.append(readableStream, { name: blobName });
readableStream.on("error", reject);
}
archive.finalize();
resolve();
});
}
async function makeHtml(userID) {
try {
const pool = await sql.connect(config);
const myproc = new sql.Request(pool);
// 1. Defensive parameter casting prevents parsing freezes
const cleanUserID = parseInt(userID, 10);
myproc.input("userID", sql.Int, cleanUserID);
console.log(`๐ก Sending "getExport" data request to Supabase for User ID: ${cleanUserID}...`);
// 2. Wrap the execution in an explicit performance tracker
const stories = await myproc.execute("getExport");
console.log("๐ Raw response received from getExport execution pass:", stories);
const storyList = (stories && (stories.recordset || stories.rows)) ? (stories.recordset || stories.rows) : (Array.isArray(stories) ? stories : []);
console.log(`๐ Found ${storyList.length} stories to compile for User ${cleanUserID}. Starting cloud-optimized loop...`);
let htmlContent = "";
// 3. High-efficiency sequential assembly loop
for (const story of storyList) {
if (!story) continue;
const newStory = await buildStorySection(story);
if (newStory) {
htmlContent = htmlContent + newStory;
}
}
// 4. Document Construction and Local Disk Stream Flush
const htmlCreateContent = "
Memories
" + htmlContent + "";
const htmlName = dirPath + "/" + cleanUserID + "_memories.html";
await writeExportHtml(htmlName, htmlCreateContent);
console.log(`๐พ HTML Export File Written Successfully: ${htmlName}`);
} catch (error) {
console.error("๐ด Fatal exception inside sequential makeHtml generator:", error);
}
return userID;
}
async function writeExportHtml(uniqueFile, htmlCreateContent) {
fs.writeFile(uniqueFile, htmlCreateContent, (error) => {
console.log("Error on html page: ", error);
});
}
// async function exportMedia(storyID) {
// let mediaContent = "";
// try {
// const pool = await sql.connect(config);
// const myproc2 = new sql.Request(pool);
// myproc2.input("storyID", sql.Int, storyID);
// await myproc2.execute("getMedia").then((media) => {
// media.recordset.forEach((img) => {
// mediaContent =
// mediaContent +
// '";
// });
// });
// } catch (error) {
// console.log("error is: ", error);
// }
// return mediaContent;
// }
async function getExportDate(userID) {
try {
const pool = await sql.connect(config);
const myproc = new sql.Request(pool);
myproc.input("userID", sql.Int, userID);
const exportDate = await myproc.execute("getExportDate");
return exportDate.recordset[0];
} catch (error) {
console.log(error);
}
}
async function checkExportStarted(userID) {
try {
const pool = await sql.connect(config);
const myproc = new sql.Request(pool);
myproc.input("userID", sql.Int, userID);
const startedReturn = await myproc.execute("checkExportStarted");
return startedReturn.recordset[0];
} catch (error) {
console.log(error);
}
}
async function checkExportCompleted(userID) {
const startedCheck = await checkExportStarted(userID);
if (startedCheck.ExportStarted === 0) {
return "no files";
}
try {
const completedPathZip = dirPath + "/" + userID + "_media.zip";
const completedPathHtml = dirPath + "/" + userID + "_memories.html";
if (fs.existsSync(completedPathZip) && fs.existsSync(completedPathHtml)) {
return "files ready";
} else {
return "no files";
}
} catch (error) {
console.log(error);
}
}
async function setExportDone(userID) {
const pool = await sql.connect(config);
const myproc = new sql.Request(pool);
myproc.input("userID", sql.Int, userID);
await myproc.execute("setExportDone");
}
async function cleanUpExports(verify) {
const today = new Date();
const dom = today.getDate();
const dow = today.getDay();
const yr = today.getFullYear();
const verifyTest = dom * dow * yr;
if (Number(verify) === Number(verifyTest)) {
try {
//Will delete all export files older than 7 days
fs.readdirSync(dirPath).forEach((file) => {
const isOlder =
fs.statSync(dirPath + "/" + file).ctime < Date.now() - 604800000; // 604800000 = 7 * 24 * 60 * 60 * 1000
if (isOlder) {
fs.unlinkSync(dirPath + "/" + file);
}
});
} catch (err) {
console.error(err);
}
}
}
// BACKGROUND PROCESSING OF EXPORT
let isExportWorkerRunning = false;
async function processOfflineExport(userID) {
const cleanUserID = parseInt(userID, 10);
if (isExportWorkerRunning) {
console.log(`๐ก๏ธ Concurrency Guard: An export is already running for User ${cleanUserID}. Dropping duplicate.`);
return;
}
isExportWorkerRunning = true;
console.log(`โ๏ธ Background Worker Activated: Starting stable disk-cached export for User ${cleanUserID}...`);
// โ
REDIRECTED TO EXTERNAL STORAGE: Bypasses your internal full Mac HD drive entirely!
// Points straight to your spacious Crucial X9 external drive partition cache space.
const externalDriveCachePath = "/Volumes/Crucial X9/code/M4U_BackEnd/server/temp_scratch";
// Establish the dedicated scratch folders safely on the external drive layout
if (!fs.existsSync(externalDriveCachePath)) {
fs.mkdirSync(externalDriveCachePath, { recursive: true });
}
const localZipPath = path.join(externalDriveCachePath, `m4u_export_${cleanUserID}_temp.zip`);
const tempStagingDir = path.join(externalDriveCachePath, `m4u_staging_${cleanUserID}`);
const exportKey = `temp-exports/${cleanUserID}_archive_${Date.now()}.zip`;
try {
const pool = await sql.connect(config);
// 1. PRE-CLEANUP GUARD: Securely unpack table rows using case-insensitive fallbacks
console.log(new Date().toLocaleTimeString(), "step 1");
const checkFileReq = new sql.Request(pool);
checkFileReq.input("userID", sql.Int, cleanUserID);
const checkFileProc = await checkFileReq.query("SELECT export_r2_key FROM usertbl WHERE userid = @userID");
const fileRows = checkFileProc.recordset || checkFileProc.rows || checkFileProc || [];
const firstRow = Array.isArray(fileRows) && fileRows.length > 0 ? fileRows[0] : (fileRows || {});
const oldKey = firstRow.export_r2_key || firstRow.ExportR2Key || null;
if (oldKey) {
console.log(`๐งน Pre-Cleanup Guard: Purging old backup archive "${oldKey}" from R2...`);
try {
await s3Client.send(new DeleteObjectCommand({ Bucket: process.env.R2_BUCKET_NAME, Key: oldKey }));
} catch (purgeErr) {
console.warn(`โ ๏ธ Pre-Cleanup Warning: ${purgeErr.message}`);
}
}
// 2. Establish a fresh local temporary staging directory folder on your disk
console.log(new Date().toLocaleTimeString(), "step 2");
if (!fs.existsSync(tempStagingDir)) {
fs.mkdirSync(tempStagingDir, { recursive: true });
}
// 3. Mark status as 'processing' in the database immediately using your custom request syntax
console.log(new Date().toLocaleTimeString(), "step 3");
const statusReq = new sql.Request(pool);
statusReq.input("userID", sql.Int, cleanUserID);
await statusReq.query("UPDATE usertbl SET export_status = 'processing', exportstarted = 1 WHERE userid = @userID");
// 4. Extract attachment metadata from your working database procedure
console.log(new Date().toLocaleTimeString(), "step 4");
const mediaProc = new sql.Request(pool);
mediaProc.input("userID", sql.Int, cleanUserID);
const mediaResult = await mediaProc.execute("getExportMedia");
let mediaList = [];
if (mediaResult) {
if (Array.isArray(mediaResult.recordset)) mediaList = mediaResult.recordset;
else if (Array.isArray(mediaResult.rows)) mediaList = mediaResult.rows;
else if (Array.isArray(mediaResult)) mediaList = mediaResult;
}
console.log(`๐ฅ Step 1: Streaming ${mediaList.length} media assets from Cloudflare R2 straight to local disk cache...`);
let mediaCounter = 1;
for (const item of mediaList) {
if (!item) continue;
const blobName = item.MediaLoc || item.medialoc || item.Media_Loc;
if (!blobName || typeof blobName !== 'string' || blobName.trim() === '') {
mediaCounter++;
continue;
}
try {
const response = await s3Client.send(
new GetObjectCommand({ Bucket: process.env.R2_BUCKET_NAME, Key: blobName.trim() })
);
// Transform the cloud stream natively into a clean Byte Array instantly
const fileUint8Array = await response.Body.transformToByteArray();
const fileBuffer = Buffer.from(fileUint8Array);
const destinationPath = path.join(tempStagingDir, path.basename(blobName.trim()));
// Write the file directly to your disk cache folder (Bypasses RAM overhead entirely!)
fs.writeFileSync(destinationPath, fileBuffer);
if (mediaCounter % 50 === 0 || mediaCounter === mediaList.length) {
console.log(`.. Disk Cache Progress: [${mediaCounter} / ${mediaList.length}] assets written to disk successfully.`);
}
mediaCounter++;
} catch (mediaErr) {
console.error(`๐ด Failed to cache asset "${blobName}" to disk:`, mediaErr.message);
mediaCounter++;
}
}
// 5. Gather family story text files via your active getExport function
console.log(new Date().toLocaleTimeString(), "step 5");
console.log("๐ Step 2: Compiling family narrative memories HTML booklet...");
const storyProc = new sql.Request(pool);
storyProc.input("userID", sql.Int, cleanUserID);
const storyResult = await storyProc.execute("getExport");
const storyList = storyResult.recordset || storyResult.rows || [];
let htmlContent = "";
for (const story of storyList) {
if (!story) continue;
const newStory = await buildStorySection(story);
if (newStory) {
htmlContent = htmlContent + newStory;
}
}
// RE-DESIGNED BRANDED HTML MATRIX FRAME
const htmlCreateContent = `
Memories For Us - Family Archive
`;
// 6. Setup the High-Performance Local Zip Write Stream
console.log(new Date().toLocaleTimeString(), "step 6");
console.log("๐ฆ Step 3: Packing disk cache folder into a single consolidated archive zip file...");
const outputStream = fs.createWriteStream(localZipPath);
const archive = archiver("zip", { zlib: { level: 4 } });
archive.pipe(outputStream);
archive.directory(tempStagingDir, "media");
archive.append(htmlCreateContent, { name: "memories.html" });
const streamClosePromise = new Promise((resolve, reject) => {
outputStream.on("close", resolve);
outputStream.on("error", reject);
});
await archive.finalize();
await streamClosePromise;
console.log(`๐ Step 4: Local compilation successful! Uploading single file up to Cloudflare R2...`);
console.log(new Date().toLocaleTimeString(), "step 7");
// 7. Stream the finished consolidated zip file up to R2 in one clean operation
const fileStream = fs.createReadStream(localZipPath);
const parallelUpload = new Upload({
client: s3Client,
params: {
Bucket: process.env.R2_BUCKET_NAME,
Key: exportKey,
Body: fileStream,
ContentType: "application/zip",
},
});
try {
await parallelUpload.done();
console.log("โ๏ธ Cloud storage write confirmation verified by Cloudflare R2!");
} catch (r2UploadErr) {
throw new Error(`Cloudflare R2 Upload Stream Failed: \${r2UploadErr.message}`);
}
// 8. Success Path: Set up a 7-day download expiration window (7 * 24h) and update statuses
const expiresAt = new Date(Date.now() + (7 * 24 * 60 * 60 * 1000));
const expiresIsoString = expiresAt.toISOString();
console.log(new Date().toLocaleTimeString(), "step 8 - Bypassing client shim translator loop...");
// Execute direct raw SQL update query to bypass parameter translation errors inside db.js!
const rawSuccessPool = await sql.connect(config);
const directQueryReq = new sql.Request(rawSuccessPool);
await directQueryReq.query(`
UPDATE usertbl
SET export_status = 'ready',
export_r2_key = '${exportKey.trim()}',
export_expires_at = '${expiresIsoString}'::timestamp,
exportstarted = 0
WHERE userid = ${cleanUserID}
`);
console.log(`๐ Background Export Complete for User ${cleanUserID}! Key saved: ${exportKey}`);
} catch (workerError) {
console.error(`๐ด Fatal Error inside background export worker loop for User ${cleanUserID}:`, workerError.message);
try {
const pool = await sql.connect(config);
const failReq = new sql.Request(pool);
await failReq.query(`UPDATE usertbl SET export_status = 'failed', exportstarted = 0 WHERE userid = ${cleanUserID}`);
console.log(`๐ Recovery complete: Status rolled back to 'failed' for User ${cleanUserID}`);
} catch (dbErr) {
console.error("๐ด Failed to write recovery status to database:", dbErr.message);
}
} finally {
// 9. HOUSEKEEPING CLEANUP: Purge temporary scratch folder and archive file from local disk space instantly!
console.log("Core Housekeeping: Initializing local disk cleanup...");
try {
if (fs.existsSync(tempStagingDir)) {
fs.rmSync(tempStagingDir, { recursive: true, force: true });
}
if (fs.existsSync(localZipPath)) {
fs.unlinkSync(localZipPath);
}
console.log("โจ Housekeeping cleanup completed successfully. Disk space cleared.");
} catch (cleanErr) {
console.warn("โ ๏ธ Minor local cleanup warning:", cleanErr.message);
}
isExportWorkerRunning = false;
}
}
// ==========================================
// ๐ผ๏ธ COMPILER HELPER: BUILD INDIVIDUAL MEMORY CARD MARKUP WITH MEDIA EMBEDS
// ==========================================
async function buildStorySection(story) {
if (!story) return "";
// Unpack columns cleanly supporting case-insensitive database fields
const title = story.StoryTitle || story.storytitle || story.Story_Title || "Untitled Memory";
const content = story.StoryContent || story.storycontent || story.StoryText || "";
const dateRecord = story.MemoryDate || story.memorydate || "";
const collectionName = story.CollectionName || story.collectionname || "General";
const mediaFile = story.MediaLoc || story.medialoc || story.Media_Loc || "";
// Format a reader-friendly calendar date text line
let displayDate = "";
if (dateRecord) {
const cleanDate = new Date(dateRecord);
if (!isNaN(cleanDate.getTime())) {
displayDate = cleanDate.toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric' });
}
}
// Compile the top metadata tracking bar row
let metaMarkup = ``;
if (displayDate) metaMarkup += `${displayDate} • `;
metaMarkup += `Collection: ${collectionName}`;
metaMarkup += `
`;
// Media Attachment Rendering Logic Grid
let mediaDisplayMarkup = "";
if (mediaFile && typeof mediaFile === 'string' && mediaFile.trim() !== '') {
const cleanFileName = path.basename(mediaFile.trim());
const fileExtension = path.extname(cleanFileName).toLowerCase();
// Route directly into the unzipped local 'media/' subdirectory folder
const localPath = `media/${cleanFileName}`;
if (['.mp4', '.mov', '.webm', '.ogg'].includes(fileExtension)) {
// Sleek HTML5 Offline Local Video Player Layout
mediaDisplayMarkup = `
`;
} else if (['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'].includes(fileExtension)) {
// Embedded Photo Element with Full-Screen Direct Access fallback badges
mediaDisplayMarkup = `
`;
} else {
// Standard fallback action badge links for audio files or generic document items
mediaDisplayMarkup = `
`;
}
}
// Compile individual story nodes into a clean card markup skeleton
return `
${metaMarkup}
${title}
${content}
${mediaDisplayMarkup}
`;
}
// โ
COMPLETE EXPLICIT MODULE EXPORTS REGISTRY
module.exports = {
exportMemoriesCheck,
createMediaZip,
getMediaExportList,
makeHtml,
getExportDate,
setExportStarted,
setExportDone,
resetReady,
cleanUpExports,
checkExportCompleted,
checkExportStarted,
processOfflineExport
};