More stability & defaults

src/ffmpegConfig.js

@@ -0,0 +1,102 @@
+const { createPlaylist } = require('./musicPlaylist');
+
+/**
+ * Build FFmpeg arguments for HLS streaming
+ * @param {Object} options - Configuration options
+ * @param {number} options.fps - Frames per second
+ * @param {boolean} options.useMusic - Whether to include audio from music files
+ * @param {string} options.musicPath - Path to music directory
+ * @returns {Promise<{args: string[], playlistFile: string|null, hasMusic: boolean}>}
+ */
+async function buildFFmpegArgs({ fps, useMusic, musicPath }) {
+  const ffmpegArgs = [];
+  let playlistFile = null;
+  let hasMusic = false;
+
+  if (useMusic) {
+    const playlistInfo = createPlaylist(musicPath);
+    
+    if (playlistInfo) {
+      playlistFile = playlistInfo.playlistFile;
+      hasMusic = true;
+
+      // Input 0: video frames
+      ffmpegArgs.push(
+        '-use_wallclock_as_timestamps', '1',
+        '-f', 'image2pipe',
+        '-framerate', fps.toString(),
+        '-i', 'pipe:0'
+      );
+
+      // Input 1: audio from concat playlist
+      ffmpegArgs.push(
+        '-f', 'concat',
+        '-safe', '0',
+        '-i', playlistFile
+      );
+
+      // Encoding with audio filtering for smooth transitions
+      ffmpegArgs.push(
+        // Use audio filter to ensure smooth transitions and consistent format
+        '-af', 'aresample=async=1:min_hard_comp=0.100000:first_pts=0,aformat=sample_rates=44100:channel_layouts=stereo',
+        '-c:v', 'libx264',
+        '-preset', 'ultrafast',
+        '-tune', 'zerolatency',
+        '-pix_fmt', 'yuv420p',
+        '-g', fps.toString(),         // Keyframe every second for 1s segments
+        '-bf', '0',                   // No B-frames for lower latency
+        '-x264opts', 'nal-hrd=cbr:no-scenecut',  // Constant bitrate, no scene detection
+        '-b:v', '2500k',              // Target bitrate for stable encoding
+        '-maxrate', '2500k',
+        '-bufsize', '5000k',
+        '-c:a', 'aac',
+        '-b:a', '128k',
+        '-ar', '44100',               // Set explicit audio sample rate
+        '-ac', '2',                   // Stereo output
+        '-avoid_negative_ts', 'make_zero',  // Prevent timestamp issues
+        '-fflags', '+genpts+igndts',  // Generate presentation timestamps, ignore decode timestamps
+        '-max_interleave_delta', '0', // Reduce audio/video sync issues during transitions
+        '-f', 'hls',
+        '-hls_time', '1',             // Smaller segments for faster startup
+        '-hls_list_size', '3',        // Fewer segments in playlist
+        '-hls_flags', 'delete_segments+omit_endlist',
+        '-hls_start_number_source', 'epoch',
+        '-start_number', '0',         // Start from segment 0
+        '-flush_packets', '1',        // Flush packets immediately
+        'pipe:1'
+      );
+
+      return { args: ffmpegArgs, playlistFile, hasMusic };
+    }
+  }
+
+  // Video only (no music)
+  ffmpegArgs.push(
+    '-use_wallclock_as_timestamps', '1',
+    '-f', 'image2pipe',
+    '-framerate', fps.toString(),
+    '-i', 'pipe:0',
+    '-c:v', 'libx264',
+    '-preset', 'ultrafast',
+    '-tune', 'zerolatency',
+    '-pix_fmt', 'yuv420p',
+    '-g', fps.toString(),         // Keyframe every second for 1s segments
+    '-bf', '0',
+    '-x264opts', 'nal-hrd=cbr:no-scenecut',
+    '-b:v', '2500k',
+    '-maxrate', '2500k',
+    '-bufsize', '5000k',
+    '-f', 'hls',
+    '-hls_time', '1',             // Smaller segments for faster startup
+    '-hls_list_size', '3',        // Fewer segments in playlist
+    '-hls_flags', 'delete_segments+omit_endlist',
+    '-hls_start_number_source', 'epoch',
+    '-start_number', '0',
+    '-flush_packets', '1',        // Flush packets immediately
+    'pipe:1'
+  );
+
+  return { args: ffmpegArgs, playlistFile, hasMusic };
+}
+
+module.exports = { buildFFmpegArgs };

src/geocode.js

@@ -0,0 +1,48 @@
+const https = require('https');
+
+/**
+ * Geocode city to lat/lon using Nominatim (OpenStreetMap)
+ * @param {string} cityQuery - City name to geocode
+ * @returns {Promise<{lat: number, lon: number, displayName: string}>}
+ */
+async function geocodeCity(cityQuery) {
+  return new Promise((resolve, reject) => {
+    const encodedQuery = encodeURIComponent(cityQuery);
+    const url = `https://nominatim.openstreetmap.org/search?q=${encodedQuery}&format=json&limit=1`;
+    
+    const options = {
+      headers: {
+        'User-Agent': 'webpage-to-hls-streaming-app/1.0'
+      }
+    };
+
+    https.get(url, options, (res) => {
+      let data = '';
+      
+      res.on('data', (chunk) => {
+        data += chunk;
+      });
+      
+      res.on('end', () => {
+        try {
+          const results = JSON.parse(data);
+          if (results && results.length > 0) {
+            resolve({
+              lat: parseFloat(results[0].lat),
+              lon: parseFloat(results[0].lon),
+              displayName: results[0].display_name
+            });
+          } else {
+            reject(new Error('No results found'));
+          }
+        } catch (error) {
+          reject(error);
+        }
+      });
+    }).on('error', (error) => {
+      reject(error);
+    });
+  });
+}
+
+module.exports = { geocodeCity };

src/musicPlaylist.js

@@ -0,0 +1,98 @@
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * Get a random music file from the music directory
+ * @param {string} musicPath - Path to music directory
+ * @returns {string|null} Path to random music file or null
+ */
+function getRandomMusicFile(musicPath) {
+  try {
+    if (!fs.existsSync(musicPath)) {
+      return null;
+    }
+    const files = fs.readdirSync(musicPath).filter(f => f.endsWith('.ogg'));
+    if (files.length === 0) {
+      return null;
+    }
+    const randomFile = files[Math.floor(Math.random() * files.length)];
+    return path.join(musicPath, randomFile);
+  } catch (error) {
+    console.error('Error getting music file:', error);
+    return null;
+  }
+}
+
+/**
+ * Get all music files for playlist generation
+ * @param {string} musicPath - Path to music directory
+ * @returns {string[]} Array of music file paths
+ */
+function getAllMusicFiles(musicPath) {
+  try {
+    if (!fs.existsSync(musicPath)) {
+      return [];
+    }
+    const files = fs.readdirSync(musicPath).filter(f => f.endsWith('.ogg'));
+    return files.map(f => path.join(musicPath, f));
+  } catch (error) {
+    console.error('Error getting music files:', error);
+    return [];
+  }
+}
+
+/**
+ * Shuffle array in place using Fisher-Yates algorithm
+ * @param {Array} array - Array to shuffle
+ * @returns {Array} Shuffled array (same reference)
+ */
+function shuffleArray(array) {
+  for (let i = array.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [array[i], array[j]] = [array[j], array[i]];
+  }
+  return array;
+}
+
+/**
+ * Create a playlist file for FFmpeg concat demuxer
+ * @param {string} musicPath - Path to music directory
+ * @returns {{playlistFile: string, trackCount: number}|null} Playlist info or null
+ */
+function createPlaylist(musicPath) {
+  const allMusicFiles = getAllMusicFiles(musicPath);
+  console.log(`Found ${allMusicFiles.length} music files in ${musicPath}`);
+  
+  if (allMusicFiles.length === 0) {
+    return null;
+  }
+
+  // Create a temporary concat playlist file
+  const playlistFile = path.join('/tmp', `playlist-${Date.now()}.txt`);
+  
+  // Build playlist content - repeat enough times for ~24 hours of playback
+  // Assuming avg 3 min per track, repeat enough to cover a full day
+  const repetitions = Math.max(20, Math.ceil(480 / allMusicFiles.length)); // At least 480 tracks (~24hrs)
+  const playlistLines = [];
+  
+  for (let i = 0; i < repetitions; i++) {
+    // Re-shuffle each repetition for more variety
+    const shuffled = shuffleArray([...allMusicFiles]);
+    shuffled.forEach(f => playlistLines.push(`file '${f}'`));
+  }
+  
+  fs.writeFileSync(playlistFile, playlistLines.join('\n'));
+  console.log(`Created playlist with ${allMusicFiles.length} tracks x${repetitions} repetitions (~${playlistLines.length} total tracks)`);
+  
+  return {
+    playlistFile,
+    trackCount: playlistLines.length
+  };
+}
+
+module.exports = {
+  getRandomMusicFile,
+  getAllMusicFiles,
+  shuffleArray,
+  createPlaylist
+};

src/pageLoader.js

@@ -0,0 +1,76 @@
+/**
+ * Setup and configure a Puppeteer page
+ * @param {Browser} browser - Puppeteer browser instance
+ * @param {Object} options - Configuration options
+ * @param {number} options.width - Page width
+ * @param {number} options.height - Page height
+ * @returns {Promise<Page>} Configured Puppeteer page
+ */
+async function setupPage(browser, { width, height }) {
+  const page = await browser.newPage();
+  
+  // Reduce memory usage by disabling caching
+  await page.setCacheEnabled(false);
+  
+  // Inject CSS early to prevent white flash during page load
+  await page.evaluateOnNewDocument(() => {
+    const style = document.createElement('style');
+    style.textContent = `
+      html, body {
+        background-color: #000 !important;
+      }
+    `;
+    document.head?.appendChild(style) || document.documentElement.appendChild(style);
+  });
+
+  return page;
+}
+
+/**
+ * Wait for page to be fully loaded with stylesheet
+ * @param {Page} page - Puppeteer page
+ * @param {string} url - URL to navigate to
+ * @returns {Promise<boolean>} True if page loaded successfully
+ */
+async function waitForPageFullyLoaded(page, url) {
+  try {
+    // Wait for DOM content and stylesheet to load
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
+    console.log('Page DOM loaded, waiting for stylesheet...');
+    
+    // Wait a brief moment for stylesheet to apply
+    await new Promise(resolve => setTimeout(resolve, 500));
+    
+    console.log('Page stylesheet loaded, switching to live frames');
+    return true;
+  } catch (err) {
+    console.error('Page load error:', err.message);
+    // Still show the page even if timeout occurs
+    return true;
+  }
+}
+
+/**
+ * Hide logo elements on the page
+ * @param {Page} page - Puppeteer page
+ */
+async function hideLogo(page) {
+  try {
+    await page.evaluate(() => {
+      const images = document.querySelectorAll('img');
+      images.forEach(img => {
+        if (img.src && img.src.includes('Logo3.png')) {
+          img.style.display = 'none';
+        }
+      });
+    });
+  } catch (err) {
+    console.error('Logo hide error:', err);
+  }
+}
+
+module.exports = {
+  setupPage,
+  waitForPageFullyLoaded,
+  hideLogo
+};

src/streamHandler.js

@@ -0,0 +1,412 @@
+const puppeteer = require('puppeteer');
+const { spawn } = require('child_process');
+const fs = require('fs');
+const { buildFFmpegArgs } = require('./ffmpegConfig');
+const { setupPage, waitForPageFullyLoaded, hideLogo } = require('./pageLoader');
+
+/**
+ * Main streaming handler - captures webpage and streams as HLS
+ * @param {Express.Request} req - Express request object
+ * @param {Express.Response} res - Express response object  
+ * @param {Object} options - Configuration options
+ * @param {boolean} options.useMusic - Whether to include music
+ * @param {string} options.musicPath - Path to music directory
+ * @param {Promise<string>} options.lateGeocodePromise - Optional promise for late URL update
+ */
+async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocodePromise = null }) {
+  const { url, width = 1920, height = 1080, fps = 30, hideLogo: hideLogoFlag = 'false' } = req.query;
+
+  if (!url) {
+    return res.status(400).send('URL parameter is required');
+  }
+
+  // Validate URL
+  try {
+    new URL(url);
+  } catch (error) {
+    return res.status(400).send('Invalid URL');
+  }
+
+  let browser = null;
+  let ffmpegProcess = null;
+  let isCleaningUp = false;
+  let playlistFile = null;
+
+  try {
+    // Set HLS headers
+    res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
+    res.setHeader('Cache-Control', 'no-cache');
+    res.setHeader('Connection', 'keep-alive');
+
+    // Build FFmpeg command and launch browser in parallel
+    const ffmpegConfigPromise = buildFFmpegArgs({
+      fps: parseInt(fps),
+      useMusic,
+      musicPath
+    });
+
+    const browserPromise = puppeteer.launch({
+      headless: true,
+      executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
+      args: [
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-gpu',
+        '--disable-extensions',
+        '--disable-default-apps',
+        '--disable-sync',
+        '--disable-translate',
+        '--disable-background-networking',
+        '--disable-background-timer-throttling',
+        '--no-first-run',
+        '--mute-audio',
+        '--disable-breakpad',
+        '--disable-component-update',
+        `--window-size=${width},${height}`,
+        `--force-device-scale-factor=1`
+      ],
+      defaultViewport: { width: parseInt(width), height: parseInt(height), deviceScaleFactor: 1 }
+    });
+
+    // Wait for both to complete in parallel
+    const [ffmpegConfig, browserInstance] = await Promise.all([ffmpegConfigPromise, browserPromise]);
+    browser = browserInstance;
+    playlistFile = ffmpegConfig.playlistFile;
+    
+    console.log('Starting stream with black frames...');
+
+    // Start FFmpeg immediately
+    ffmpegProcess = spawn('ffmpeg', ['-loglevel', 'error', '-hide_banner', ...ffmpegConfig.args], {
+      stdio: ['pipe', 'pipe', 'pipe']
+    });
+
+    // Pipe FFmpeg output to response
+    ffmpegProcess.stdout.pipe(res);
+
+    ffmpegProcess.stderr.on('data', (data) => {
+      const message = data.toString();
+      // Log important warnings about audio discontinuities or errors
+      if (message.includes('Non-monotonous DTS') || 
+          message.includes('Application provided invalid') ||
+          message.includes('past duration') ||
+          message.includes('Error while decoding stream') ||
+          message.includes('Invalid data found')) {
+        console.warn(`FFmpeg warning: ${message.trim()}`);
+      }
+    });
+
+    ffmpegProcess.on('error', (error) => {
+      console.error('FFmpeg error:', error);
+      cleanup();
+    });
+    
+    ffmpegProcess.on('close', (code) => {
+      if (code && code !== 0 && !isCleaningUp) {
+        console.error(`FFmpeg exited with code ${code}`);
+        cleanup();
+      }
+    });
+
+    ffmpegProcess.stdin.on('error', (error) => {
+      if (error.code !== 'EPIPE') {
+        console.error('FFmpeg stdin error:', error);
+      }
+    });
+
+    // Setup Puppeteer page
+    const page = await setupPage(browser, { width: parseInt(width), height: parseInt(height) });
+    
+    // Black frame control
+    let sendBlackFrames = true;
+    let waitingForCorrectUrl = !!lateGeocodePromise;
+    
+    // Load initial page only if we're not waiting for late geocoding
+    if (!waitingForCorrectUrl) {
+      waitForPageFullyLoaded(page, url)
+        .then(async (loaded) => {
+          if (loaded) {
+            sendBlackFrames = false;
+            if (hideLogoFlag === 'true') {
+              await hideLogo(page);
+            }
+          }
+        })
+        .catch(err => {
+          console.error('Page load promise error:', err.message);
+          sendBlackFrames = false;
+        });
+    }
+
+    // Handle late geocoding if provided
+    if (lateGeocodePromise) {
+      lateGeocodePromise.then(async (updatedUrl) => {
+        if (!isCleaningUp && page && !page.isClosed() && updatedUrl && updatedUrl !== url) {
+          try {
+            console.log('Updating to correct location...');
+            sendBlackFrames = true;
+            
+            await waitForPageFullyLoaded(page, updatedUrl);
+            
+            console.log('Correct location fully loaded, switching to live frames');
+            waitingForCorrectUrl = false;
+            sendBlackFrames = false;
+            
+            if (hideLogoFlag === 'true') {
+              await hideLogo(page);
+            }
+          } catch (err) {
+            console.error('Location update error:', err.message);
+            waitingForCorrectUrl = false;
+            sendBlackFrames = false;
+          }
+        } else if (!updatedUrl || updatedUrl === url) {
+          console.log('Using initial URL, waiting for page load to complete...');
+          // URL is the same, so load it now
+          try {
+            await waitForPageFullyLoaded(page, url);
+            waitingForCorrectUrl = false;
+            sendBlackFrames = false;
+            if (hideLogoFlag === 'true') {
+              await hideLogo(page);
+            }
+          } catch (err) {
+            console.error('Initial page load error:', err.message);
+            waitingForCorrectUrl = false;
+            sendBlackFrames = false;
+          }
+        }
+      }).catch(() => {
+        console.warn('Geocoding failed, waiting for fallback location to load');
+        // Load the fallback URL
+        waitForPageFullyLoaded(page, url)
+          .then(async () => {
+            waitingForCorrectUrl = false;
+            sendBlackFrames = false;
+            if (hideLogoFlag === 'true') {
+              await hideLogo(page);
+            }
+          })
+          .catch(() => {
+            waitingForCorrectUrl = false;
+            sendBlackFrames = false;
+          });
+      });
+    }
+    
+    // Periodic page refresh
+    const pageRefreshInterval = setInterval(async () => {
+      if (!isCleaningUp && page && !page.isClosed()) {
+        try {
+          console.log('Refreshing page for stability...');
+          await page.reload({ waitUntil: 'domcontentloaded', timeout: 10000 });
+          if (hideLogoFlag === 'true') {
+            await hideLogo(page);
+          }
+        } catch (err) {
+          console.error('Page refresh error:', err.message);
+        }
+      }
+    }, 30 * 60 * 1000);
+
+    // Frame capture
+    const frameInterval = 1000 / fps;
+    let captureLoopActive = true;
+    let consecutiveErrors = 0;
+    const MAX_CONSECUTIVE_ERRORS = 5;
+    
+    const createBlackFrame = () => {
+      try {
+        const canvas = require('canvas');
+        const canvasObj = canvas.createCanvas(parseInt(width), parseInt(height));
+        const ctx = canvasObj.getContext('2d');
+        ctx.fillStyle = '#000000';
+        ctx.fillRect(0, 0, parseInt(width), parseInt(height));
+        return canvasObj.toBuffer('image/jpeg', { quality: 0.5 });
+      } catch (err) {
+        console.error('Error creating black frame:', err);
+        return Buffer.alloc(0);
+      }
+    };
+    
+    let blackFrameBuffer = null;
+
+    const captureLoop = async () => {
+      while (!isCleaningUp && captureLoopActive && ffmpegProcess && !ffmpegProcess.killed) {
+        const start = Date.now();
+        try {
+          if (page.isClosed()) {
+            console.error('Page was closed unexpectedly');
+            break;
+          }
+          
+          let screenshot;
+          
+          if (sendBlackFrames) {
+            if (!blackFrameBuffer) {
+              blackFrameBuffer = createBlackFrame();
+            }
+            screenshot = blackFrameBuffer;
+          } else {
+            screenshot = await page.screenshot({ 
+              type: 'jpeg', 
+              quality: 80,
+              optimizeForSpeed: true,
+              fromSurface: true
+            });
+          }
+
+          if (screenshot && screenshot.length > 0 && ffmpegProcess && ffmpegProcess.stdin.writable && !isCleaningUp) {
+            const canWrite = ffmpegProcess.stdin.write(screenshot);
+            if (!canWrite) {
+              await new Promise((resolve) => {
+                let resolved = false;
+                const currentProcess = ffmpegProcess;
+                const onDrain = () => { if (!resolved) { resolved = true; cleanupListeners(); resolve(); } };
+                const onError = () => { if (!resolved) { resolved = true; cleanupListeners(); resolve(); } };
+                const timeout = setTimeout(() => { if (!resolved) { resolved = true; cleanupListeners(); resolve(); } }, 800);
+                function cleanupListeners() {
+                  clearTimeout(timeout);
+                  if (currentProcess && currentProcess.stdin) {
+                    currentProcess.stdin.removeListener('drain', onDrain);
+                    currentProcess.stdin.removeListener('error', onError);
+                  }
+                }
+                if (currentProcess && currentProcess.stdin) {
+                  currentProcess.stdin.once('drain', onDrain);
+                  currentProcess.stdin.once('error', onError);
+                } else {
+                  resolve();
+                }
+              });
+            }
+          }
+          
+          consecutiveErrors = 0;
+          
+        } catch (error) {
+          if (!isCleaningUp) {
+            consecutiveErrors++;
+            console.error(`Capture error (${consecutiveErrors}/${MAX_CONSECUTIVE_ERRORS}):`, error.message || error);
+            
+            if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
+              console.error('Too many consecutive errors, stopping stream');
+              try { await cleanup(); } catch (e) {}
+              break;
+            }
+            
+            await new Promise(r => setTimeout(r, 1000));
+          } else {
+            break;
+          }
+        }
+
+        const elapsed = Date.now() - start;
+        const wait = Math.max(0, frameInterval - elapsed);
+        if (wait > 0) await new Promise(r => setTimeout(r, wait));
+      }
+    };
+
+    captureLoop();
+
+    // Cleanup function
+    const cleanup = async () => {
+      if (isCleaningUp) return;
+      isCleaningUp = true;
+      console.log('Cleaning up stream...');
+      
+      try { captureLoopActive = false; } catch (e) {}
+      try { clearInterval(pageRefreshInterval); } catch (e) {}
+      
+      if (ffmpegProcess && !ffmpegProcess.killed) {
+        try {
+          ffmpegProcess.stdin.end();
+        } catch (err) {}
+        ffmpegProcess.kill('SIGTERM');
+        
+        setTimeout(() => {
+          if (ffmpegProcess && !ffmpegProcess.killed) {
+            ffmpegProcess.kill('SIGKILL');
+          }
+        }, 5000);
+        
+        ffmpegProcess = null;
+      }
+      
+      if (playlistFile) {
+        try {
+          fs.unlinkSync(playlistFile);
+        } catch (err) {}
+      }
+      
+      if (browser) {
+        try {
+          await browser.close();
+        } catch (err) {}
+        browser = null;
+      }
+      
+      if (!res.headersSent && !res.writableEnded) {
+        res.end();
+      }
+    };
+
+    // Handle client disconnect
+    let disconnectLogged = false;
+    
+    req.on('close', () => {
+      if (!disconnectLogged) {
+        console.log('Client disconnected');
+        disconnectLogged = true;
+      }
+      cleanup();
+    });
+    
+    req.on('error', (error) => {
+      if (error.code === 'ECONNRESET' || error.code === 'EPIPE') {
+        if (!disconnectLogged) {
+          console.log('Client disconnected');
+          disconnectLogged = true;
+        }
+      } else {
+        console.error('Request error:', error);
+      }
+      cleanup();
+    });
+
+    res.on('error', (error) => {
+      if (error.code === 'ECONNRESET' || error.code === 'EPIPE') {
+        if (!disconnectLogged) {
+          console.log('Client disconnected');
+          disconnectLogged = true;
+        }
+      } else {
+        console.error('Response error:', error);
+      }
+      cleanup();
+    });
+    
+    // Keepalive monitoring
+    const keepaliveInterval = setInterval(() => {
+      if (isCleaningUp || !ffmpegProcess || ffmpegProcess.killed) {
+        clearInterval(keepaliveInterval);
+        return;
+      }
+      if (res.writableEnded || res.socket?.destroyed) {
+        console.log('Connection lost, cleaning up');
+        clearInterval(keepaliveInterval);
+        cleanup();
+      }
+    }, 10000);
+
+  } catch (error) {
+    console.error('Error:', error);
+    if (ffmpegProcess && !ffmpegProcess.killed) ffmpegProcess.kill();
+    if (browser) await browser.close();
+    if (!res.headersSent) {
+      res.status(500).send('Internal server error');
+    }
+  }
+}
+
+module.exports = { streamHandler };