const express = require('express'); const puppeteer = require('puppeteer'); const { spawn } = require('child_process'); const path = require('path'); const fs = require('fs'); const https = require('https'); const app = express(); const PORT = process.env.PORT || 3000; const MUSIC_PATH = process.env.MUSIC_PATH || '/music'; // Geocode city to lat/lon using Nominatim (OpenStreetMap) async function geocodeCity(cityQuery) { return new Promise((resolve, reject) => { const encodedQuery = encodeURIComponent(cityQuery); const url = `https://nominatim.openstreetmap.org/search?q=${encodedQuery}&format=json&limit=1`; const options = { headers: { 'User-Agent': 'webpage-to-hls-streaming-app/1.0' } }; https.get(url, options, (res) => { let data = ''; res.on('data', (chunk) => { data += chunk; }); res.on('end', () => { try { const results = JSON.parse(data); if (results && results.length > 0) { resolve({ lat: parseFloat(results[0].lat), lon: parseFloat(results[0].lon), displayName: results[0].display_name }); } else { reject(new Error('No results found')); } } catch (error) { reject(error); } }); }).on('error', (error) => { reject(error); }); }); } // Get random music file function getRandomMusicFile() { try { if (!fs.existsSync(MUSIC_PATH)) { return null; } const files = fs.readdirSync(MUSIC_PATH).filter(f => f.endsWith('.ogg')); if (files.length === 0) { return null; } const randomFile = files[Math.floor(Math.random() * files.length)]; return path.join(MUSIC_PATH, randomFile); } catch (error) { console.error('Error getting music file:', error); return null; } } // Get all music files for shuffling function getAllMusicFiles() { try { if (!fs.existsSync(MUSIC_PATH)) { return []; } const files = fs.readdirSync(MUSIC_PATH).filter(f => f.endsWith('.ogg')); return files.map(f => path.join(MUSIC_PATH, f)); } catch (error) { console.error('Error getting music files:', error); return []; } } // Shuffle array in place function shuffleArray(array) { for (let i = array.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)); [array[i], array[j]] = [array[j], array[i]]; } return array; } // Main streaming handler async function streamHandler(req, res, useMusic = false, lateGeocodePromise = null) { const { url, width = 1920, height = 1080, fps = 30, hideLogo = 'false' } = req.query; if (!url) { return res.status(400).send('URL parameter is required'); } // Validate URL try { new URL(url); } catch (error) { return res.status(400).send('Invalid URL'); } let browser = null; let ffmpegProcess = null; let isCleaningUp = false; try { // Set HLS headers res.setHeader('Content-Type', 'application/vnd.apple.mpegurl'); res.setHeader('Cache-Control', 'no-cache'); res.setHeader('Connection', 'keep-alive'); // Build FFmpeg command and playlist in parallel with browser launch const ffmpegArgs = []; let playlistFile = null; const prepareFFmpegPromise = (async () => { if (useMusic) { // Get all music files and shuffle them const allMusicFiles = getAllMusicFiles(); if (allMusicFiles.length > 0 && allMusicFiles.length > 0) { // Create a temporary concat playlist file playlistFile = path.join('/tmp', `playlist-${Date.now()}.txt`); // Build playlist content - just 1 repetition since we loop infinitely with -stream_loop const currentShuffle = shuffleArray([...allMusicFiles]); const playlistLines = currentShuffle.map(f => `file '${f}'`); fs.writeFileSync(playlistFile, playlistLines.join('\n')); console.log(`Created shuffled playlist with ${allMusicFiles.length} tracks (infinite loop)`); // Input 0: video frames ffmpegArgs.push( '-use_wallclock_as_timestamps', '1', '-f', 'image2pipe', '-framerate', fps.toString(), '-i', 'pipe:0' ); // Input 1: audio from concat playlist ffmpegArgs.push( '-f', 'concat', '-safe', '0', '-stream_loop', '-1', // Loop playlist infinitely '-probesize', '32', // Minimal probing for faster startup '-analyzeduration', '0', // Skip analysis for faster startup '-i', playlistFile ); // Encoding ffmpegArgs.push( '-c:v', 'libx264', '-preset', 'ultrafast', '-tune', 'zerolatency', '-pix_fmt', 'yuv420p', '-g', fps.toString(), // Keyframe every second for 1s segments '-bf', '0', // No B-frames for lower latency '-x264opts', 'nal-hrd=cbr:no-scenecut', // Constant bitrate, no scene detection '-b:v', '2500k', // Target bitrate for stable encoding '-maxrate', '2500k', '-bufsize', '5000k', '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', // Set explicit audio sample rate '-f', 'hls', '-hls_time', '1', // Smaller segments for faster startup '-hls_list_size', '3', // Fewer segments in playlist '-hls_flags', 'delete_segments+omit_endlist', '-hls_start_number_source', 'epoch', '-start_number', '0', // Start from segment 0 '-flush_packets', '1', // Flush packets immediately 'pipe:1' ); return true; } } // Video only (no music) ffmpegArgs.push( '-use_wallclock_as_timestamps', '1', '-f', 'image2pipe', '-framerate', fps.toString(), '-i', 'pipe:0', '-c:v', 'libx264', '-preset', 'ultrafast', '-tune', 'zerolatency', '-pix_fmt', 'yuv420p', '-g', fps.toString(), // Keyframe every second for 1s segments '-bf', '0', '-x264opts', 'nal-hrd=cbr:no-scenecut', '-b:v', '2500k', '-maxrate', '2500k', '-bufsize', '5000k', '-f', 'hls', '-hls_time', '1', // Smaller segments for faster startup '-hls_list_size', '3', // Fewer segments in playlist '-hls_flags', 'delete_segments+omit_endlist', '-hls_start_number_source', 'epoch', '-start_number', '0', '-flush_packets', '1', // Flush packets immediately 'pipe:1' ); return false; })(); // Launch browser in parallel with FFmpeg preparation const browserPromise = puppeteer.launch({ headless: true, executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined, args: [ '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--disable-extensions', '--disable-default-apps', '--disable-sync', '--disable-translate', '--disable-background-networking', '--disable-background-timer-throttling', '--no-first-run', '--mute-audio', '--disable-breakpad', '--disable-component-update', `--window-size=${width},${height}`, `--force-device-scale-factor=1` // Ensure no DPI scaling issues ], defaultViewport: { width: parseInt(width), height: parseInt(height), deviceScaleFactor: 1 } }); // Wait for both to complete in parallel const [hasMusic] = await Promise.all([prepareFFmpegPromise, browserPromise.then(b => { browser = b; })]); console.log('Starting stream with black frames...'); // Start FFmpeg immediately - don't wait for page ffmpegProcess = spawn('ffmpeg', ['-loglevel', 'error', '-hide_banner', ...ffmpegArgs], { stdio: ['pipe', 'pipe', 'pipe'] }); // Pipe FFmpeg output to response immediately ffmpegProcess.stdout.pipe(res); ffmpegProcess.stderr.on('data', (data) => { // Suppress verbose FFmpeg output // console.error(`FFmpeg: ${data}`); }); ffmpegProcess.on('error', (error) => { console.error('FFmpeg error:', error); cleanup(); }); ffmpegProcess.on('close', (code) => { if (code && code !== 0 && !isCleaningUp) { console.error(`FFmpeg exited with code ${code}`); cleanup(); } }); ffmpegProcess.stdin.on('error', (error) => { // Ignore EPIPE errors when client disconnects if (error.code !== 'EPIPE') { console.error('FFmpeg stdin error:', error); } }); // Start creating page in parallel with FFmpeg starting const page = await browser.newPage(); // Reduce memory usage by disabling caching await page.setCacheEnabled(false); // Inject CSS early to prevent white flash during page load await page.evaluateOnNewDocument(() => { const style = document.createElement('style'); style.textContent = ` html, body { background-color: #000 !important; } `; document.head?.appendChild(style) || document.documentElement.appendChild(style); }); // Always start with black frames until the CORRECT page loads let sendBlackFrames = true; let waitingForCorrectUrl = !!lateGeocodePromise; // Track if we're waiting for geocoding to complete // Start loading the page in the background - don't wait for it const pageLoadPromise = page.goto(url, { waitUntil: 'load', timeout: 30000 }) .then(() => { // Only switch to live frames if we're not waiting for the correct URL if (!waitingForCorrectUrl) { console.log('Page loaded, switching to live frames'); sendBlackFrames = false; // Hide logo if requested if (hideLogo === 'true') { page.evaluate(() => { const images = document.querySelectorAll('img'); images.forEach(img => { if (img.src && img.src.includes('Logo3.png')) { img.style.display = 'none'; } }); }).catch(err => console.error('Logo hide error:', err)); } } }) .catch(err => { console.error('Page load error:', err.message); if (!waitingForCorrectUrl) { sendBlackFrames = false; } }); // If we have a late geocoding promise, navigate to correct URL when ready if (lateGeocodePromise) { lateGeocodePromise.then(async (updatedUrl) => { if (!isCleaningUp && page && !page.isClosed() && updatedUrl && updatedUrl !== url) { try { console.log('Updating to correct location...'); await page.goto(updatedUrl, { waitUntil: 'load', timeout: 10000 }); console.log('Correct location loaded, switching to live frames'); waitingForCorrectUrl = false; sendBlackFrames = false; // Now show real frames if (hideLogo === 'true') { await page.evaluate(() => { const images = document.querySelectorAll('img'); images.forEach(img => { if (img.src && img.src.includes('Logo3.png')) { img.style.display = 'none'; } }); }).catch(() => {}); } } catch (err) { console.error('Location update error:', err.message); waitingForCorrectUrl = false; sendBlackFrames = false; } } else if (!updatedUrl || updatedUrl === url) { // Geocoding completed but URL is the same (was already correct) waitingForCorrectUrl = false; sendBlackFrames = false; } }).catch(() => { // Geocoding failed - use fallback location console.warn('Geocoding failed, using fallback location'); waitingForCorrectUrl = false; sendBlackFrames = false; }); } // Add periodic page refresh to prevent memory leaks and stale content // Refresh every 30 minutes to keep stream stable const pageRefreshInterval = setInterval(async () => { if (!isCleaningUp && page && !page.isClosed()) { try { console.log('Refreshing page for stability...'); await page.reload({ waitUntil: 'domcontentloaded', timeout: 10000 }); if (hideLogo === 'true') { await page.evaluate(() => { const images = document.querySelectorAll('img'); images.forEach(img => { if (img.src && img.src.includes('Logo3.png')) { img.style.display = 'none'; } }); }).catch(() => {}); } } catch (err) { console.error('Page refresh error:', err.message); } } }, 30 * 60 * 1000); // 30 minutes // Capture frames using a sequential loop (avoids overlapping screenshots) // and handle backpressure on ffmpeg.stdin (wait for 'drain' with timeout). const frameInterval = 1000 / fps; let captureLoopActive = true; let consecutiveErrors = 0; const MAX_CONSECUTIVE_ERRORS = 5; // Create a black frame buffer once (reused for all black frames) const createBlackFrame = () => { try { // Create a minimal black JPEG with exact dimensions matching the stream const canvas = require('canvas'); const canvasObj = canvas.createCanvas(parseInt(width), parseInt(height)); const ctx = canvasObj.getContext('2d'); ctx.fillStyle = '#000000'; ctx.fillRect(0, 0, parseInt(width), parseInt(height)); return canvasObj.toBuffer('image/jpeg', { quality: 0.5 }); } catch (err) { console.error('Error creating black frame:', err); // Fallback: create a minimal valid JPEG header (won't look good but won't crash) return Buffer.alloc(0); } }; let blackFrameBuffer = null; const captureLoop = async () => { while (!isCleaningUp && captureLoopActive && ffmpegProcess && !ffmpegProcess.killed) { const start = Date.now(); try { // Check if page is still valid if (page.isClosed()) { console.error('Page was closed unexpectedly'); break; } let screenshot; // Send black frames if we're waiting for page load if (sendBlackFrames) { if (!blackFrameBuffer) { blackFrameBuffer = createBlackFrame(); } screenshot = blackFrameBuffer; } else { // Take a screenshot with optimized settings screenshot = await page.screenshot({ type: 'jpeg', quality: 80, optimizeForSpeed: true, fromSurface: true }); } // Ensure we have valid data before writing if (screenshot && screenshot.length > 0 && ffmpegProcess && ffmpegProcess.stdin.writable && !isCleaningUp) { const canWrite = ffmpegProcess.stdin.write(screenshot); if (!canWrite) { // Backpressure — wait for drain but don't block forever await new Promise((resolve) => { let resolved = false; const currentProcess = ffmpegProcess; // Capture reference to avoid null access const onDrain = () => { if (!resolved) { resolved = true; cleanupListeners(); resolve(); } }; const onError = () => { if (!resolved) { resolved = true; cleanupListeners(); resolve(); } }; const timeout = setTimeout(() => { if (!resolved) { resolved = true; cleanupListeners(); resolve(); } }, 800); function cleanupListeners() { clearTimeout(timeout); if (currentProcess && currentProcess.stdin) { currentProcess.stdin.removeListener('drain', onDrain); currentProcess.stdin.removeListener('error', onError); } } if (currentProcess && currentProcess.stdin) { currentProcess.stdin.once('drain', onDrain); currentProcess.stdin.once('error', onError); } else { resolve(); } }); } } // Reset error counter on success consecutiveErrors = 0; } catch (error) { if (!isCleaningUp) { consecutiveErrors++; console.error(`Capture error (${consecutiveErrors}/${MAX_CONSECUTIVE_ERRORS}):`, error.message || error); // If too many consecutive errors, give up if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) { console.error('Too many consecutive errors, stopping stream'); try { await cleanup(); } catch (e) {} break; } // Wait a bit before retrying on error await new Promise(r => setTimeout(r, 1000)); } else { break; } } const elapsed = Date.now() - start; const wait = Math.max(0, frameInterval - elapsed); // Wait the remaining time before next frame if (wait > 0) await new Promise(r => setTimeout(r, wait)); } }; // Start the capture loop (no overlapping runs) captureLoop(); // Cleanup function const cleanup = async () => { if (isCleaningUp) return; isCleaningUp = true; console.log('Cleaning up stream...'); // stop capture loop try { captureLoopActive = false; } catch (e) {} // Clear page refresh interval try { clearInterval(pageRefreshInterval); } catch (e) {} if (ffmpegProcess && !ffmpegProcess.killed) { try { ffmpegProcess.stdin.end(); } catch (err) { // Ignore errors during cleanup } ffmpegProcess.kill('SIGTERM'); // Force kill after 5 seconds if still running setTimeout(() => { if (ffmpegProcess && !ffmpegProcess.killed) { ffmpegProcess.kill('SIGKILL'); } }, 5000); ffmpegProcess = null; } // Clean up temporary playlist file if (playlistFile) { try { fs.unlinkSync(playlistFile); } catch (err) { // Ignore errors during cleanup } } if (browser) { try { await browser.close(); } catch (err) { // Ignore errors during cleanup } browser = null; } if (!res.headersSent && !res.writableEnded) { res.end(); } }; // Handle client disconnect let disconnectLogged = false; req.on('close', () => { if (!disconnectLogged) { console.log('Client disconnected'); disconnectLogged = true; } cleanup(); }); req.on('error', (error) => { // Ignore expected disconnect errors if (error.code === 'ECONNRESET' || error.code === 'EPIPE') { if (!disconnectLogged) { console.log('Client disconnected'); disconnectLogged = true; } } else { console.error('Request error:', error); } cleanup(); }); res.on('error', (error) => { // Ignore expected disconnect errors if (error.code === 'ECONNRESET' || error.code === 'EPIPE') { if (!disconnectLogged) { console.log('Client disconnected'); disconnectLogged = true; } } else { console.error('Response error:', error); } cleanup(); }); // Add keepalive monitoring const keepaliveInterval = setInterval(() => { if (isCleaningUp || !ffmpegProcess || ffmpegProcess.killed) { clearInterval(keepaliveInterval); return; } // Check if connection is still alive if (res.writableEnded || res.socket?.destroyed) { console.log('Connection lost, cleaning up'); clearInterval(keepaliveInterval); cleanup(); } }, 10000); // Check every 10 seconds } catch (error) { console.error('Error:', error); if (ffmpegProcess && !ffmpegProcess.killed) ffmpegProcess.kill(); if (browser) await browser.close(); if (!res.headersSent) { res.status(500).send('Internal server error'); } } } // Stream endpoint app.get('/stream', (req, res) => streamHandler(req, res, false)); app.get('/weather', async (req, res) => { const { city = 'Toronto, ON, CAN', width = 1920, height = 1080, fps = 30, hideLogo = 'false', units = 'metric', // 'metric' or 'imperial' timeFormat = '24h', // '12h' or '24h' // Forecast section toggles - commonly used sections (default: true) showHazards = 'true', showCurrent = 'true', showHourly = 'true', showHourlyGraph = 'true', showLocalForecast = 'true', showExtendedForecast = 'true', showRadar = 'true', showAQI = 'true', showAlmanac = 'true', showLatestObservations = 'true', showRegionalForecast = 'true', // Less common sections (default: false) showTravel = 'false', showMarineForecast = 'false' } = req.query; // Unit conversions for ws4kp const isMetric = units.toLowerCase() === 'metric'; const temperatureUnit = isMetric ? '1.00' : '2.00'; const windUnit = isMetric ? '1.00' : '2.00'; const distanceUnit = isMetric ? '1.00' : '2.00'; const pressureUnit = isMetric ? '1.00' : '2.00'; const hoursFormat = timeFormat === '12h' ? '1.00' : '2.00'; const ws4kpBaseUrl = process.env.WS4KP_URL || 'http://localhost:8080'; // Function to build URL with given coordinates const buildUrl = (latitude, longitude) => { const ws4kpParams = new URLSearchParams({ 'hazards-checkbox': showHazards, 'current-weather-checkbox': showCurrent, 'latest-observations-checkbox': showLatestObservations, 'hourly-checkbox': showHourly, 'hourly-graph-checkbox': showHourlyGraph, 'travel-checkbox': showTravel, 'regional-forecast-checkbox': showRegionalForecast, 'local-forecast-checkbox': showLocalForecast, 'extended-forecast-checkbox': showExtendedForecast, 'almanac-checkbox': showAlmanac, 'radar-checkbox': showRadar, 'marine-forecast-checkbox': showMarineForecast, 'aqi-forecast-checkbox': showAQI, 'settings-experimentalFeatures-checkbox': 'false', 'settings-hideWebamp-checkbox': 'true', 'settings-kiosk-checkbox': 'false', 'settings-scanLines-checkbox': 'false', 'settings-wide-checkbox': 'true', 'chkAutoRefresh': 'true', 'settings-windUnits-select': windUnit, 'settings-marineWindUnits-select': '1.00', 'settings-marineWaveHeightUnits-select': '1.00', 'settings-temperatureUnits-select': temperatureUnit, 'settings-distanceUnits-select': distanceUnit, 'settings-pressureUnits-select': pressureUnit, 'settings-hoursFormat-select': hoursFormat, 'settings-speed-select': '1.00', 'latLonQuery': city, 'latLon': JSON.stringify({ lat: latitude, lon: longitude }), 'kiosk': 'true' }); return `${ws4kpBaseUrl}/?${ws4kpParams.toString()}`; }; // Start geocoding immediately in the background - don't block anything let lateGeocodePromise = null; let initialUrl = 'data:text/html,'; // Dummy black page if (city && city !== 'Toronto, ON, CAN') { // Start geocoding - this will be the only URL we load const geocodePromise = Promise.race([ geocodeCity(city), new Promise((_, reject) => setTimeout(() => reject(new Error('Geocoding timeout')), 1000)) ]).then(geoResult => { console.log(`Geocoded: ${city} -> ${geoResult.displayName}`); const finalUrl = buildUrl(geoResult.lat, geoResult.lon); console.log(`URL: ${finalUrl}`); return { url: finalUrl, lat: geoResult.lat, lon: geoResult.lon }; }).catch(error => { // Geocoding timed out or failed - continue in background return geocodeCity(city).then(geoResult => { const finalUrl = buildUrl(geoResult.lat, geoResult.lon); console.log(`Geocoding completed: ${geoResult.displayName} (${geoResult.lat}, ${geoResult.lon})`); console.log(`Final URL: ${finalUrl}`); return { url: finalUrl, lat: geoResult.lat, lon: geoResult.lon, isLate: true }; }).catch(err => { console.warn(`Geocoding failed: ${err.message}`); // Fall back to Toronto if geocoding completely fails const fallbackUrl = buildUrl(43.6532, -79.3832); return { url: fallbackUrl, lat: 43.6532, lon: -79.3832, isLate: true }; }); }); // Always wait for geocoding to complete (or timeout and continue in background) lateGeocodePromise = geocodePromise.then(result => { return result.url; }); } else { // Toronto - use directly const lat = 43.6532; const lon = -79.3832; initialUrl = buildUrl(lat, lon); console.log(`URL: ${initialUrl}`); } console.log(`Stream starting: ${city}`); // Forward to the main stream endpoint WITH MUSIC req.query.url = initialUrl; req.query.width = width; req.query.height = height; req.query.fps = fps; req.query.hideLogo = hideLogo; // Call the stream handler with music enabled and late geocode promise return streamHandler(req, res, true, lateGeocodePromise); }); app.get('/health', (req, res) => { res.send('OK'); }); app.listen(PORT, () => { console.log(`Webpage to HLS server running on port ${PORT}`); console.log(`Usage: http://localhost:${PORT}/stream?url=http://example.com`); console.log(`Weather: http://localhost:${PORT}/weather?city=YourCity`); });