seth-public/ws4kp-to-hls
1
0
Fork 0
Code Activity

Lots of attempts at optimization

Browse Source
This commit is contained in:
Seth Van Niekerk
2025-11-07 15:58:12 -05:00
parent b2c65553a6
commit f59b0de539
3 changed files with 456 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
Dockerfile
View File

@@ -1,12 +1,24 @@
FROM ghcr.io/mwood77/ws4kp-international:latest
# Install FFmpeg, Chromium, wget, and unzip
# Install FFmpeg, Chromium, wget, unzip, and canvas dependencies
RUN apk add --no-cache \
chromium \
ffmpeg \
font-noto-emoji \
wget \
unzip
unzip \
cairo-dev \
jpeg-dev \
pango-dev \
giflib-dev \
pixman-dev \
pangomm-dev \
libjpeg-turbo-dev \
freetype-dev \
build-base \
g++ \
make \
python3
# Download and extract Weatherscan music
RUN mkdir -p /music && \

474
index.js
View File

@@ -92,7 +92,7 @@ function shuffleArray(array) {
}
// Main streaming handler
async function streamHandler(req, res, useMusic = false) {
async function streamHandler(req, res, useMusic = false, lateGeocodePromise = null) {
const { url, width = 1920, height = 1080, fps = 30, hideLogo = 'false' } = req.query;
if (!url) {
@@ -111,71 +111,29 @@ async function streamHandler(req, res, useMusic = false) {
let isCleaningUp = false;
try {
console.log(`Starting stream for: ${url}`);
// Set HLS headers
res.setHeader('Content-Type', 'application/vnd.apple.mpegurl');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
// Launch browser
browser = await puppeteer.launch({
headless: true,
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-gpu',
`--window-size=${width},${height}`
],
defaultViewport: { width: parseInt(width), height: parseInt(height) }
});
const page = await browser.newPage();
await page.setViewport({ width: parseInt(width), height: parseInt(height) });
await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
// Hide logo if requested
if (hideLogo === 'true') {
await page.evaluate(() => {
const images = document.querySelectorAll('img');
images.forEach(img => {
if (img.src && img.src.includes('Logo3.png')) {
img.style.display = 'none';
}
});
});
console.log('Logo hidden');
}
console.log('Page loaded, starting FFmpeg...');
// Build FFmpeg command with optional music
// Build FFmpeg command and playlist in parallel with browser launch
const ffmpegArgs = [];
let playlistFile = null;
const prepareFFmpegPromise = (async () => {
if (useMusic) {
// Get all music files and shuffle them
const allMusicFiles = getAllMusicFiles();
if (allMusicFiles.length > 0) {
const shuffledFiles = shuffleArray([...allMusicFiles]);
if (allMusicFiles.length > 0 && allMusicFiles.length > 0) {
// Create a temporary concat playlist file
playlistFile = path.join('/tmp', `playlist-${Date.now()}.txt`);
// Build playlist content - repeat the shuffled list multiple times to ensure we don't run out
let playlistContent = '';
for (let repeat = 0; repeat < 100; repeat++) {
// Re-shuffle for each repeat to keep it truly random
// Build playlist content - just 1 repetition since we loop infinitely with -stream_loop
const currentShuffle = shuffleArray([...allMusicFiles]);
currentShuffle.forEach(file => {
playlistContent += `file '${file}'\n`;
});
}
const playlistLines = currentShuffle.map(f => `file '${f}'`);
fs.writeFileSync(playlistFile, playlistContent);
console.log(`Created shuffled playlist with ${allMusicFiles.length} unique tracks`);
fs.writeFileSync(playlistFile, playlistLines.join('\n'));
console.log(`Created shuffled playlist with ${allMusicFiles.length} tracks (infinite loop)`);
// Input 0: video frames
ffmpegArgs.push(
@@ -188,6 +146,9 @@ async function streamHandler(req, res, useMusic = false) {
ffmpegArgs.push(
'-f', 'concat',
'-safe', '0',
'-stream_loop', '-1', // Loop playlist infinitely
'-probesize', '32', // Minimal probing for faster startup
'-analyzeduration', '0', // Skip analysis for faster startup
'-i', playlistFile
);
// Encoding
@@ -196,23 +157,28 @@ async function streamHandler(req, res, useMusic = false) {
'-preset', 'ultrafast',
'-tune', 'zerolatency',
'-pix_fmt', 'yuv420p',
'-g', (fps * 2).toString(),
'-g', fps.toString(), // Keyframe every second for 1s segments
'-bf', '0', // No B-frames for lower latency
'-x264opts', 'nal-hrd=cbr:no-scenecut', // Constant bitrate, no scene detection
'-b:v', '2500k', // Target bitrate for stable encoding
'-maxrate', '2500k',
'-bufsize', '5000k',
'-c:a', 'aac',
'-b:a', '128k',
'-shortest',
'-ar', '44100', // Set explicit audio sample rate
'-f', 'hls',
'-hls_time', '2',
'-hls_list_size', '5',
'-hls_flags', 'delete_segments',
'-hls_time', '1', // Smaller segments for faster startup
'-hls_list_size', '3', // Fewer segments in playlist
'-hls_flags', 'delete_segments+omit_endlist',
'-hls_start_number_source', 'epoch',
'-start_number', '0', // Start from segment 0
'-flush_packets', '1', // Flush packets immediately
'pipe:1'
);
} else {
console.warn('No music files found, streaming without audio');
useMusic = false;
return true;
}
}
if (!useMusic) {
// Video only (no music)
ffmpegArgs.push(
'-use_wallclock_as_timestamps', '1',
@@ -223,19 +189,60 @@ async function streamHandler(req, res, useMusic = false) {
'-preset', 'ultrafast',
'-tune', 'zerolatency',
'-pix_fmt', 'yuv420p',
'-g', (fps * 2).toString(),
'-g', fps.toString(), // Keyframe every second for 1s segments
'-bf', '0',
'-x264opts', 'nal-hrd=cbr:no-scenecut',
'-b:v', '2500k',
'-maxrate', '2500k',
'-bufsize', '5000k',
'-f', 'hls',
'-hls_time', '2',
'-hls_list_size', '5',
'-hls_flags', 'delete_segments',
'-hls_time', '1', // Smaller segments for faster startup
'-hls_list_size', '3', // Fewer segments in playlist
'-hls_flags', 'delete_segments+omit_endlist',
'-hls_start_number_source', 'epoch',
'-start_number', '0',
'-flush_packets', '1', // Flush packets immediately
'pipe:1'
);
}
return false;
})();
// Start FFmpeg
ffmpegProcess = spawn('ffmpeg', ffmpegArgs);
// Launch browser in parallel with FFmpeg preparation
const browserPromise = puppeteer.launch({
headless: true,
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || undefined,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-gpu',
'--disable-extensions',
'--disable-default-apps',
'--disable-sync',
'--disable-translate',
'--disable-background-networking',
'--disable-background-timer-throttling',
'--no-first-run',
'--mute-audio',
'--disable-breakpad',
'--disable-component-update',
`--window-size=${width},${height}`,
`--force-device-scale-factor=1` // Ensure no DPI scaling issues
],
defaultViewport: { width: parseInt(width), height: parseInt(height), deviceScaleFactor: 1 }
});
// Pipe FFmpeg output to response
// Wait for both to complete in parallel
const [hasMusic] = await Promise.all([prepareFFmpegPromise, browserPromise.then(b => { browser = b; })]);
console.log('Starting stream with black frames...');
// Start FFmpeg immediately - don't wait for page
ffmpegProcess = spawn('ffmpeg', ['-loglevel', 'error', '-hide_banner', ...ffmpegArgs], {
stdio: ['pipe', 'pipe', 'pipe']
});
// Pipe FFmpeg output to response immediately
ffmpegProcess.stdout.pipe(res);
ffmpegProcess.stderr.on('data', (data) => {
@@ -248,6 +255,13 @@ async function streamHandler(req, res, useMusic = false) {
cleanup();
});
ffmpegProcess.on('close', (code) => {
if (code && code !== 0 && !isCleaningUp) {
console.error(`FFmpeg exited with code ${code}`);
cleanup();
}
});
ffmpegProcess.stdin.on('error', (error) => {
// Ignore EPIPE errors when client disconnects
if (error.code !== 'EPIPE') {
@@ -255,43 +269,219 @@ async function streamHandler(req, res, useMusic = false) {
}
});
// Start creating page in parallel with FFmpeg starting
const page = await browser.newPage();
// Reduce memory usage by disabling caching
await page.setCacheEnabled(false);
// Inject CSS early to prevent white flash during page load
await page.evaluateOnNewDocument(() => {
const style = document.createElement('style');
style.textContent = `
html, body {
background-color: #000 !important;
}
`;
document.head?.appendChild(style) || document.documentElement.appendChild(style);
});
// Always start with black frames until the CORRECT page loads
let sendBlackFrames = true;
let waitingForCorrectUrl = !!lateGeocodePromise; // Track if we're waiting for geocoding to complete
// Start loading the page in the background - don't wait for it
const pageLoadPromise = page.goto(url, { waitUntil: 'load', timeout: 30000 })
.then(() => {
// Only switch to live frames if we're not waiting for the correct URL
if (!waitingForCorrectUrl) {
console.log('Page loaded, switching to live frames');
sendBlackFrames = false;
// Hide logo if requested
if (hideLogo === 'true') {
page.evaluate(() => {
const images = document.querySelectorAll('img');
images.forEach(img => {
if (img.src && img.src.includes('Logo3.png')) {
img.style.display = 'none';
}
});
}).catch(err => console.error('Logo hide error:', err));
}
}
})
.catch(err => {
console.error('Page load error:', err.message);
if (!waitingForCorrectUrl) {
sendBlackFrames = false;
}
});
// If we have a late geocoding promise, navigate to correct URL when ready
if (lateGeocodePromise) {
lateGeocodePromise.then(async (updatedUrl) => {
if (!isCleaningUp && page && !page.isClosed() && updatedUrl && updatedUrl !== url) {
try {
console.log('Updating to correct location...');
await page.goto(updatedUrl, { waitUntil: 'load', timeout: 10000 });
console.log('Correct location loaded, switching to live frames');
waitingForCorrectUrl = false;
sendBlackFrames = false; // Now show real frames
if (hideLogo === 'true') {
await page.evaluate(() => {
const images = document.querySelectorAll('img');
images.forEach(img => {
if (img.src && img.src.includes('Logo3.png')) {
img.style.display = 'none';
}
});
}).catch(() => {});
}
} catch (err) {
console.error('Location update error:', err.message);
waitingForCorrectUrl = false;
sendBlackFrames = false;
}
} else if (!updatedUrl || updatedUrl === url) {
// Geocoding completed but URL is the same (was already correct)
waitingForCorrectUrl = false;
sendBlackFrames = false;
}
}).catch(() => {
// Geocoding failed - use fallback location
console.warn('Geocoding failed, using fallback location');
waitingForCorrectUrl = false;
sendBlackFrames = false;
});
}
// Add periodic page refresh to prevent memory leaks and stale content
// Refresh every 30 minutes to keep stream stable
const pageRefreshInterval = setInterval(async () => {
if (!isCleaningUp && page && !page.isClosed()) {
try {
console.log('Refreshing page for stability...');
await page.reload({ waitUntil: 'domcontentloaded', timeout: 10000 });
if (hideLogo === 'true') {
await page.evaluate(() => {
const images = document.querySelectorAll('img');
images.forEach(img => {
if (img.src && img.src.includes('Logo3.png')) {
img.style.display = 'none';
}
});
}).catch(() => {});
}
} catch (err) {
console.error('Page refresh error:', err.message);
}
}
}, 30 * 60 * 1000); // 30 minutes
// Capture frames using a sequential loop (avoids overlapping screenshots)
// and handle backpressure on ffmpeg.stdin (wait for 'drain' with timeout).
const frameInterval = 1000 / fps;
let captureLoopActive = true;
let consecutiveErrors = 0;
const MAX_CONSECUTIVE_ERRORS = 5;
// Create a black frame buffer once (reused for all black frames)
const createBlackFrame = () => {
try {
// Create a minimal black JPEG with exact dimensions matching the stream
const canvas = require('canvas');
const canvasObj = canvas.createCanvas(parseInt(width), parseInt(height));
const ctx = canvasObj.getContext('2d');
ctx.fillStyle = '#000000';
ctx.fillRect(0, 0, parseInt(width), parseInt(height));
return canvasObj.toBuffer('image/jpeg', { quality: 0.5 });
} catch (err) {
console.error('Error creating black frame:', err);
// Fallback: create a minimal valid JPEG header (won't look good but won't crash)
return Buffer.alloc(0);
}
};
let blackFrameBuffer = null;
const captureLoop = async () => {
while (!isCleaningUp && captureLoopActive && ffmpegProcess && !ffmpegProcess.killed) {
const start = Date.now();
try {
// Take a screenshot (lower quality a bit to reduce CPU/pipe pressure)
const screenshot = await page.screenshot({ type: 'jpeg', quality: 70 });
// Check if page is still valid
if (page.isClosed()) {
console.error('Page was closed unexpectedly');
break;
}
if (ffmpegProcess && ffmpegProcess.stdin.writable && !isCleaningUp) {
let screenshot;
// Send black frames if we're waiting for page load
if (sendBlackFrames) {
if (!blackFrameBuffer) {
blackFrameBuffer = createBlackFrame();
}
screenshot = blackFrameBuffer;
} else {
// Take a screenshot with optimized settings
screenshot = await page.screenshot({
type: 'jpeg',
quality: 80,
optimizeForSpeed: true,
fromSurface: true
});
}
// Ensure we have valid data before writing
if (screenshot && screenshot.length > 0 && ffmpegProcess && ffmpegProcess.stdin.writable && !isCleaningUp) {
const canWrite = ffmpegProcess.stdin.write(screenshot);
if (!canWrite) {
// Backpressure — wait for drain but don't block forever
await new Promise((resolve) => {
let resolved = false;
const currentProcess = ffmpegProcess; // Capture reference to avoid null access
const onDrain = () => { if (!resolved) { resolved = true; cleanupListeners(); resolve(); } };
const onError = () => { if (!resolved) { resolved = true; cleanupListeners(); resolve(); } };
const timeout = setTimeout(() => { if (!resolved) { resolved = true; cleanupListeners(); resolve(); } }, 800);
function cleanupListeners() {
clearTimeout(timeout);
ffmpegProcess.stdin.removeListener('drain', onDrain);
ffmpegProcess.stdin.removeListener('error', onError);
if (currentProcess && currentProcess.stdin) {
currentProcess.stdin.removeListener('drain', onDrain);
currentProcess.stdin.removeListener('error', onError);
}
}
if (currentProcess && currentProcess.stdin) {
currentProcess.stdin.once('drain', onDrain);
currentProcess.stdin.once('error', onError);
} else {
resolve();
}
ffmpegProcess.stdin.once('drain', onDrain);
ffmpegProcess.stdin.once('error', onError);
});
}
}
// Reset error counter on success
consecutiveErrors = 0;
} catch (error) {
if (!isCleaningUp) {
console.error('Capture error:', error.message || error);
consecutiveErrors++;
console.error(`Capture error (${consecutiveErrors}/${MAX_CONSECUTIVE_ERRORS}):`, error.message || error);
// If too many consecutive errors, give up
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
console.error('Too many consecutive errors, stopping stream');
try { await cleanup(); } catch (e) {}
break;
}
// Wait a bit before retrying on error
await new Promise(r => setTimeout(r, 1000));
} else {
break;
}
}
const elapsed = Date.now() - start;
@@ -308,9 +498,14 @@ async function streamHandler(req, res, useMusic = false) {
const cleanup = async () => {
if (isCleaningUp) return;
isCleaningUp = true;
console.log('Cleaning up stream...');
// stop capture loop
try { captureLoopActive = false; } catch (e) {}
// Clear page refresh interval
try { clearInterval(pageRefreshInterval); } catch (e) {}
if (ffmpegProcess && !ffmpegProcess.killed) {
try {
ffmpegProcess.stdin.end();
@@ -318,6 +513,14 @@ async function streamHandler(req, res, useMusic = false) {
// Ignore errors during cleanup
}
ffmpegProcess.kill('SIGTERM');
// Force kill after 5 seconds if still running
setTimeout(() => {
if (ffmpegProcess && !ffmpegProcess.killed) {
ffmpegProcess.kill('SIGKILL');
}
}, 5000);
ffmpegProcess = null;
}
@@ -345,16 +548,56 @@ async function streamHandler(req, res, useMusic = false) {
};
// Handle client disconnect
let disconnectLogged = false;
req.on('close', () => {
if (!disconnectLogged) {
console.log('Client disconnected');
disconnectLogged = true;
}
cleanup();
});
req.on('error', (error) => {
// Ignore expected disconnect errors
if (error.code === 'ECONNRESET' || error.code === 'EPIPE') {
if (!disconnectLogged) {
console.log('Client disconnected');
disconnectLogged = true;
}
} else {
console.error('Request error:', error);
}
cleanup();
});
res.on('error', (error) => {
// Ignore expected disconnect errors
if (error.code === 'ECONNRESET' || error.code === 'EPIPE') {
if (!disconnectLogged) {
console.log('Client disconnected');
disconnectLogged = true;
}
} else {
console.error('Response error:', error);
}
cleanup();
});
// Add keepalive monitoring
const keepaliveInterval = setInterval(() => {
if (isCleaningUp || !ffmpegProcess || ffmpegProcess.killed) {
clearInterval(keepaliveInterval);
return;
}
// Check if connection is still alive
if (res.writableEnded || res.socket?.destroyed) {
console.log('Connection lost, cleaning up');
clearInterval(keepaliveInterval);
cleanup();
}
}, 10000); // Check every 10 seconds
} catch (error) {
console.error('Error:', error);
if (ffmpegProcess && !ffmpegProcess.killed) ffmpegProcess.kill();
@@ -394,30 +637,7 @@ app.get('/weather', async (req, res) => {
showMarineForecast = 'false'
} = req.query;
// Default coordinates (Toronto, ON, Canada)
let lat = 43.6532;
let lon = -79.3832;
// Try to geocode the city if provided
if (city) {
try {
console.log(`Geocoding city: ${city}`);
const geoResult = await geocodeCity(city);
lat = geoResult.lat;
lon = geoResult.lon;
console.log(`Geocoded to: ${geoResult.displayName} (${lat}, ${lon})`);
} catch (error) {
console.warn(`Geocoding failed for "${city}", using default coordinates:`, error.message);
// Fall back to defaults
}
}
// Unit conversions for ws4kp
// Temperature: 1.00=Celsius, 2.00=Fahrenheit
// Wind: 1.00=kph, 2.00=mph
// Distance: 1.00=km, 2.00=miles
// Pressure: 1.00=mb, 2.00=inHg
// Hours: 1.00=12h, 2.00=24h
const isMetric = units.toLowerCase() === 'metric';
const temperatureUnit = isMetric ? '1.00' : '2.00';
const windUnit = isMetric ? '1.00' : '2.00';
@@ -425,9 +645,10 @@ app.get('/weather', async (req, res) => {
const pressureUnit = isMetric ? '1.00' : '2.00';
const hoursFormat = timeFormat === '12h' ? '1.00' : '2.00';
// Build the ws4kp URL with all the parameters
// ws4kp runs on port 8080 inside the container
const ws4kpBaseUrl = process.env.WS4KP_URL || 'http://localhost:8080';
// Function to build URL with given coordinates
const buildUrl = (latitude, longitude) => {
const ws4kpParams = new URLSearchParams({
'hazards-checkbox': showHazards,
'current-weather-checkbox': showCurrent,
@@ -457,23 +678,64 @@ app.get('/weather', async (req, res) => {
'settings-hoursFormat-select': hoursFormat,
'settings-speed-select': '1.00',
'latLonQuery': city,
'latLon': JSON.stringify({ lat: lat, lon: lon }),
'latLon': JSON.stringify({ lat: latitude, lon: longitude }),
'kiosk': 'true'
});
return `${ws4kpBaseUrl}/?${ws4kpParams.toString()}`;
};
const weatherUrl = `${ws4kpBaseUrl}/?${ws4kpParams.toString()}`;
// Start geocoding immediately in the background - don't block anything
let lateGeocodePromise = null;
let initialUrl = 'data:text/html,<html><body style="margin:0;padding:0;background:#000"></body></html>'; // Dummy black page
console.log(`Weather stream requested for: ${city} (${lat}, ${lon})`);
if (city && city !== 'Toronto, ON, CAN') {
// Start geocoding - this will be the only URL we load
const geocodePromise = Promise.race([
geocodeCity(city),
new Promise((_, reject) => setTimeout(() => reject(new Error('Geocoding timeout')), 1000))
]).then(geoResult => {
console.log(`Geocoded: ${city} -> ${geoResult.displayName}`);
const finalUrl = buildUrl(geoResult.lat, geoResult.lon);
console.log(`URL: ${finalUrl}`);
return { url: finalUrl, lat: geoResult.lat, lon: geoResult.lon };
}).catch(error => {
// Geocoding timed out or failed - continue in background
return geocodeCity(city).then(geoResult => {
const finalUrl = buildUrl(geoResult.lat, geoResult.lon);
console.log(`Geocoding completed: ${geoResult.displayName} (${geoResult.lat}, ${geoResult.lon})`);
console.log(`Final URL: ${finalUrl}`);
return { url: finalUrl, lat: geoResult.lat, lon: geoResult.lon, isLate: true };
}).catch(err => {
console.warn(`Geocoding failed: ${err.message}`);
// Fall back to Toronto if geocoding completely fails
const fallbackUrl = buildUrl(43.6532, -79.3832);
return { url: fallbackUrl, lat: 43.6532, lon: -79.3832, isLate: true };
});
});
// Always wait for geocoding to complete (or timeout and continue in background)
lateGeocodePromise = geocodePromise.then(result => {
return result.url;
});
} else {
// Toronto - use directly
const lat = 43.6532;
const lon = -79.3832;
initialUrl = buildUrl(lat, lon);
console.log(`URL: ${initialUrl}`);
}
console.log(`Stream starting: ${city}`);
// Forward to the main stream endpoint WITH MUSIC
req.query.url = weatherUrl;
req.query.url = initialUrl;
req.query.width = width;
req.query.height = height;
req.query.fps = fps;
req.query.hideLogo = hideLogo;
// Call the stream handler with music enabled
return streamHandler(req, res, true);
// Call the stream handler with music enabled and late geocode promise
return streamHandler(req, res, true, lateGeocodePromise);
});
app.get('/health', (req, res) => {

8
package.json
View File

@@ -6,10 +6,16 @@
"scripts": {
"start": "node index.js"
},
"keywords": ["puppeteer", "ffmpeg", "hls", "streaming"],
"keywords": [
"puppeteer",
"ffmpeg",
"hls",
"streaming"
],
"author": "",
"license": "MIT",
"dependencies": {
"canvas": "^2.11.2",
"express": "^4.18.2",
"puppeteer": "^24.15.0"
}