seth-public/ws4kp-to-hls
1
0
Fork 0
Code Activity

More performance improvements

Browse Source
This commit is contained in:
sethwv-alt
2025-11-25 18:32:06 -05:00
parent 36ff1f2698
commit be7a047318
5 changed files with 161 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
index.js
View File

@@ -141,13 +141,16 @@ app.get('/weather', async (req, res) => {
timeFormat timeFormat
}; };
// Generate session ID for logging
const sessionId = Math.floor(Math.random() * 100000);
let lateGeocodePromise = null; let lateGeocodePromise = null;
let geocodeDataPromise = null; let geocodeDataPromise = null;
let initialUrl = 'data:text/html,<html><body style="margin:0;padding:0;background:#000"></body></html>'; let initialUrl = 'data:text/html,<html><body style="margin:0;padding:0;background:#000"></body></html>';
if (city && city !== 'Toronto, ON, CAN') { if (city && city !== 'Toronto, ON, CAN') {
// Start geocoding in background - don't wait for it // Start geocoding in background - don't wait for it
const geocodePromise = geocodeCity(city); const geocodePromise = geocodeCity(city, sessionId);
geocodeDataPromise = geocodePromise; // Save for city name overlay geocodeDataPromise = geocodePromise; // Save for city name overlay
// Always start with black screen immediately // Always start with black screen immediately
@@ -179,6 +182,10 @@ app.get('/weather', async (req, res) => {
// Call stream handler with music enabled // Call stream handler with music enabled
const { debug = DEBUG_MODE ? 'true' : 'false' } = req.query; const { debug = DEBUG_MODE ? 'true' : 'false' } = req.query;
// Build request path for logging
const requestPath = `/weather?city=${encodeURIComponent(city)}`;
return streamHandler(req, res, { return streamHandler(req, res, {
useMusic: true, useMusic: true,
musicPath: MUSIC_PATH, musicPath: MUSIC_PATH,
@@ -188,7 +195,9 @@ app.get('/weather', async (req, res) => {
defaultFps: DEFAULT_FPS, defaultFps: DEFAULT_FPS,
screenshotFormat: SCREENSHOT_FORMAT, screenshotFormat: SCREENSHOT_FORMAT,
screenshotQuality: SCREENSHOT_QUALITY, screenshotQuality: SCREENSHOT_QUALITY,
debugMode: debug === 'true' debugMode: debug === 'true',
sessionId,
requestPath
}); });
}); });

13
src/ffmpegConfig.js
View File

@@ -12,9 +12,10 @@ const { createPlaylist } = require('./musicPlaylist');
* @param {string} options.cityName - City name to display (optional) * @param {string} options.cityName - City name to display (optional)
* @param {number} options.videoWidth - Video width for centering text * @param {number} options.videoWidth - Video width for centering text
* @param {number} options.videoHeight - Video height for scaling text * @param {number} options.videoHeight - Video height for scaling text
* @param {string} options.sessionId - Session ID for logging (optional)
* @returns {Promise<{args: string[], playlistFile: string|null, hasMusic: boolean, captureFps: number}>} * @returns {Promise<{args: string[], playlistFile: string|null, hasMusic: boolean, captureFps: number}>}
*/ */
async function buildFFmpegArgs({ fps, useMusic, musicPath, inputFormat = 'jpeg', captureAtHigherFps = false, debugMode = false, cityName = null, videoWidth = 1920, videoHeight = 1080 }) { async function buildFFmpegArgs({ fps, useMusic, musicPath, inputFormat = 'jpeg', captureAtHigherFps = false, debugMode = false, cityName = null, videoWidth = 1920, videoHeight = 1080, sessionId = null }) {
const captureFps = captureAtHigherFps ? fps * 2 : fps; const captureFps = captureAtHigherFps ? fps * 2 : fps;
// Scale text sizes and positions based on video height (1080p as reference) // Scale text sizes and positions based on video height (1080p as reference)
@@ -41,7 +42,7 @@ async function buildFFmpegArgs({ fps, useMusic, musicPath, inputFormat = 'jpeg',
let hasMusic = false; let hasMusic = false;
if (useMusic) { if (useMusic) {
const playlistInfo = createPlaylist(musicPath); const playlistInfo = createPlaylist(musicPath, sessionId);
if (playlistInfo) { if (playlistInfo) {
playlistFile = playlistInfo.playlistFile; playlistFile = playlistInfo.playlistFile;
@@ -90,7 +91,7 @@ async function buildFFmpegArgs({ fps, useMusic, musicPath, inputFormat = 'jpeg',
...(captureAtHigherFps ? ['-r', fps.toString()] : []), ...(captureAtHigherFps ? ['-r', fps.toString()] : []),
'-tune', 'zerolatency', '-tune', 'zerolatency',
'-pix_fmt', 'yuv420p', '-pix_fmt', 'yuv420p',
'-g', fps.toString(), // Keyframe every 1 second for 1s segments '-g', (fps * 8).toString(), // Keyframe every 8 seconds for 8s segments
'-bf', '0', // No B-frames for lower latency '-bf', '0', // No B-frames for lower latency
'-x264opts', 'nal-hrd=cbr:no-scenecut', // Constant bitrate, no scene detection '-x264opts', 'nal-hrd=cbr:no-scenecut', // Constant bitrate, no scene detection
'-b:v', '2500k', // Target bitrate for stable encoding '-b:v', '2500k', // Target bitrate for stable encoding
@@ -109,7 +110,7 @@ async function buildFFmpegArgs({ fps, useMusic, musicPath, inputFormat = 'jpeg',
'-max_interleave_delta', '500000', // Increased for smoother transitions (500ms) '-max_interleave_delta', '500000', // Increased for smoother transitions (500ms)
'-err_detect', 'ignore_err', // Continue on minor audio errors '-err_detect', 'ignore_err', // Continue on minor audio errors
'-f', 'hls', '-f', 'hls',
'-hls_time', '1', // 1-second segments for faster startup '-hls_time', '8', // 8-second segments (standard HLS)
'-hls_list_size', '3', // Minimal segments for faster startup '-hls_list_size', '3', // Minimal segments for faster startup
'-hls_flags', 'omit_endlist+program_date_time+independent_segments', '-hls_flags', 'omit_endlist+program_date_time+independent_segments',
'-hls_segment_type', 'mpegts', '-hls_segment_type', 'mpegts',
@@ -141,14 +142,14 @@ async function buildFFmpegArgs({ fps, useMusic, musicPath, inputFormat = 'jpeg',
...(captureAtHigherFps ? ['-r', fps.toString()] : []), ...(captureAtHigherFps ? ['-r', fps.toString()] : []),
'-tune', 'zerolatency', '-tune', 'zerolatency',
'-pix_fmt', 'yuv420p', '-pix_fmt', 'yuv420p',
'-g', fps.toString(), // Keyframe every 1 second for 1s segments '-g', (fps * 8).toString(), // Keyframe every 8 seconds for 8s segments
'-bf', '0', '-bf', '0',
'-x264opts', 'nal-hrd=cbr:no-scenecut', '-x264opts', 'nal-hrd=cbr:no-scenecut',
'-b:v', '2500k', '-b:v', '2500k',
'-maxrate', '2500k', '-maxrate', '2500k',
'-bufsize', '5000k', '-bufsize', '5000k',
'-f', 'hls', '-f', 'hls',
'-hls_time', '1', // 1-second segments for faster startup '-hls_time', '8', // 8-second segments (standard HLS)
'-hls_list_size', '3', // Minimal segments for faster startup '-hls_list_size', '3', // Minimal segments for faster startup
'-hls_flags', 'omit_endlist+program_date_time+independent_segments', '-hls_flags', 'omit_endlist+program_date_time+independent_segments',
'-hls_segment_type', 'mpegts', '-hls_segment_type', 'mpegts',

133
src/geocode.js
View File

@@ -64,11 +64,13 @@ function saveCache(cache) {
} }
/** /**
* Get cached geocode data if available * Get cached geocode data if available (synchronous for immediate response)
* @param {string} cityQuery - City name * @param {string} cityQuery - City name
* @param {string} sessionId - Optional session ID for logging
* @returns {Object|null} Cached data or null * @returns {Object|null} Cached data or null
*/ */
function getCachedGeocode(cityQuery) { function getCachedGeocodeSync(cityQuery, sessionId = null) {
const prefix = sessionId ? `[${sessionId}] ` : '';
try { try {
const cache = loadCache(); const cache = loadCache();
const normalized = normalizeQuery(cityQuery); const normalized = normalizeQuery(cityQuery);
@@ -77,7 +79,7 @@ function getCachedGeocode(cityQuery) {
const locationKey = cache.queries[normalized]; const locationKey = cache.queries[normalized];
if (locationKey && cache.locations[locationKey]) { if (locationKey && cache.locations[locationKey]) {
const location = cache.locations[locationKey]; const location = cache.locations[locationKey];
console.log(`Geocode: ${cityQuery} -> ${location.cityName} (cached)`); console.log(`${prefix}Geocode: ${cityQuery} -> ${location.cityName} (cached)`);
return { return {
...location, ...location,
query: cityQuery // Return with original query query: cityQuery // Return with original query
@@ -143,69 +145,80 @@ function saveCachedGeocode(cityQuery, data) {
/** /**
* Geocode city to lat/lon using Nominatim (OpenStreetMap) * Geocode city to lat/lon using Nominatim (OpenStreetMap)
* @param {string} cityQuery - City name to geocode * @param {string} cityQuery - City name to geocode
* @param {string} sessionId - Optional session ID for logging
* @returns {Promise<{lat: number, lon: number, displayName: string}>} * @returns {Promise<{lat: number, lon: number, displayName: string}>}
*/ */
async function geocodeCity(cityQuery) { async function geocodeCity(cityQuery, sessionId = null) {
// Check cache first const prefix = sessionId ? `[${sessionId}] ` : '';
const cached = getCachedGeocode(cityQuery); // Check cache first - return immediately if found, don't block
if (cached) {
return cached;
}
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const encodedQuery = encodeURIComponent(cityQuery); // Check cache in nextTick to avoid blocking the event loop
const url = `https://nominatim.openstreetmap.org/search?q=${encodedQuery}&format=json&limit=1&addressdetails=1`; setImmediate(() => {
const cached = getCachedGeocodeSync(cityQuery, sessionId);
const options = { if (cached) {
headers: { return resolve(cached);
'User-Agent': 'webpage-to-hls-streaming-app/1.0'
} }
};
https.get(url, options, (res) => { // Not in cache, fetch from API
let data = ''; fetchFromAPI();
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
try {
const results = JSON.parse(data);
if (results && results.length > 0) {
const result = results[0];
const address = result.address || {};
// Extract clean city name - prefer city, town, village, municipality, or the name field
const cityName = address.city || address.town || address.village ||
address.municipality || address.hamlet || result.name ||
cityQuery.split(',')[0].trim();
const geocodeResult = {
query: cityQuery,
lat: parseFloat(result.lat),
lon: parseFloat(result.lon),
displayName: result.display_name,
cityName: cityName,
state: address.state || null,
stateDistrict: address.state_district || null,
county: address.county || null,
country: address.country || null,
countryCode: address.country_code ? address.country_code.toUpperCase() : null
};
console.log(`Geocode: ${cityQuery} -> ${geocodeResult.cityName}, ${geocodeResult.state || geocodeResult.country} (API)`);
// Save to cache
saveCachedGeocode(cityQuery, geocodeResult);
resolve(geocodeResult);
} else {
reject(new Error('No results found'));
}
} catch (error) {
reject(error);
}
});
}).on('error', (error) => {
reject(error);
}); });
function fetchFromAPI() {
const encodedQuery = encodeURIComponent(cityQuery);
const url = `https://nominatim.openstreetmap.org/search?q=${encodedQuery}&format=json&limit=1&addressdetails=1`;
const options = {
headers: {
'User-Agent': 'webpage-to-hls-streaming-app/1.0'
}
};
https.get(url, options, (res) => {
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
try {
const results = JSON.parse(data);
if (results && results.length > 0) {
const result = results[0];
const address = result.address || {};
// Extract clean city name - prefer city, town, village, municipality, or the name field
const cityName = address.city || address.town || address.village ||
address.municipality || address.hamlet || result.name ||
cityQuery.split(',')[0].trim();
const geocodeResult = {
query: cityQuery,
lat: parseFloat(result.lat),
lon: parseFloat(result.lon),
displayName: result.display_name,
cityName: cityName,
state: address.state || null,
stateDistrict: address.state_district || null,
county: address.county || null,
country: address.country || null,
countryCode: address.country_code ? address.country_code.toUpperCase() : null
};
console.log(`${prefix}Geocode: ${cityQuery} -> ${geocodeResult.cityName}, ${geocodeResult.state || geocodeResult.country} (API)`);
// Save to cache
saveCachedGeocode(cityQuery, geocodeResult);
resolve(geocodeResult);
} else {
reject(new Error('No results found'));
}
} catch (error) {
reject(error);
}
});
}).on('error', (error) => {
reject(error);
});
}
}); });
} }

6
src/musicPlaylist.js
View File

@@ -173,9 +173,11 @@ function initializeSharedPlaylist(musicPath) {
/** /**
* Create a rotated playlist starting at a random position * Create a rotated playlist starting at a random position
* @param {string} musicPath - Path to music directory * @param {string} musicPath - Path to music directory
* @param {string} sessionId - Optional session ID for logging
* @returns {{playlistFile: string, trackCount: number}|null} Playlist info or null * @returns {{playlistFile: string, trackCount: number}|null} Playlist info or null
*/ */
function createPlaylist(musicPath) { function createPlaylist(musicPath, sessionId = null) {
const prefix = sessionId ? `[${sessionId}] ` : '';
// Check if we have a shared playlist // Check if we have a shared playlist
if (!sharedPlaylistFile || !fs.existsSync(sharedPlaylistFile)) { if (!sharedPlaylistFile || !fs.existsSync(sharedPlaylistFile)) {
console.warn('Warning: Shared playlist not found, initializing...'); console.warn('Warning: Shared playlist not found, initializing...');
@@ -219,7 +221,7 @@ function createPlaylist(musicPath) {
} }
fs.writeFileSync(playlistFile, playlistLines.join('\n')); fs.writeFileSync(playlistFile, playlistLines.join('\n'));
console.log(`Stream playlist created, starting at position ${randomStart}/${allMusicFiles.length}`); console.log(`${prefix}Stream playlist created, starting at position ${randomStart}/${allMusicFiles.length}`);
return { return {
playlistFile, playlistFile,

78
src/streamHandler.js
View File

@@ -19,7 +19,7 @@ const { setupPage, waitForPageFullyLoaded, hideLogo, startAutoScroll } = require
* @param {number} options.screenshotQuality - JPEG quality (1-100) * @param {number} options.screenshotQuality - JPEG quality (1-100)
* @param {boolean} options.debugMode - Whether to show debug stats overlay * @param {boolean} options.debugMode - Whether to show debug stats overlay
*/ */
async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocodePromise = null, geocodeDataPromise = null, startTime = Date.now(), defaultFps = 30, screenshotFormat = 'jpeg', screenshotQuality = 95, debugMode = false }) { async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocodePromise = null, geocodeDataPromise = null, startTime = Date.now(), defaultFps = 30, screenshotFormat = 'jpeg', screenshotQuality = 95, debugMode = false, sessionId: providedSessionId = null, requestPath = null }) {
const { url, width = 1920, height = 1080, fps: fpsParam, hideLogo: hideLogoFlag = 'false', refreshInterval = 90, scroll = 'false', scrollPause = 30 } = req.query; const { url, width = 1920, height = 1080, fps: fpsParam, hideLogo: hideLogoFlag = 'false', refreshInterval = 90, scroll = 'false', scrollPause = 30 } = req.query;
// Parse numeric parameters - use query param if provided, otherwise use default from env // Parse numeric parameters - use query param if provided, otherwise use default from env
@@ -36,12 +36,19 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod
return res.status(400).send('Invalid URL'); return res.status(400).send('Invalid URL');
} }
// Use provided session ID or generate a new one
const sessionId = providedSessionId || Math.floor(Math.random() * 100000);
let streamId = `[${sessionId}]`;
// Log the actual request path if provided, otherwise the URL
const logUrl = requestPath || url;
console.log(`${streamId} New stream request: ${logUrl}`);
let browser = null; let browser = null;
let ffmpegProcess = null; let ffmpegProcess = null;
let isCleaningUp = false; let isCleaningUp = false;
let playlistFile = null; let playlistFile = null;
let cleanup; // Declare cleanup function variable early let cleanup; // Declare cleanup function variable early
let streamId = '[STREAM]'; // Default stream ID
let stopAutoScroll = null; // Function to stop auto-scrolling let stopAutoScroll = null; // Function to stop auto-scrolling
try { try {
@@ -53,7 +60,7 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod
// Build FFmpeg command and launch browser in parallel // Build FFmpeg command and launch browser in parallel
const useScroll = scroll === 'true'; const useScroll = scroll === 'true';
// Get city name if geocode data is available // Get city name from geocoding - wait for it to ensure overlay is correct
let cityName = null; let cityName = null;
if (geocodeDataPromise) { if (geocodeDataPromise) {
try { try {
@@ -71,9 +78,10 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod
inputFormat: screenshotFormat, inputFormat: screenshotFormat,
captureAtHigherFps: useScroll, // Capture at 2x FPS when scrolling for smoother output captureAtHigherFps: useScroll, // Capture at 2x FPS when scrolling for smoother output
debugMode, debugMode,
cityName, cityName, // Will be null if geocoding takes >50ms
videoWidth: parseInt(width), videoWidth: parseInt(width),
videoHeight: parseInt(height) videoHeight: parseInt(height),
sessionId
}); });
const browserPromise = puppeteer.launch({ const browserPromise = puppeteer.launch({
@@ -118,9 +126,6 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod
playlistFile = ffmpegConfig.playlistFile; playlistFile = ffmpegConfig.playlistFile;
const captureFps = ffmpegConfig.captureFps || fps; // Use capture FPS (may be 2x target FPS) const captureFps = ffmpegConfig.captureFps || fps; // Use capture FPS (may be 2x target FPS)
// Update stream identifier from browser process PID
streamId = `[${browser.process()?.pid || 'STREAM'}]`;
// Start FFmpeg immediately // Start FFmpeg immediately
ffmpegProcess = spawn('ffmpeg', ['-loglevel', 'error', '-hide_banner', ...ffmpegConfig.args], { ffmpegProcess = spawn('ffmpeg', ['-loglevel', 'error', '-hide_banner', ...ffmpegConfig.args], {
stdio: ['pipe', 'pipe', 'pipe'] stdio: ['pipe', 'pipe', 'pipe']
@@ -129,6 +134,55 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod
// Pipe FFmpeg output to response // Pipe FFmpeg output to response
ffmpegProcess.stdout.pipe(res); ffmpegProcess.stdout.pipe(res);
// Log HLS stream ready time
const hlsElapsed = ((Date.now() - startTime) / 1000).toFixed(2);
console.log(`${streamId} HLS stream ready (${hlsElapsed}s)`);
// Pre-fill buffer with black frames to create initial segments instantly
// This allows clients to start playback immediately (10 seconds worth)
const preFillSeconds = 10;
const preFillFrames = fps * preFillSeconds;
const preBlackFrame = (() => {
try {
const canvas = require('canvas');
const canvasObj = canvas.createCanvas(parseInt(width), parseInt(height));
const ctx = canvasObj.getContext('2d');
ctx.fillStyle = '#000000';
ctx.fillRect(0, 0, parseInt(width), parseInt(height));
if (screenshotFormat === 'png') {
return canvasObj.toBuffer('image/png');
} else {
return canvasObj.toBuffer('image/jpeg', { quality: 0.5 });
}
} catch (err) {
return null;
}
})();
// Write frames asynchronously to avoid blocking
if (preBlackFrame && preBlackFrame.length > 0 && ffmpegProcess && ffmpegProcess.stdin.writable) {
let framesPushed = 0;
const pushFrame = () => {
if (!ffmpegProcess || !ffmpegProcess.stdin.writable || framesPushed >= preFillFrames) {
return;
}
while (framesPushed < preFillFrames) {
const canWrite = ffmpegProcess.stdin.write(preBlackFrame);
framesPushed++;
if (!canWrite) {
// Wait for drain before continuing
ffmpegProcess.stdin.once('drain', pushFrame);
return;
}
}
};
// Start pushing frames immediately
setImmediate(pushFrame);
}
ffmpegProcess.stderr.on('data', (data) => { ffmpegProcess.stderr.on('data', (data) => {
const message = data.toString(); const message = data.toString();
@@ -198,7 +252,7 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod
.then(async (loaded) => { .then(async (loaded) => {
if (loaded) { if (loaded) {
const elapsed = ((Date.now() - startTime) / 1000).toFixed(2); const elapsed = ((Date.now() - startTime) / 1000).toFixed(2);
console.log(`${streamId} Stream ready (${elapsed}s)`); console.log(`${streamId} Page fully loaded (${elapsed}s)`);
sendBlackFrames = false; sendBlackFrames = false;
if (hideLogoFlag === 'true') { if (hideLogoFlag === 'true') {
await hideLogo(page); await hideLogo(page);
@@ -224,7 +278,7 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod
await waitForPageFullyLoaded(page, updatedUrl); await waitForPageFullyLoaded(page, updatedUrl);
const elapsed = ((Date.now() - startTime) / 1000).toFixed(2); const elapsed = ((Date.now() - startTime) / 1000).toFixed(2);
console.log(`${streamId} Stream ready (${elapsed}s)`); console.log(`${streamId} Page fully loaded (${elapsed}s)`);
waitingForCorrectUrl = false; waitingForCorrectUrl = false;
sendBlackFrames = false; sendBlackFrames = false;
@@ -244,7 +298,7 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod
try { try {
await waitForPageFullyLoaded(page, url); await waitForPageFullyLoaded(page, url);
const elapsed = ((Date.now() - startTime) / 1000).toFixed(2); const elapsed = ((Date.now() - startTime) / 1000).toFixed(2);
console.log(`${streamId} Stream ready (${elapsed}s)`); console.log(`${streamId} Page fully loaded (${elapsed}s)`);
waitingForCorrectUrl = false; waitingForCorrectUrl = false;
sendBlackFrames = false; sendBlackFrames = false;
if (hideLogoFlag === 'true') { if (hideLogoFlag === 'true') {
@@ -264,7 +318,7 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod
waitForPageFullyLoaded(page, url) waitForPageFullyLoaded(page, url)
.then(async () => { .then(async () => {
const elapsed = ((Date.now() - startTime) / 1000).toFixed(2); const elapsed = ((Date.now() - startTime) / 1000).toFixed(2);
console.log(`${streamId} Stream ready (${elapsed}s)`); console.log(`${streamId} Page fully loaded (${elapsed}s)`);
waitingForCorrectUrl = false; waitingForCorrectUrl = false;
sendBlackFrames = false; sendBlackFrames = false;
if (hideLogoFlag === 'true') { if (hideLogoFlag === 'true') {