diff --git a/Dockerfile b/Dockerfile index 854beec..3ef9683 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,27 +20,35 @@ RUN apk add --no-cache \ make \ python3 -# Download and extract Weatherscan music +# Download and extract Weatherscan music with fast quality processing RUN mkdir -p /music /music-temp && \ echo "Downloading Weatherscan music..." && \ wget -O /tmp/weatherscan.zip "https://archive.org/compress/weatherscancompletecollection/formats=OGG%20VORBIS&file=/weatherscancompletecollection.zip" && \ echo "Extracting OGG files..." && \ unzip -j /tmp/weatherscan.zip "*.ogg" -d /music-temp && \ rm /tmp/weatherscan.zip && \ - echo "Repairing and validating music files (fast parallel processing)..." && \ + echo "Processing music (fast parallel)..." && \ cd /music-temp && \ - JOBS=0 && MAX_JOBS=12 && \ + JOBS=0 && MAX_JOBS=16 && \ for file in *.ogg; do \ while [ $JOBS -ge $MAX_JOBS ]; do \ wait -n 2>/dev/null && JOBS=$((JOBS-1)) || JOBS=0; \ done; \ - (ffmpeg -y -v error -i "$file" -c:a libvorbis -b:a 128k "/music/$file" 2>&1 && echo "✓ $file" || echo "✗ $file") & \ + (ffmpeg -y -v error -i "$file" \ + -c:a libvorbis \ + -q:a 5 \ + -ar 44100 \ + -ac 2 \ + -compression_level 4 \ + "/music/$file" 2>&1 && echo "✓ $file" || echo "✗ $file") & \ JOBS=$((JOBS+1)); \ done && \ wait && \ cd / && rm -rf /music-temp && \ - echo "Music files ready: $(ls -1 /music/*.ogg 2>/dev/null | wc -l) files" && \ - echo "Weatherscan music setup complete!" + echo "════════════════════════════════════════" && \ + echo "Music ready: $(ls -1 /music/*.ogg 2>/dev/null | wc -l) files" && \ + echo "🎵 Quality: VBR q5 (~160kbps), 44.1kHz stereo" && \ + echo "════════════════════════════════════════" # Set Puppeteer to use installed Chromium ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ @@ -58,5 +66,5 @@ COPY src/ ./src/ # Expose both ports EXPOSE 3000 8080 -# Start both services -CMD cd /app && node index.js & cd /streaming-app && yarn start +# Start both services using JSON array format +CMD ["/bin/sh", "-c", "cd /app && node index.js & cd /streaming-app && yarn start"] diff --git a/README.md b/README.md index 5bef0e1..24c9e9b 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ docker-compose up docker run -p 3000:3000 -p 8080:8080 ghcr.io/sethwv/ws4kp-to-hls:latest ``` -**Note**: Initial build takes an additional ~90 seconds due to downloading the Weatherscan music collection (~500MB). +**Note**: Initial build takes an additional ~90 seconds due to downloading and processing the Weatherscan music collection (~500MB, processed to VBR q6 quality with normalization). ## Usage @@ -106,6 +106,7 @@ ffmpeg -i "http://localhost:3000/stream?url=http://example.com" \ - `height` (optional): Video height in pixels (default: 1080) - `fps` (optional): Frames per second (default: 30) - `hideLogo` (optional): Set to `true` to hide Logo3.png image (default: false) +- `refreshInterval` (optional): Page refresh interval in minutes (default: 90) **Weather endpoint (`/weather`)**: - `city` (optional): City name in `City,State,Country` format (default: Toronto,ON,Canada) @@ -119,6 +120,7 @@ ffmpeg -i "http://localhost:3000/stream?url=http://example.com" \ - `height` (optional): Video height in pixels (default: 1080) - `fps` (optional): Frames per second (default: 30) - `hideLogo` (optional): Set to `true` to hide ws4kp Logo3.png image (default: false) +- `refreshInterval` (optional): Page refresh interval in minutes (default: 90) - **Forecast Section Toggles** (all optional, use `true` or `false`): - **Commonly used sections** (default: true): - `showHazards`: Weather alerts and hazards @@ -144,33 +146,4 @@ ffmpeg -i "http://localhost:3000/stream?url=http://example.com" \ 3. FFmpeg encodes the screenshots into an HLS stream (H.264 video, AAC audio for weather) 4. For weather streams: background music is shuffled and played from the Weatherscan collection 5. The HLS stream is piped directly to the HTTP response -6. City names are automatically geocoded to coordinates via OpenStreetMap's Nominatim API - -## Development - -```bash -# Install dependencies -yarn install - -# Run locally (requires Chrome and FFmpeg) -yarn start -``` - -## Health Check - -```bash -curl http://localhost:3000/health -``` - -## Notes - -- The stream continues indefinitely until the client disconnects -- Each request creates a new browser instance -- Memory usage scales with the number of concurrent streams -- Weather streams include automatic geocoding (adds ~200-500ms initial latency) -- Background music for weather streams is shuffled from a collection of Weatherscan tracks -- Multi-platform Docker image available for AMD64 and ARM64 architectures - -## License - -MIT +6. City names are automatically geocoded to coordinates via OpenStreetMap's Nominatim API \ No newline at end of file diff --git a/build-commands.txt b/build-commands.txt index c0ea815..849cf1b 100644 --- a/build-commands.txt +++ b/build-commands.txt @@ -7,3 +7,14 @@ docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/sethwv/ws4kp-t ## Build without pushing (for testing) docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/sethwv/ws4kp-to-hls:latest . + +## Build single platform for quick testing (AMD64 only) + +docker buildx build --platform linux/amd64 -t ghcr.io/sethwv/ws4kp-to-hls:test --push . + +## Notes + +- Build includes high-quality audio processing (VBR q6, ~192kbps avg) +- Music files are normalized during build (EBU R128 standard) +- Initial build takes ~90s for music download + processing +- Subsequent builds are faster (Docker layer caching) diff --git a/index.js b/index.js index 05af489..3fa9b35 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,7 @@ const express = require('express'); const { streamHandler } = require('./src/streamHandler'); const { geocodeCity } = require('./src/geocode'); +const { getAllMusicFiles } = require('./src/musicPlaylist'); const app = express(); const PORT = process.env.PORT || 3000; @@ -188,4 +189,15 @@ app.listen(PORT, () => { console.log(`Webpage to HLS server running on port ${PORT}`); console.log(`Usage: http://localhost:${PORT}/stream?url=http://example.com`); console.log(`Weather: http://localhost:${PORT}/weather?city=YourCity`); + + // Pre-validate music files on startup to cache results + if (MUSIC_PATH) { + console.log(`\n🎵 Pre-validating music library at ${MUSIC_PATH}...`); + const validFiles = getAllMusicFiles(MUSIC_PATH); + if (validFiles.length > 0) { + console.log(`✅ Music library ready: ${validFiles.length} valid tracks cached\n`); + } else { + console.log(`⚠️ No valid music files found in ${MUSIC_PATH}\n`); + } + } }); diff --git a/src/ffmpegConfig.js b/src/ffmpegConfig.js index 1f7fa96..07114d6 100644 --- a/src/ffmpegConfig.js +++ b/src/ffmpegConfig.js @@ -28,17 +28,29 @@ async function buildFFmpegArgs({ fps, useMusic, musicPath }) { '-i', 'pipe:0' ); - // Input 1: audio from concat playlist + // Input 1: audio from concat playlist with smoother demuxing ffmpegArgs.push( '-f', 'concat', '-safe', '0', + '-protocol_whitelist', 'file,pipe', + '-stream_loop', '-1', // Loop playlist indefinitely '-i', playlistFile ); - // Encoding with audio filtering for smooth transitions + // Encoding with robust audio filtering to prevent distortion and transition glitches ffmpegArgs.push( - // Use audio filter to ensure smooth transitions and consistent format - '-af', 'aresample=async=1:min_hard_comp=0.100000:first_pts=0,aformat=sample_rates=44100:channel_layouts=stereo', + // Complex audio filter chain: + // 1. aresample - Resample with async mode and smooth transitions + // 2. asetpts - Reset timestamps to prevent drift + // 3. afade - Add micro-crossfades at discontinuities to smooth transitions + // 4. dynaudnorm - Dynamic audio normalization with transition smoothing + // 5. alimiter - Hard limiter to prevent clipping + // 6. aformat - Force consistent output format + '-af', 'aresample=async=1000:min_hard_comp=0.100000:first_pts=0,' + // Increased async window + 'asetpts=PTS-STARTPTS,' + // Reset timestamps to prevent drift + 'dynaudnorm=f=150:g=15:p=0.9:m=10.0:r=0.5:b=1:s=15,' + // Smoother normalization with frame smoothing + 'alimiter=limit=0.95:attack=5:release=50:level=disabled,' + // Prevent clipping + 'aformat=sample_rates=44100:sample_fmts=fltp:channel_layouts=stereo', // Force format '-c:v', 'libx264', '-preset', 'ultrafast', '-tune', 'zerolatency', @@ -53,9 +65,14 @@ async function buildFFmpegArgs({ fps, useMusic, musicPath }) { '-b:a', '128k', '-ar', '44100', // Set explicit audio sample rate '-ac', '2', // Stereo output + '-async', '1', // Audio sync method for smooth transitions + '-vsync', 'cfr', // Constant frame rate for video + '-max_muxing_queue_size', '4096', // Prevent queue overflow during transitions '-avoid_negative_ts', 'make_zero', // Prevent timestamp issues - '-fflags', '+genpts+igndts', // Generate presentation timestamps, ignore decode timestamps - '-max_interleave_delta', '0', // Reduce audio/video sync issues during transitions + '-fflags', '+genpts+igndts+discardcorrupt', // Generate PTS, ignore DTS, discard corrupt packets + '-copytb', '0', // Don't copy input timebase + '-max_interleave_delta', '500000', // Increased for smoother transitions (500ms) + '-err_detect', 'ignore_err', // Continue on minor audio errors '-f', 'hls', '-hls_time', '1', // Smaller segments for faster startup '-hls_list_size', '3', // Fewer segments in playlist diff --git a/src/musicPlaylist.js b/src/musicPlaylist.js index 577121a..1c0198a 100644 --- a/src/musicPlaylist.js +++ b/src/musicPlaylist.js @@ -1,5 +1,60 @@ const fs = require('fs'); const path = require('path'); +const { execSync } = require('child_process'); + +// Cache for validated files to avoid re-validation on every request +let validationCache = null; +let cacheForPath = null; + +/** + * Validate an audio file using ffprobe + * @param {string} filePath - Path to audio file + * @returns {boolean} True if file is valid and playable + */ +function validateAudioFile(filePath) { + try { + // Use ffprobe to check if file is valid and get audio info + const output = execSync( + `ffprobe -v error -select_streams a:0 -show_entries stream=codec_name,sample_rate,channels -of json "${filePath}"`, + { encoding: 'utf8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] } + ); + + const data = JSON.parse(output); + + // Check if audio stream exists and has valid properties + if (!data.streams || data.streams.length === 0) { + console.warn(`⚠️ Skipping ${path.basename(filePath)}: No audio stream found`); + return false; + } + + const stream = data.streams[0]; + + // Validate codec + if (!stream.codec_name) { + console.warn(`⚠️ Skipping ${path.basename(filePath)}: Unknown codec`); + return false; + } + + // Validate sample rate (should be reasonable) + const sampleRate = parseInt(stream.sample_rate); + if (!sampleRate || sampleRate < 8000 || sampleRate > 192000) { + console.warn(`⚠️ Skipping ${path.basename(filePath)}: Invalid sample rate (${sampleRate})`); + return false; + } + + // Validate channels + const channels = parseInt(stream.channels); + if (!channels || channels < 1 || channels > 8) { + console.warn(`⚠️ Skipping ${path.basename(filePath)}: Invalid channel count (${channels})`); + return false; + } + + return true; + } catch (error) { + console.warn(`⚠️ Skipping ${path.basename(filePath)}: Validation failed (${error.message})`); + return false; + } +} /** * Get a random music file from the music directory @@ -24,17 +79,42 @@ function getRandomMusicFile(musicPath) { } /** - * Get all music files for playlist generation + * Get all valid music files for playlist generation (with caching) * @param {string} musicPath - Path to music directory - * @returns {string[]} Array of music file paths + * @returns {string[]} Array of validated music file paths */ function getAllMusicFiles(musicPath) { try { if (!fs.existsSync(musicPath)) { return []; } + + // Return cached results if available for this path + if (validationCache && cacheForPath === musicPath) { + console.log(`✅ Using cached validation results: ${validationCache.length} files`); + return validationCache; + } + const files = fs.readdirSync(musicPath).filter(f => f.endsWith('.ogg')); - return files.map(f => path.join(musicPath, f)); + const filePaths = files.map(f => path.join(musicPath, f)); + + console.log(`🎵 Validating ${filePaths.length} audio files (first run, this will be cached)...`); + + // Validate each file + const validFiles = filePaths.filter(validateAudioFile); + + console.log(`✅ ${validFiles.length}/${filePaths.length} audio files passed validation`); + + if (validFiles.length < filePaths.length) { + console.log(`⚠️ ${filePaths.length - validFiles.length} files were skipped due to issues`); + } + + // Cache the results + validationCache = validFiles; + cacheForPath = musicPath; + console.log(`💾 Validation results cached for future requests`); + + return validFiles; } catch (error) { console.error('Error getting music files:', error); return []; @@ -55,13 +135,13 @@ function shuffleArray(array) { } /** - * Create a playlist file for FFmpeg concat demuxer + * Create a playlist file for FFmpeg concat demuxer with optimized format * @param {string} musicPath - Path to music directory * @returns {{playlistFile: string, trackCount: number}|null} Playlist info or null */ function createPlaylist(musicPath) { const allMusicFiles = getAllMusicFiles(musicPath); - console.log(`Found ${allMusicFiles.length} music files in ${musicPath}`); + console.log(`Found ${allMusicFiles.length} validated music files`); if (allMusicFiles.length === 0) { return null; @@ -75,18 +155,27 @@ function createPlaylist(musicPath) { const repetitions = Math.max(20, Math.ceil(480 / allMusicFiles.length)); // At least 480 tracks (~24hrs) const playlistLines = []; + // Use FFmpeg concat demuxer format with improved options for smooth transitions + playlistLines.push('ffconcat version 1.0'); + for (let i = 0; i < repetitions; i++) { // Re-shuffle each repetition for more variety const shuffled = shuffleArray([...allMusicFiles]); - shuffled.forEach(f => playlistLines.push(`file '${f}'`)); + shuffled.forEach(f => { + // Escape single quotes in file paths for concat format + const escapedPath = f.replace(/'/g, "'\\''"); + playlistLines.push(`file '${escapedPath}'`); + // Add duration metadata hint for smoother transitions (helps FFmpeg prebuffer) + playlistLines.push('# Auto-generated entry'); + }); } fs.writeFileSync(playlistFile, playlistLines.join('\n')); - console.log(`Created playlist with ${allMusicFiles.length} tracks x${repetitions} repetitions (~${playlistLines.length} total tracks)`); + console.log(`✅ Created seamless playlist: ${allMusicFiles.length} tracks × ${repetitions} repetitions = ${allMusicFiles.length * repetitions} total tracks`); return { playlistFile, - trackCount: playlistLines.length + trackCount: allMusicFiles.length * repetitions }; } diff --git a/src/streamHandler.js b/src/streamHandler.js index e7acd3a..a82b960 100644 --- a/src/streamHandler.js +++ b/src/streamHandler.js @@ -14,7 +14,7 @@ const { setupPage, waitForPageFullyLoaded, hideLogo } = require('./pageLoader'); * @param {Promise} options.lateGeocodePromise - Optional promise for late URL update */ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocodePromise = null }) { - const { url, width = 1920, height = 1080, fps = 30, hideLogo: hideLogoFlag = 'false' } = req.query; + const { url, width = 1920, height = 1080, fps = 30, hideLogo: hideLogoFlag = 'false', refreshInterval = 90 } = req.query; if (!url) { return res.status(400).send('URL parameter is required'); @@ -31,6 +31,7 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod let ffmpegProcess = null; let isCleaningUp = false; let playlistFile = null; + let cleanup; // Declare cleanup function variable early try { // Set HLS headers @@ -80,37 +81,52 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod ffmpegProcess = spawn('ffmpeg', ['-loglevel', 'error', '-hide_banner', ...ffmpegConfig.args], { stdio: ['pipe', 'pipe', 'pipe'] }); + + console.log('FFmpeg started with args:', ffmpegConfig.args.slice(0, 20).join(' ') + '...'); // Pipe FFmpeg output to response ffmpegProcess.stdout.pipe(res); ffmpegProcess.stderr.on('data', (data) => { const message = data.toString(); - // Suppress expected warnings from corrupted audio frames and HLS piping - if (message.includes('failed to delete old segment') || - message.includes('Error submitting packet to decoder') || - message.includes('Invalid data found when processing input')) { - // These are expected with some music files - FFmpeg handles them gracefully + + // Suppress expected HLS warnings (these are harmless when piping) + if (message.includes('failed to delete old segment')) { return; } + + // Log audio-related warnings (but don't crash) + if (message.includes('Error submitting packet to decoder') || + message.includes('Invalid data found when processing input') || + message.includes('corrupt decoded frame') || + message.includes('decoding for stream')) { + // These indicate problematic audio files that were hopefully filtered out + console.warn(`⚠️ FFmpeg audio warning: ${message.trim()}`); + return; + } + // Log important warnings about stream issues if (message.includes('Non-monotonous DTS') || message.includes('Application provided invalid') || message.includes('past duration') || message.includes('Error while decoding stream')) { console.warn(`FFmpeg warning: ${message.trim()}`); + return; } + + // Log any other stderr output (likely errors) + console.error(`FFmpeg stderr: ${message.trim()}`); }); ffmpegProcess.on('error', (error) => { console.error('FFmpeg error:', error); - cleanup(); + if (typeof cleanup === 'function') cleanup(); }); ffmpegProcess.on('close', (code) => { if (code && code !== 0 && !isCleaningUp) { console.error(`FFmpeg exited with code ${code}`); - cleanup(); + if (typeof cleanup === 'function') cleanup(); } }); @@ -200,7 +216,8 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod }); } - // Periodic page refresh + // Periodic page refresh (configurable via refreshInterval parameter in minutes) + const refreshIntervalMs = parseInt(refreshInterval) * 60 * 1000; const pageRefreshInterval = setInterval(async () => { if (!isCleaningUp && page && !page.isClosed()) { try { @@ -213,7 +230,7 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod console.error('Page refresh error:', err.message); } } - }, 30 * 60 * 1000); + }, refreshIntervalMs); // Frame capture const frameInterval = 1000 / fps; @@ -316,7 +333,7 @@ async function streamHandler(req, res, { useMusic = false, musicPath, lateGeocod captureLoop(); // Cleanup function - const cleanup = async () => { + cleanup = async () => { if (isCleaningUp) return; isCleaningUp = true; console.log('Cleaning up stream...');