From 48d2f7b2f6151bb428797dd01eac32be65391223 Mon Sep 17 00:00:00 2001 From: Martin Bell Date: Sun, 25 Oct 2026 20:28:43 +0000 Subject: [PATCH 1/3] Add YouTube video availability check workflow and script --- .github/workflows/video-check.yml | 39 ++++++ package.json | 1 + scripts/check-youtube-videos.js | 207 ++++++++++++++++++++++++++++++ 3 files changed, 247 insertions(+) create mode 100644 .github/workflows/video-check.yml create mode 100644 scripts/check-youtube-videos.js diff --git a/.github/workflows/video-check.yml b/.github/workflows/video-check.yml new file mode 100644 index 00000000..12f5927e --- /dev/null +++ b/.github/workflows/video-check.yml @@ -0,0 +1,39 @@ +name: YouTube Video Availability Check + +on: + schedule: + - cron: "0 6 * * 1" # Every Monday at 6:00 AM UTC + workflow_dispatch: # Allow manual triggering + +jobs: + check-videos: + name: Check YouTube Video Availability + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + + - name: Install dependencies + run: npm install + + - name: Run video availability check + run: node scripts/check-youtube-videos.js + + - name: Upload results + uses: actions/upload-artifact@v4 + if: always() + with: + name: youtube-check-results + path: | + static/youtube-status.json + static/youtube-check-results.txt + retention-days: 30 + + - name: Report unavailable videos + if: failure() + run: | + echo "Some YouTube videos are unavailable:" + cat static/youtube-check-results.txt | grep "UNAVAILABLE" diff --git a/package.json b/package.json index 5e186ffb..eb79f8b9 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "get-tts-results": "node scripts/get-tts-results.js", "update-results": "node scripts/update-results.js", "set-points": "node scripts/set-points.js", + "check-youtube": "node scripts/check-youtube-videos.js", "start-marsh": "grunt build && concurrently \"grunt concurrent\" \"node scripts/firebase.js serve\"" }, "devDependencies": { diff --git a/scripts/check-youtube-videos.js b/scripts/check-youtube-videos.js new file mode 100644 index 00000000..0e3c0447 --- /dev/null +++ b/scripts/check-youtube-videos.js @@ -0,0 +1,207 @@ +// Video Availability Checker +// Periodically checks YouTube video availability for embedded videos in pages + +const fs = require('fs'); +const path = require('path'); +const childProcess = require('child_process'); + +const PAGE_DIR = path.join(__dirname, '../pages'); +const OUTPUT_FILE = path.join(__dirname, '../static/youtube-status.json'); +const LOG_FILE = path.join(__dirname, '../static/youtube-check-results.txt'); + +// Error patterns that indicate unavailable videos +const UNAVAILABLE_PATTERNS = [ + 'Video player configuration error', + 'Error 153', + 'this video is unavailable', + 'video has been removed', + 'private video', +]; + +/** + * Extract YouTube video IDs from page markdown files + * @returns {string[]} - Array of unique video IDs + */ +function extractVideoIds() { + const ids = new Set(); + + // Get all .md files in pages directory + const mdFiles = fs.readdirSync(PAGE_DIR).filter(f => f.endsWith('.md')); + + for (const file of mdFiles) { + const content = fs.readFileSync(path.join(PAGE_DIR, file), 'utf8'); + + // Extract video IDs from @youtube[] patterns + const regex = /@\[youtube\]\([^)]*?([a-zA-Z0-9_-]{11})[^)]*\)/g; + let match; + while ((match = regex.exec(content)) !== null) { + ids.add(match[1]); + } + } + + return Array.from(ids).sort(); +} + +/** + * Check if a video is available using the oEmbed API + * @param {string} videoId - YouTube video ID to check + * @returns {Promise<{available: boolean, videoId: string, error?: string}>} + */ +async function checkVideoAvailability(videoId) { + return new Promise((resolve) => { + const oembedUrl = `https://www.youtube.com/oembed?url=https://www.youtube.com/watch?v=${videoId}`; + + const cmd = `curl -s -w '\\n%{http_code}' '${oembedUrl}'`; + + childProcess.exec(cmd, { maxBuffer: 1024 * 1024 }, (error, stdout, stderr) => { + const lines = stdout.trim().split('\n'); + const httpCode = lines.pop(); // Last line is the HTTP status code + const body = lines.join('\n'); + + if (httpCode !== '200') { + resolve({ + available: false, + videoId, + error: `HTTP ${httpCode}: ${stderr || 'Video unavailable'}`, + }); + return; + } + + try { + const data = JSON.parse(body); + resolve({ + available: true, + videoId, + title: data.title, + author: data.author_name, + }); + } catch (parseError) { + resolve({ + available: false, + videoId, + error: 'Invalid JSON response from oEmbed API', + }); + } + }); + }); +} + +/** + * Check all videos and return results + * @param {string[]} videoIds - Array of video IDs to check + * @returns {Promise<{total: number, available: number, unavailable: number, results: Array}>} + */ +async function checkAllVideos(videoIds) { + const results = []; + + console.log(`Checking ${videoIds.length} video IDs...`); + + for (const videoId of videoIds) { + const result = await checkVideoAvailability(videoId); + results.push(result); + + // Progress indicator + if (results.length % 50 === 0) { + console.log(` Checked ${results.length}/${videoIds.length} videos...`); + } + } + + const available = results.filter((r) => r.available).length; + const unavailable = results.filter((r) => !r.available).length; + + return { + total: results.length, + available, + unavailable, + results, + }; +} + +/** + * Format results for display + * @param {{total: number, available: number, unavailable: number, results: Array}} results + * @param {boolean} includeTimestamp - Whether to include timestamp + * @returns {string} + */ +function formatResults(results, includeTimestamp = true) { + const lines = []; + + if (includeTimestamp) { + lines.push(`YouTube Video Availability Check - ${new Date().toISOString()}`); + lines.push(`========================================`); + } else { + lines.push(`YouTube Video Availability Check Results`); + lines.push(`================================`); + } + lines.push(`Total: ${results.total}`); + lines.push(`Available: ${results.available}`); + lines.push(`Unavailable: ${results.unavailable}`); + lines.push(''); + + if (results.results.length > 0) { + lines.push('Details:'); + lines.push('--------'); + + for (const result of results.results) { + const status = result.available ? '✓ AVAILABLE' : '✗ UNAVAILABLE'; + lines.push(`${status}: ${result.videoId}`); + lines.push(` Embed URL: https://www.youtube.com/embed/${result.videoId}`); + + if (result.available && result.title) { + lines.push(` Title: ${result.title}`); + } + if (!result.available && result.error) { + lines.push(` Error: ${result.error}`); + } + lines.push(''); + } + } + + return lines.join('\n'); +} + +/** + * Save results to output files + * @param {string} formattedResults + * @param {Array} resultsArray - The actual results array + */ +function saveResults(formattedResults, resultsArray) { + // Save to JSON file + const jsonOutput = { + timestamp: new Date().toISOString(), + total: resultsArray.total, + available: resultsArray.available, + unavailable: resultsArray.unavailable, + results: resultsArray.results, + }; + fs.writeFileSync(OUTPUT_FILE, JSON.stringify(jsonOutput, null, 2)); + console.log(`Results saved to ${OUTPUT_FILE}`); + + // Save to text log file + fs.writeFileSync(LOG_FILE, formattedResults); + console.log(`Log saved to ${LOG_FILE}`); +} + +// Main execution +async function run() { + const videoIds = extractVideoIds(); + const results = await checkAllVideos(videoIds); + const formatted = formatResults(results); + + console.log(formatted); + saveResults(formatted, results); + + // Exit with error if any videos are unavailable + if (results.unavailable > 0) { + console.log('\nWARNING: Some videos are unavailable!'); + process.exit(1); + } +} + +// Export for testing +module.exports = { extractVideoIds, checkVideoAvailability, checkAllVideos, formatResults, saveResults, run }; + +// Run if called directly +if (require.main === module) { + run().catch(console.error); +} From 943243c2148a6e5596c5c529ac7d57ea8f2eed87 Mon Sep 17 00:00:00 2001 From: Martin Bell Date: Sun, 25 Oct 2026 21:11:28 +0000 Subject: [PATCH 2/3] only check files in pages.yaml --- scripts/check-youtube-videos.js | 319 +++++++++++++++++++------------- 1 file changed, 187 insertions(+), 132 deletions(-) diff --git a/scripts/check-youtube-videos.js b/scripts/check-youtube-videos.js index 0e3c0447..55d80852 100644 --- a/scripts/check-youtube-videos.js +++ b/scripts/check-youtube-videos.js @@ -1,68 +1,117 @@ -// Video Availability Checker -// Periodically checks YouTube video availability for embedded videos in pages +// YouTube Video Availability Checker +// Checks only videos from .md files listed in static/pages.yaml const fs = require('fs'); const path = require('path'); const childProcess = require('child_process'); -const PAGE_DIR = path.join(__dirname, '../pages'); -const OUTPUT_FILE = path.join(__dirname, '../static/youtube-status.json'); -const LOG_FILE = path.join(__dirname, '../static/youtube-check-results.txt'); +const PROJECT_ROOT = path.resolve(__dirname, '..'); +const PAGES_DIR = path.join(PROJECT_ROOT, 'pages'); +const PAGES_YAML = path.join(PROJECT_ROOT, 'static/pages.yaml'); +const OUTPUT_JSON = path.join(PROJECT_ROOT, 'static/youtube-status.json'); +const OUTPUT_LOG = path.join(PROJECT_ROOT, 'static/youtube-check-results.txt'); -// Error patterns that indicate unavailable videos -const UNAVAILABLE_PATTERNS = [ - 'Video player configuration error', - 'Error 153', - 'this video is unavailable', - 'video has been removed', - 'private video', -]; +// YouTube oEmbed API URL +const OEMBED_URL = (id) => `https://www.youtube.com/oembed?url=https://www.youtube.com/watch?v=${id}`; /** - * Extract YouTube video IDs from page markdown files - * @returns {string[]} - Array of unique video IDs + * Parse static/pages.yaml to get list of pages + * @returns {Array<{url: string, title?: string}>} */ -function extractVideoIds() { - const ids = new Set(); +function getPagesFromYaml() { + const content = fs.readFileSync(PAGES_YAML, 'utf8'); + const pages = []; - // Get all .md files in pages directory - const mdFiles = fs.readdirSync(PAGE_DIR).filter(f => f.endsWith('.md')); + // Parse YAML manually - extract lines that start with " - url:" + const lines = content.split('\n'); + let inYearSection = false; - for (const file of mdFiles) { - const content = fs.readFileSync(path.join(PAGE_DIR, file), 'utf8'); + for (const line of lines) { + // Check if we're in a year section (year6:, year7:, etc.) + if (line.match(/^year\d+:$/)) { + inYearSection = true; + continue; + } - // Extract video IDs from @youtube[] patterns - const regex = /@\[youtube\]\([^)]*?([a-zA-Z0-9_-]{11})[^)]*\)/g; - let match; - while ((match = regex.exec(content)) !== null) { - ids.add(match[1]); + if (inYearSection) { + // Check if we've hit a new top-level section + if (line.match(/^[a-zA-Z]/)) { + inYearSection = false; + continue; + } + + // Extract url from " - url: filename" + const urlMatch = line.match(/^[ ]*- url: ([^\s]+)/); + if (urlMatch) { + pages.push({ url: urlMatch[1] }); + } else if (line.trim().startsWith('- url:')) { + const urlMatch2 = line.match(/- url: ([^\s]+)/); + if (urlMatch2) { + pages.push({ url: urlMatch2[1] }); + } + } } } - return Array.from(ids).sort(); + return pages; } /** - * Check if a video is available using the oEmbed API - * @param {string} videoId - YouTube video ID to check - * @returns {Promise<{available: boolean, videoId: string, error?: string}>} + * Get video IDs from a page's .md file + * @param {string} url - The URL slug from pages.yaml (without .md extension) + * @returns {Array<{id: string, ref: string}>} - Array of video ID objects */ -async function checkVideoAvailability(videoId) { - return new Promise((resolve) => { - const oembedUrl = `https://www.youtube.com/oembed?url=https://www.youtube.com/watch?v=${videoId}`; - - const cmd = `curl -s -w '\\n%{http_code}' '${oembedUrl}'`; +function getVideoIdsFromPage(url) { + const mdFile = path.join(PAGES_DIR, `${url}.md`); + + if (!fs.existsSync(mdFile)) { + return []; + } + + const content = fs.readFileSync(mdFile, 'utf8'); + const videoIds = []; + + // Match @youtube patterns like: @[youtube](videoId?...) or @[youtube](watch?v=videoId?...) + const regex = /@\[youtube\]\([^)]*\)/g; + let match; + + while ((match = regex.exec(content)) !== null) { + const youtubeRef = match[0]; - childProcess.exec(cmd, { maxBuffer: 1024 * 1024 }, (error, stdout, stderr) => { + // Extract video ID from various formats + // @[youtube](GhsZUZmJvaM?end=324&rel=0) + // @[youtube](watch?v=GFFcRY235fI?rel=0) + const idMatch = youtubeRef.match(/([a-zA-Z0-9_-]{11})/); + if (idMatch) { + videoIds.push({ + id: idMatch[1], + ref: youtubeRef, + }); + } + } + + return videoIds; +} + +/** + * Check video availability using YouTube oEmbed API + * @param {string} videoId - YouTube video ID + * @returns {Promise<{available: boolean, videoId: string, title?: string, error?: string}>} + */ +function checkVideoAvailability(videoId) { + return new Promise((resolve) => { + childProcess.exec(`curl -s -w '\\n%{http_code}' '${OEMBED_URL(videoId)}'`, { + maxBuffer: 1024 * 1024 + }, (error, stdout, stderr) => { const lines = stdout.trim().split('\n'); - const httpCode = lines.pop(); // Last line is the HTTP status code + const httpCode = lines.pop(); const body = lines.join('\n'); if (httpCode !== '200') { resolve({ available: false, videoId, - error: `HTTP ${httpCode}: ${stderr || 'Video unavailable'}`, + error: `HTTP ${httpCode}`, }); return; } @@ -73,13 +122,12 @@ async function checkVideoAvailability(videoId) { available: true, videoId, title: data.title, - author: data.author_name, }); - } catch (parseError) { + } catch { resolve({ available: false, videoId, - error: 'Invalid JSON response from oEmbed API', + error: 'Invalid JSON from oEmbed', }); } }); @@ -87,121 +135,128 @@ async function checkVideoAvailability(videoId) { } /** - * Check all videos and return results - * @param {string[]} videoIds - Array of video IDs to check - * @returns {Promise<{total: number, available: number, unavailable: number, results: Array}>} + * Main execution */ -async function checkAllVideos(videoIds) { - const results = []; +async function run() { + console.log('Reading pages from static/pages.yaml...'); + const pages = getPagesFromYaml(); + console.log(`Found ${pages.length} pages in pages.yaml`); - console.log(`Checking ${videoIds.length} video IDs...`); + // Collect all videos from pages listed in pages.yaml + const videos = []; + for (const page of pages) { + const pageVideos = getVideoIdsFromPage(page.url); + for (const video of pageVideos) { + videos.push({ + ...video, + pageUrl: page.url, + mdFilename: `${page.url}.md`, + }); + } + } - for (const videoId of videoIds) { - const result = await checkVideoAvailability(videoId); - results.push(result); + console.log(`Found ${videos.length} YouTube videos in pages listed in pages.yaml`); + + // Check each video + const results = []; + for (let i = 0; i < videos.length; i++) { + const video = videos[i]; + console.log(`[${i + 1}/${videos.length}] Checking ${video.id} (${video.mdFilename})...`); - // Progress indicator - if (results.length % 50 === 0) { - console.log(` Checked ${results.length}/${videoIds.length} videos...`); - } + const checkResult = await checkVideoAvailability(video.id); + results.push({ + ...checkResult, + mdFilename: video.mdFilename, + pageUrl: video.pageUrl, + ref: video.ref, + }); } - const available = results.filter((r) => r.available).length; - const unavailable = results.filter((r) => !r.available).length; + // Summary + const available = results.filter(r => r.available).length; + const unavailable = results.filter(r => !r.available).length; - return { - total: results.length, + const summary = { + timestamp: new Date().toISOString(), + pagesChecked: pages.length, + videosTotal: videos.length, available, unavailable, results, }; -} - -/** - * Format results for display - * @param {{total: number, available: number, unavailable: number, results: Array}} results - * @param {boolean} includeTimestamp - Whether to include timestamp - * @returns {string} - */ -function formatResults(results, includeTimestamp = true) { - const lines = []; - - if (includeTimestamp) { - lines.push(`YouTube Video Availability Check - ${new Date().toISOString()}`); - lines.push(`========================================`); - } else { - lines.push(`YouTube Video Availability Check Results`); - lines.push(`================================`); - } - lines.push(`Total: ${results.total}`); - lines.push(`Available: ${results.available}`); - lines.push(`Unavailable: ${results.unavailable}`); - lines.push(''); - - if (results.results.length > 0) { - lines.push('Details:'); - lines.push('--------'); - - for (const result of results.results) { - const status = result.available ? '✓ AVAILABLE' : '✗ UNAVAILABLE'; - lines.push(`${status}: ${result.videoId}`); - lines.push(` Embed URL: https://www.youtube.com/embed/${result.videoId}`); - - if (result.available && result.title) { - lines.push(` Title: ${result.title}`); - } - if (!result.available && result.error) { - lines.push(` Error: ${result.error}`); - } - lines.push(''); + + // Output + console.log('\n' + '='.repeat(60)); + console.log('YouTube Video Availability Check Results'); + console.log('='.repeat(60)); + console.log(`Pages in static/pages.yaml: ${pages.length}`); + console.log(`Total videos found: ${videos.length}`); + console.log(`Available: ${available}`); + console.log(`Unavailable: ${unavailable}`); + console.log('='.repeat(60)); + + if (unavailable > 0) { + console.log('\n❌ UNAVAILABLE VIDEOS:'); + for (const r of results.filter(r => !r.available)) { + console.log(` ${r.videoId} (${r.mdFilename})`); + if (r.error) console.log(` Error: ${r.error}`); } } - return lines.join('\n'); -} - -/** - * Save results to output files - * @param {string} formattedResults - * @param {Array} resultsArray - The actual results array - */ -function saveResults(formattedResults, resultsArray) { - // Save to JSON file - const jsonOutput = { - timestamp: new Date().toISOString(), - total: resultsArray.total, - available: resultsArray.available, - unavailable: resultsArray.unavailable, - results: resultsArray.results, - }; - fs.writeFileSync(OUTPUT_FILE, JSON.stringify(jsonOutput, null, 2)); - console.log(`Results saved to ${OUTPUT_FILE}`); + // Save results + fs.writeFileSync(OUTPUT_JSON, JSON.stringify(summary, null, 2)); + console.log(`\n✓ JSON results saved to: ${OUTPUT_JSON}`); - // Save to text log file - fs.writeFileSync(LOG_FILE, formattedResults); - console.log(`Log saved to ${LOG_FILE}`); -} - -// Main execution -async function run() { - const videoIds = extractVideoIds(); - const results = await checkAllVideos(videoIds); - const formatted = formatResults(results); + const logLines = [ + `YouTube Video Availability Check`, + `Generated: ${new Date().toISOString()}`, + ``, + `Pages in static/pages.yaml: ${pages.length}`, + `Total videos found: ${videos.length}`, + `Available: ${available}`, + `Unavailable: ${unavailable}`, + ``, + `Details:`, + `--------`, + ]; - console.log(formatted); - saveResults(formatted, results); + for (const r of results) { + const status = r.available ? '✓' : '✗'; + logLines.push(`${status} ${r.videoId} (${r.mdFilename})`); + if (r.available && r.title) { + logLines.push(` Title: ${r.title}`); + } + if (!r.available && r.error) { + logLines.push(` Error: ${r.error}`); + } + logLines.push(''); + } + + fs.writeFileSync(OUTPUT_LOG, logLines.join('\n')); + console.log(`✓ Log saved to: ${OUTPUT_LOG}`); // Exit with error if any videos are unavailable - if (results.unavailable > 0) { - console.log('\nWARNING: Some videos are unavailable!'); + if (unavailable > 0) { + console.log('\n⚠️ WARNING: Some videos are unavailable!'); process.exit(1); } + + console.log('\n✓ All videos are available!'); + process.exit(0); } // Export for testing -module.exports = { extractVideoIds, checkVideoAvailability, checkAllVideos, formatResults, saveResults, run }; +module.exports = { + getPagesFromYaml, + getVideoIdsFromPage, + checkVideoAvailability, + run, +}; // Run if called directly if (require.main === module) { - run().catch(console.error); + run().catch(err => { + console.error('Error:', err); + process.exit(1); + }); } From 20dec98eacddbc7738e2433f7ee5c9e13f451000 Mon Sep 17 00:00:00 2001 From: Martin Bell Date: Sun, 25 Oct 2026 21:53:35 +0000 Subject: [PATCH 3/3] tidy up some icky ai generated code --- scripts/check-youtube-videos.js | 44 +++++---------------------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/scripts/check-youtube-videos.js b/scripts/check-youtube-videos.js index 55d80852..fac6cde4 100644 --- a/scripts/check-youtube-videos.js +++ b/scripts/check-youtube-videos.js @@ -3,6 +3,7 @@ const fs = require('fs'); const path = require('path'); +const yaml = require('yamljs'); const childProcess = require('child_process'); const PROJECT_ROOT = path.resolve(__dirname, '..'); @@ -16,44 +17,13 @@ const OEMBED_URL = (id) => `https://www.youtube.com/oembed?url=https://www.youtu /** * Parse static/pages.yaml to get list of pages - * @returns {Array<{url: string, title?: string}>} + * @returns {Array<{url: string}>} */ function getPagesFromYaml() { - const content = fs.readFileSync(PAGES_YAML, 'utf8'); - const pages = []; + const pageData = yaml.load(PAGES_YAML); + const pages = [...pageData.year6, ...pageData.year7, ...pageData.year8, ...pageData.year9, ...pageData.year10, ...pageData.year11, ...pageData.test, ...pageData.homework]; - // Parse YAML manually - extract lines that start with " - url:" - const lines = content.split('\n'); - let inYearSection = false; - - for (const line of lines) { - // Check if we're in a year section (year6:, year7:, etc.) - if (line.match(/^year\d+:$/)) { - inYearSection = true; - continue; - } - - if (inYearSection) { - // Check if we've hit a new top-level section - if (line.match(/^[a-zA-Z]/)) { - inYearSection = false; - continue; - } - - // Extract url from " - url: filename" - const urlMatch = line.match(/^[ ]*- url: ([^\s]+)/); - if (urlMatch) { - pages.push({ url: urlMatch[1] }); - } else if (line.trim().startsWith('- url:')) { - const urlMatch2 = line.match(/- url: ([^\s]+)/); - if (urlMatch2) { - pages.push({ url: urlMatch2[1] }); - } - } - } - } - - return pages; + return pages.map(p => ({ url: p.url })); } /** @@ -149,7 +119,6 @@ async function run() { for (const video of pageVideos) { videos.push({ ...video, - pageUrl: page.url, mdFilename: `${page.url}.md`, }); } @@ -161,13 +130,12 @@ async function run() { const results = []; for (let i = 0; i < videos.length; i++) { const video = videos[i]; - console.log(`[${i + 1}/${videos.length}] Checking ${video.id} (${video.mdFilename})...`); + //console.log(`[${i + 1}/${videos.length}] Checking ${video.id} (${video.mdFilename})...`); const checkResult = await checkVideoAvailability(video.id); results.push({ ...checkResult, mdFilename: video.mdFilename, - pageUrl: video.pageUrl, ref: video.ref, }); }