ci: Enhance logging for troubleshooting test failures not due to broken links
Jason Stirnaman
parent
4ae1dec4be
commit
8a26400577
|
|
@ -47,7 +47,9 @@ describe('Article', () => {
|
|||
cy.wrap(Promise.all(urlPromises)).then((urls) => {
|
||||
subjects = urls;
|
||||
|
||||
cy.log(`📊 Cache Analysis: ${results.cacheStats.hitRate}% hit rate`);
|
||||
cy.log(
|
||||
`📊 Cache Analysis: ${results.cacheStats.hitRate}% hit rate`
|
||||
);
|
||||
cy.log(
|
||||
`🔄 Testing ${subjects.length} pages (${results.cacheStats.cacheHits} cached)`
|
||||
);
|
||||
|
|
@ -60,7 +62,23 @@ describe('Article', () => {
|
|||
})
|
||||
.catch((error) => {
|
||||
cy.log('❌ Error during incremental validation task: ' + error.message);
|
||||
Cypress.fail('Incremental validation task failed. See logs for details.');
|
||||
|
||||
// Provide more debugging information for validation failures
|
||||
cy.log('🔍 Validation Error Details:');
|
||||
cy.log(` • Error Type: ${error.name || 'Unknown'}`);
|
||||
cy.log(` • Error Message: ${error.message}`);
|
||||
if (error.stack) {
|
||||
const stackLines = error.stack.split('\n').slice(0, 3);
|
||||
cy.log(` • Stack Trace: ${stackLines.join(' -> ')}`);
|
||||
}
|
||||
cy.log(
|
||||
'💡 This error occurred during cache analysis or file validation setup'
|
||||
);
|
||||
cy.log(' Check that all files exist and are readable');
|
||||
|
||||
Cypress.fail(
|
||||
`Incremental validation task failed: ${error.message}. Check logs for details.`
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -170,30 +188,99 @@ describe('Article', () => {
|
|||
}
|
||||
|
||||
// Test implementation for subjects
|
||||
// Add debugging information about test subjects
|
||||
it('Test Setup Validation', function () {
|
||||
cy.log(`📋 Test Configuration:`);
|
||||
cy.log(` • Test subjects count: ${subjects.length}`);
|
||||
cy.log(` • Validation strategy: ${validationStrategy || 'Not set'}`);
|
||||
|
||||
if (subjects.length === 0) {
|
||||
cy.log('⚠️ No test subjects found - this may indicate:');
|
||||
cy.log(' • All files were cached and skipped');
|
||||
cy.log(' • No files matched the test criteria');
|
||||
cy.log(' • File mapping failed during setup');
|
||||
|
||||
// Don't fail if this is expected (cache hit scenario)
|
||||
const testSubjectsData = Cypress.env('test_subjects_data');
|
||||
if (testSubjectsData) {
|
||||
cy.log(
|
||||
'ℹ️ Test subjects data is available, cache optimization likely active'
|
||||
);
|
||||
} else {
|
||||
cy.log('❌ No test subjects data available - potential setup issue');
|
||||
}
|
||||
} else {
|
||||
cy.log(`✅ Ready to test ${subjects.length} pages`);
|
||||
subjects.slice(0, 5).forEach((subject) => cy.log(` • ${subject}`));
|
||||
if (subjects.length > 5) {
|
||||
cy.log(` ... and ${subjects.length - 5} more pages`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
subjects.forEach((subject) => {
|
||||
it(`${subject} has valid internal links`, function () {
|
||||
cy.visit(`${subject}`, { timeout: 20000 });
|
||||
// Add error handling for page visit failures
|
||||
cy.visit(`${subject}`, { timeout: 20000 })
|
||||
.then(() => {
|
||||
cy.log(`✅ Successfully loaded page: ${subject}`);
|
||||
})
|
||||
.catch((error) => {
|
||||
cy.log(`❌ Failed to load page: ${subject}`);
|
||||
cy.log(` • Error: ${error.message}`);
|
||||
cy.log('💡 This could indicate:');
|
||||
cy.log(' • Hugo server not running or crashed');
|
||||
cy.log(' • Invalid URL or routing issue');
|
||||
cy.log(' • Network connectivity problems');
|
||||
throw error; // Re-throw to fail the test properly
|
||||
});
|
||||
|
||||
// Test internal links
|
||||
cy.get('article, .api-content').then(($article) => {
|
||||
// Find links without failing the test if none are found
|
||||
const $links = $article.find('a[href^="/"]');
|
||||
if ($links.length === 0) {
|
||||
cy.log('No internal links found on this page');
|
||||
return;
|
||||
}
|
||||
cy.get('article, .api-content')
|
||||
.then(($article) => {
|
||||
// Find links without failing the test if none are found
|
||||
const $links = $article.find('a[href^="/"]');
|
||||
if ($links.length === 0) {
|
||||
cy.log('No internal links found on this page');
|
||||
return;
|
||||
}
|
||||
|
||||
// Now test each link
|
||||
cy.wrap($links).each(($a) => {
|
||||
const href = $a.attr('href');
|
||||
const linkText = $a.text().trim();
|
||||
testLink(href, linkText, subject);
|
||||
cy.log(`🔍 Testing ${$links.length} internal links on ${subject}`);
|
||||
|
||||
// Now test each link
|
||||
cy.wrap($links).each(($a) => {
|
||||
const href = $a.attr('href');
|
||||
const linkText = $a.text().trim();
|
||||
|
||||
try {
|
||||
testLink(href, linkText, subject);
|
||||
} catch (error) {
|
||||
cy.log(`❌ Error testing link ${href}: ${error.message}`);
|
||||
throw error; // Re-throw to fail the test
|
||||
}
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
cy.log(`❌ Error finding article content on ${subject}`);
|
||||
cy.log(` • Error: ${error.message}`);
|
||||
cy.log('💡 This could indicate:');
|
||||
cy.log(' • Page structure changed (missing article/.api-content)');
|
||||
cy.log(' • Page failed to render properly');
|
||||
cy.log(' • JavaScript errors preventing DOM updates');
|
||||
throw error;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it(`${subject} has valid anchor links`, function () {
|
||||
cy.visit(`${subject}`);
|
||||
cy.visit(`${subject}`)
|
||||
.then(() => {
|
||||
cy.log(`✅ Successfully loaded page for anchor testing: ${subject}`);
|
||||
})
|
||||
.catch((error) => {
|
||||
cy.log(`❌ Failed to load page for anchor testing: ${subject}`);
|
||||
cy.log(` • Error: ${error.message}`);
|
||||
throw error;
|
||||
});
|
||||
|
||||
// Define selectors for anchor links to ignore, such as behavior triggers
|
||||
const ignoreLinks = ['.tabs a[href^="#"]', '.code-tabs a[href^="#"]'];
|
||||
|
|
@ -208,6 +295,8 @@ describe('Article', () => {
|
|||
return;
|
||||
}
|
||||
|
||||
cy.log(`🔗 Testing ${$anchorLinks.length} anchor links on ${subject}`);
|
||||
|
||||
cy.wrap($anchorLinks).each(($a) => {
|
||||
const href = $a.prop('href');
|
||||
const linkText = $a.text().trim();
|
||||
|
|
@ -250,48 +339,82 @@ describe('Article', () => {
|
|||
return;
|
||||
}
|
||||
|
||||
cy.visit(`${subject}`);
|
||||
cy.visit(`${subject}`)
|
||||
.then(() => {
|
||||
cy.log(
|
||||
`✅ Successfully loaded page for external link testing: ${subject}`
|
||||
);
|
||||
})
|
||||
.catch((error) => {
|
||||
cy.log(
|
||||
`❌ Failed to load page for external link testing: ${subject}`
|
||||
);
|
||||
cy.log(` • Error: ${error.message}`);
|
||||
throw error;
|
||||
});
|
||||
|
||||
// Define allowed external domains to test
|
||||
const allowedExternalDomains = ['github.com', 'kapa.ai'];
|
||||
|
||||
// Test external links
|
||||
cy.get('article, .api-content').then(($article) => {
|
||||
// Find links without failing the test if none are found
|
||||
const $links = $article.find('a[href^="http"]');
|
||||
if ($links.length === 0) {
|
||||
cy.log('No external links found on this page');
|
||||
return;
|
||||
}
|
||||
|
||||
// Filter links to only include allowed domains
|
||||
const $allowedLinks = $links.filter((_, el) => {
|
||||
const href = el.getAttribute('href');
|
||||
try {
|
||||
const url = new URL(href);
|
||||
return allowedExternalDomains.some(
|
||||
(domain) =>
|
||||
url.hostname === domain || url.hostname.endsWith(`.${domain}`)
|
||||
);
|
||||
} catch (e) {
|
||||
return false;
|
||||
cy.get('article, .api-content')
|
||||
.then(($article) => {
|
||||
// Find links without failing the test if none are found
|
||||
const $links = $article.find('a[href^="http"]');
|
||||
if ($links.length === 0) {
|
||||
cy.log('No external links found on this page');
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
if ($allowedLinks.length === 0) {
|
||||
cy.log('No links to allowed external domains found on this page');
|
||||
return;
|
||||
}
|
||||
cy.log(
|
||||
`🔍 Found ${$links.length} total external links on ${subject}`
|
||||
);
|
||||
|
||||
cy.log(
|
||||
`Found ${$allowedLinks.length} links to allowed external domains to test`
|
||||
);
|
||||
cy.wrap($allowedLinks).each(($a) => {
|
||||
const href = $a.attr('href');
|
||||
const linkText = $a.text().trim();
|
||||
testLink(href, linkText, subject);
|
||||
// Filter links to only include allowed domains
|
||||
const $allowedLinks = $links.filter((_, el) => {
|
||||
const href = el.getAttribute('href');
|
||||
try {
|
||||
const url = new URL(href);
|
||||
return allowedExternalDomains.some(
|
||||
(domain) =>
|
||||
url.hostname === domain || url.hostname.endsWith(`.${domain}`)
|
||||
);
|
||||
} catch (urlError) {
|
||||
cy.log(`⚠️ Invalid URL found: ${href}`);
|
||||
return false;
|
||||
}
|
||||
});
|
||||
|
||||
if ($allowedLinks.length === 0) {
|
||||
cy.log('No links to allowed external domains found on this page');
|
||||
cy.log(
|
||||
` • Allowed domains: ${allowedExternalDomains.join(', ')}`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
cy.log(
|
||||
`🌐 Testing ${$allowedLinks.length} links to allowed external domains`
|
||||
);
|
||||
cy.wrap($allowedLinks).each(($a) => {
|
||||
const href = $a.attr('href');
|
||||
const linkText = $a.text().trim();
|
||||
|
||||
try {
|
||||
testLink(href, linkText, subject);
|
||||
} catch (error) {
|
||||
cy.log(
|
||||
`❌ Error testing external link ${href}: ${error.message}`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
cy.log(`❌ Error processing external links on ${subject}`);
|
||||
cy.log(` • Error: ${error.message}`);
|
||||
throw error;
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -393,6 +393,93 @@ async function main() {
|
|||
' This usually indicates test errors unrelated to link validation.'
|
||||
);
|
||||
|
||||
// Provide detailed failure analysis
|
||||
if (results) {
|
||||
console.warn('📊 Detailed Test Results:');
|
||||
console.warn(` • Total Tests: ${results.totalTests || 0}`);
|
||||
console.warn(` • Tests Passed: ${results.totalPassed || 0}`);
|
||||
console.warn(` • Tests Failed: ${results.totalFailed || 0}`);
|
||||
console.warn(` • Tests Pending: ${results.totalPending || 0}`);
|
||||
console.warn(` • Tests Skipped: ${results.totalSkipped || 0}`);
|
||||
console.warn(` • Duration: ${results.totalDuration || '0'}ms`);
|
||||
|
||||
// Show run-level information
|
||||
if (results.runs && results.runs.length > 0) {
|
||||
console.warn(` • Spec Files: ${results.runs.length}`);
|
||||
|
||||
// Show failures by spec file
|
||||
const failedRuns = results.runs.filter(
|
||||
(run) => run.stats?.failures > 0
|
||||
);
|
||||
if (failedRuns.length > 0) {
|
||||
console.warn('📋 Failed Spec Files:');
|
||||
failedRuns.forEach((run) => {
|
||||
console.warn(
|
||||
` • ${run.spec?.relative || run.spec?.name || 'Unknown spec'}`
|
||||
);
|
||||
if (run.stats) {
|
||||
console.warn(` - Failures: ${run.stats.failures}`);
|
||||
console.warn(` - Duration: ${run.stats.duration || 0}ms`);
|
||||
}
|
||||
|
||||
// Show test-level failures if available
|
||||
if (run.tests) {
|
||||
const failedTests = run.tests.filter(
|
||||
(test) => test.state === 'failed'
|
||||
);
|
||||
if (failedTests.length > 0) {
|
||||
console.warn(` - Failed Tests:`);
|
||||
failedTests.forEach((test, idx) => {
|
||||
if (idx < 3) {
|
||||
// Limit to first 3 failed tests per spec
|
||||
console.warn(` * ${test.title || 'Unnamed test'}`);
|
||||
if (test.err?.message) {
|
||||
// Truncate very long error messages
|
||||
const errorMsg =
|
||||
test.err.message.length > 200
|
||||
? test.err.message.substring(0, 200) + '...'
|
||||
: test.err.message;
|
||||
console.warn(` Error: ${errorMsg}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
if (failedTests.length > 3) {
|
||||
console.warn(
|
||||
` ... and ${failedTests.length - 3} more failed tests`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check for browser/system level issues
|
||||
if (results.browserName) {
|
||||
console.warn(
|
||||
` • Browser: ${results.browserName} ${results.browserVersion || ''}`
|
||||
);
|
||||
}
|
||||
|
||||
// Suggest common solutions
|
||||
console.warn('💡 Common Causes & Solutions:');
|
||||
console.warn(
|
||||
' • Page load timeouts: Check if Hugo server is responding properly'
|
||||
);
|
||||
console.warn(
|
||||
' • Network timeouts: Verify external link connectivity'
|
||||
);
|
||||
console.warn(
|
||||
' • Browser crashes: Check for memory or resource issues'
|
||||
);
|
||||
console.warn(
|
||||
' • Test logic errors: Review test assertions and selectors'
|
||||
);
|
||||
console.warn(
|
||||
` • Hugo server logs: Check ${HUGO_LOG_FILE} for errors`
|
||||
);
|
||||
}
|
||||
|
||||
// We should not consider special case domains (those with expected errors) as failures
|
||||
// but we'll still report other test failures
|
||||
cypressFailed = true;
|
||||
|
|
@ -408,9 +495,52 @@ async function main() {
|
|||
}
|
||||
} catch (err) {
|
||||
console.error(`❌ Cypress execution error: ${err.message}`);
|
||||
|
||||
// Provide more detailed error information
|
||||
if (err.stack) {
|
||||
console.error('📋 Error Stack Trace:');
|
||||
// Only show the first few lines of the stack trace to avoid overwhelming output
|
||||
const stackLines = err.stack.split('\n').slice(0, 5);
|
||||
stackLines.forEach((line) => console.error(` ${line}`));
|
||||
if (err.stack.split('\n').length > 5) {
|
||||
console.error(' ... (truncated)');
|
||||
}
|
||||
}
|
||||
|
||||
// Check if error is related to common issues
|
||||
const errorMsg = err.message.toLowerCase();
|
||||
if (errorMsg.includes('timeout') || errorMsg.includes('timed out')) {
|
||||
console.error('🕐 Timeout detected - possible causes:');
|
||||
console.error(
|
||||
' • Hugo server not responding (check if it started properly)'
|
||||
);
|
||||
console.error(' • Network connectivity issues');
|
||||
console.error(' • External links taking too long to respond');
|
||||
console.error(' • Page load timeouts (heavy pages or slow rendering)');
|
||||
} else if (
|
||||
errorMsg.includes('connection') ||
|
||||
errorMsg.includes('econnrefused')
|
||||
) {
|
||||
console.error('🔌 Connection issues detected:');
|
||||
console.error(' • Hugo server may not be running or accessible');
|
||||
console.error(` • Check if port ${HUGO_PORT} is available`);
|
||||
console.error(' • Firewall or network restrictions');
|
||||
} else if (errorMsg.includes('browser') || errorMsg.includes('chrome')) {
|
||||
console.error('🌐 Browser issues detected:');
|
||||
console.error(
|
||||
' • Chrome/browser may not be available in CI environment'
|
||||
);
|
||||
console.error(' • Browser crashed or failed to start');
|
||||
console.error(' • Insufficient memory or resources');
|
||||
}
|
||||
|
||||
console.error(
|
||||
`Check Hugo server logs at ${HUGO_LOG_FILE} for any server issues`
|
||||
`📝 Hugo server logs: Check ${HUGO_LOG_FILE} for server issues`
|
||||
);
|
||||
console.error('💡 Additional debugging steps:');
|
||||
console.error(' • Verify Hugo server started successfully');
|
||||
console.error(' • Check if test URLs are accessible manually');
|
||||
console.error(' • Review Cypress screenshots/videos if available');
|
||||
|
||||
// Still try to display broken links report if available
|
||||
displayBrokenLinksReport();
|
||||
|
|
@ -454,6 +584,21 @@ async function main() {
|
|||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(`Fatal error: ${err}`);
|
||||
console.error(`💥 Fatal error during test execution: ${err.message || err}`);
|
||||
|
||||
if (err.stack) {
|
||||
console.error('📋 Fatal Error Stack Trace:');
|
||||
console.error(err.stack);
|
||||
}
|
||||
|
||||
console.error(
|
||||
'🔍 This error occurred in the main test runner flow, not within Cypress tests.'
|
||||
);
|
||||
console.error('💡 Common causes:');
|
||||
console.error(' • File system permissions issues');
|
||||
console.error(' • Missing dependencies or modules');
|
||||
console.error(' • Hugo server startup failures');
|
||||
console.error(' • Process management errors');
|
||||
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue