Merge pull request #6527 from influxdata/jmercer/complete-plugin-sync-fix
Implements end-to-end automation for syncing plugin docs from influxdb3_plugins to docs-v2 with validation, transformation, and automated PR creation.revert-6527-jmercer/complete-plugin-sync-fix
Jameelah Mercer
GitHub
commit
6372de7994
|
|
@ -80,11 +80,15 @@ jobs:
|
|||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Checkout influxdb3_plugins (sparse)
|
||||
run: |
|
||||
git clone --filter=blob:none --sparse https://github.com/influxdata/influxdb3_plugins.git .ext/influxdb3_plugins
|
||||
cd .ext/influxdb3_plugins
|
||||
git sparse-checkout set influxdata/ scripts/
|
||||
git checkout ${{ steps.inputs.outputs.source_commit }}
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: influxdata/influxdb3_plugins
|
||||
token: ${{ secrets.PLUGINS_CONTENT_READ_TOKEN }}
|
||||
path: .ext/influxdb3_plugins
|
||||
sparse-checkout: |
|
||||
influxdata/
|
||||
scripts/
|
||||
ref: ${{ steps.inputs.outputs.source_commit }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
|
|
@ -172,24 +176,109 @@ jobs:
|
|||
|
||||
core.setFailed('Plugin validation failed');
|
||||
|
||||
- name: Debug - Check file structure
|
||||
if: steps.validate.outputs.validation_passed == 'true'
|
||||
run: |
|
||||
echo "======================================"
|
||||
echo "DEBUG: Checking file structure"
|
||||
echo "======================================"
|
||||
echo ""
|
||||
|
||||
echo "1. Current working directory:"
|
||||
pwd
|
||||
echo ""
|
||||
|
||||
echo "2. List workspace root:"
|
||||
ls -la
|
||||
echo ""
|
||||
|
||||
echo "3. Check if .ext exists:"
|
||||
if [ -d ".ext" ]; then
|
||||
echo "✅ .ext directory exists"
|
||||
ls -la .ext/
|
||||
else
|
||||
echo "❌ .ext directory NOT found"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "4. Check if .ext/influxdb3_plugins exists:"
|
||||
if [ -d ".ext/influxdb3_plugins" ]; then
|
||||
echo "✅ .ext/influxdb3_plugins exists"
|
||||
ls -la .ext/influxdb3_plugins/
|
||||
else
|
||||
echo "❌ .ext/influxdb3_plugins NOT found"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "5. Check if influxdata directory exists:"
|
||||
if [ -d ".ext/influxdb3_plugins/influxdata" ]; then
|
||||
echo "✅ influxdata directory exists"
|
||||
ls -la .ext/influxdb3_plugins/influxdata/
|
||||
else
|
||||
echo "❌ influxdata directory NOT found"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "6. Check if basic_transformation exists:"
|
||||
if [ -d ".ext/influxdb3_plugins/influxdata/basic_transformation" ]; then
|
||||
echo "✅ basic_transformation directory exists"
|
||||
ls -la .ext/influxdb3_plugins/influxdata/basic_transformation/
|
||||
else
|
||||
echo "❌ basic_transformation directory NOT found"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "7. Now checking from script directory:"
|
||||
cd docs-v2/helper-scripts/influxdb3-plugins
|
||||
echo "Current directory:"
|
||||
pwd
|
||||
echo ""
|
||||
|
||||
echo "8. Trying to access with ../../../.ext/influxdb3_plugins:"
|
||||
if [ -d "../../../.ext/influxdb3_plugins" ]; then
|
||||
echo "✅ Can access via ../../../.ext/influxdb3_plugins"
|
||||
ls -la ../../../.ext/influxdb3_plugins/
|
||||
else
|
||||
echo "❌ Cannot access via ../../../.ext/influxdb3_plugins"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "9. Checking all possible paths from script dir:"
|
||||
echo "Checking ../.ext/influxdb3_plugins:"
|
||||
ls -la ../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
|
||||
echo ""
|
||||
echo "Checking ../../.ext/influxdb3_plugins:"
|
||||
ls -la ../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
|
||||
echo ""
|
||||
echo "Checking ../../../.ext/influxdb3_plugins:"
|
||||
ls -la ../../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
|
||||
echo ""
|
||||
echo "Checking ../../../../.ext/influxdb3_plugins:"
|
||||
ls -la ../../../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
|
||||
echo ""
|
||||
|
||||
echo "======================================"
|
||||
echo "DEBUG: Complete"
|
||||
echo "======================================"
|
||||
|
||||
- name: Transform plugin documentation
|
||||
if: steps.validate.outputs.validation_passed == 'true'
|
||||
run: |
|
||||
cd docs-v2
|
||||
|
||||
# Set PLUGIN_DIR for the transformation script
|
||||
export INFLUXDB3_PLUGINS_PATH=".ext/influxdb3_plugins"
|
||||
|
||||
cd docs-v2/helper-scripts/influxdb3-plugins
|
||||
|
||||
# Set path to plugins repo (relative from script directory)
|
||||
export INFLUXDB3_PLUGINS_PATH="../../../.ext/influxdb3_plugins"
|
||||
|
||||
# Run the transformation
|
||||
if [[ "${{ steps.inputs.outputs.plugins }}" == "all" ]]; then
|
||||
node helper-scripts/influxdb3-plugins/port_to_docs.js
|
||||
node port_to_docs.js # ← CHANGE: Script is in current dir now
|
||||
else
|
||||
# Transform specific plugins
|
||||
IFS=',' read -ra PLUGIN_ARRAY <<< "${{ steps.inputs.outputs.plugins }}"
|
||||
for plugin in "${PLUGIN_ARRAY[@]}"; do
|
||||
plugin=$(echo "$plugin" | xargs) # trim whitespace
|
||||
echo "Transforming plugin: $plugin"
|
||||
node helper-scripts/influxdb3-plugins/port_to_docs.js --plugin "$plugin"
|
||||
node port_to_docs.js --plugin "$plugin"
|
||||
done
|
||||
fi
|
||||
|
||||
|
|
@ -201,9 +290,11 @@ jobs:
|
|||
- name: Setup Playwright
|
||||
run: |
|
||||
cd docs-v2
|
||||
npm install playwright
|
||||
npx playwright install chromium
|
||||
|
||||
- name: Generate screenshots
|
||||
if: false # Disable screenshots for MVP
|
||||
id: screenshots
|
||||
run: |
|
||||
cd docs-v2
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
plugins:
|
||||
basic_transformation:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/basic_transformation/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/basic_transformation/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/basic-transformation.md
|
||||
category: official
|
||||
additional_sections:
|
||||
|
|
@ -11,7 +11,7 @@ plugins:
|
|||
- logging
|
||||
|
||||
downsampler:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/downsampler/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/downsampler/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/downsampler.md
|
||||
category: official
|
||||
additional_sections:
|
||||
|
|
@ -19,63 +19,63 @@ plugins:
|
|||
- logging
|
||||
|
||||
forecast_error_evaluator:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/forecast_error_evaluator/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/forecast_error_evaluator/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/forecast-error-evaluator.md
|
||||
category: official
|
||||
additional_sections:
|
||||
- logging
|
||||
|
||||
influxdb_to_iceberg:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/influxdb_to_iceberg/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/influxdb_to_iceberg/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/influxdb-to-iceberg.md
|
||||
category: official
|
||||
additional_sections:
|
||||
- logging
|
||||
|
||||
mad_check:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/mad_check/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/mad_check/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/mad-check.md
|
||||
category: official
|
||||
additional_sections:
|
||||
- logging
|
||||
|
||||
notifier:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/notifier/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/notifier/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/notifier.md
|
||||
category: official
|
||||
additional_sections:
|
||||
- logging
|
||||
|
||||
prophet_forecasting:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/prophet_forecasting/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/prophet_forecasting/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/prophet-forecasting.md
|
||||
category: official
|
||||
additional_sections:
|
||||
- logging
|
||||
|
||||
state_change:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/state_change/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/state_change/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/state-change.md
|
||||
category: official
|
||||
additional_sections:
|
||||
- logging
|
||||
|
||||
stateless_adtk_detector:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/stateless_adtk_detector/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/stateless_adtk_detector/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/stateless-adtk-detector.md
|
||||
category: official
|
||||
additional_sections:
|
||||
- logging
|
||||
|
||||
system_metrics:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/system_metrics/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/system_metrics/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/system-metrics.md
|
||||
category: official
|
||||
additional_sections:
|
||||
- logging
|
||||
|
||||
threshold_deadman_checks:
|
||||
source: ../../.ext/influxdb3_plugins/influxdata/threshold_deadman_checks/README.md
|
||||
source: ../../../.ext/influxdb3_plugins/influxdata/threshold_deadman_checks/README.md
|
||||
target: ../../content/shared/influxdb3-plugins/plugins-library/official/threshold-deadman-checks.md
|
||||
category: official
|
||||
additional_sections:
|
||||
|
|
|
|||
|
|
@ -34,28 +34,84 @@ function removeEmojiMetadata(content) {
|
|||
return content.replace(pattern, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove level 1 heading from content.
|
||||
*/
|
||||
function removeTitleHeading(content) {
|
||||
// Title is in frontmatter, remove H1 from content
|
||||
return content.replace(/^#\s+.+$\n*/m, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove standalone Description heading.
|
||||
*/
|
||||
function removeDescriptionHeading(content) {
|
||||
// Remove "## Description" heading when it appears alone on a line
|
||||
content = content.replace(/^##\s+Description\s*$/m, '');
|
||||
return content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Expand common abbreviations for readability.
|
||||
*/
|
||||
function expandAbbreviations(content) {
|
||||
// Replace e.g., with "for example,"
|
||||
content = content.replace(/\be\.g\.,\s*/g, 'for example, ');
|
||||
// Replace i.e., with "that is,"
|
||||
content = content.replace(/\bi\.e\.,\s*/g, 'that is, ');
|
||||
return content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert README TOML links to internal section links.
|
||||
*/
|
||||
function convertTomlReadmeLinks(content) {
|
||||
// If document has TOML configuration section, link to it instead of external README
|
||||
if (content.includes('## Using TOML Configuration Files')) {
|
||||
content = content.replace(
|
||||
/\[([^\]]*TOML[^\]]*)\]\(https:\/\/github\.com\/influxdata\/influxdb3_plugins\/blob\/master\/README\.md\)/gi,
|
||||
'[$1](#using-toml-configuration-files)'
|
||||
);
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert relative links to GitHub URLs.
|
||||
*/
|
||||
function convertRelativeLinks(content, pluginName) {
|
||||
const baseUrl = `https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/${pluginName}/`;
|
||||
const rootUrl =
|
||||
'https://github.com/influxdata/influxdb3_plugins/blob/master/';
|
||||
|
||||
// Convert relative README links (../../README.md, ../README.md, etc.)
|
||||
content = content.replace(
|
||||
/\[([^\]]+)\]\((\.\.\/)+README\.md\)/g,
|
||||
`[$1](${rootUrl}README.md)`
|
||||
);
|
||||
|
||||
// Convert TOML file links
|
||||
content = content.replace(
|
||||
/\[([^\]]+\.toml)\]\(([^)]+\.toml)\)/g,
|
||||
(match, linkText, linkPath) => `[${linkText}](${baseUrl}${linkPath})`
|
||||
/\[([^\]]+\.toml)\]\(\.?\/?([^)]+\.toml)\)/g,
|
||||
(match, linkText, linkPath) => {
|
||||
const cleanPath = linkPath.replace(/^\.\//, '');
|
||||
return `[${linkText}](${baseUrl}${cleanPath})`;
|
||||
}
|
||||
);
|
||||
|
||||
// Convert Python file links
|
||||
content = content.replace(
|
||||
/\[([^\]]+\.py)\]\(([^)]+\.py)\)/g,
|
||||
(match, linkText, linkPath) => `[${linkText}](${baseUrl}${linkPath})`
|
||||
/\[([^\]]+\.py)\]\(\.?\/?([^)]+\.py)\)/g,
|
||||
(match, linkText, linkPath) => {
|
||||
const cleanPath = linkPath.replace(/^\.\//, '');
|
||||
return `[${linkText}](${baseUrl}${cleanPath})`;
|
||||
}
|
||||
);
|
||||
|
||||
// Convert main README reference
|
||||
content = content.replace(
|
||||
'[influxdb3_plugins/README.md](/README.md)',
|
||||
'[influxdb3_plugins/README.md](https://github.com/influxdata/influxdb3_plugins/blob/master/README.md)'
|
||||
`[influxdb3_plugins/README.md](${rootUrl}README.md)`
|
||||
);
|
||||
|
||||
return content;
|
||||
|
|
@ -214,7 +270,11 @@ Each downsampled record includes three additional metadata columns:
|
|||
function transformContent(content, pluginName, config) {
|
||||
// Apply transformations in order
|
||||
content = removeEmojiMetadata(content);
|
||||
content = removeTitleHeading(content);
|
||||
content = removeDescriptionHeading(content);
|
||||
content = convertRelativeLinks(content, pluginName);
|
||||
content = expandAbbreviations(content);
|
||||
content = convertTomlReadmeLinks(content);
|
||||
content = addProductShortcodes(content);
|
||||
content = enhanceOpeningParagraph(content);
|
||||
content = fixCodeBlockFormatting(content);
|
||||
|
|
|
|||
Loading…
Reference in New Issue