diff --git a/.buildkite/scripts/generate-nightly-index.py b/.buildkite/scripts/generate-nightly-index.py index 4d28ec9619de9..f10cb2f0b6e21 100644 --- a/.buildkite/scripts/generate-nightly-index.py +++ b/.buildkite/scripts/generate-nightly-index.py @@ -9,6 +9,7 @@ import argparse import json import sys from dataclasses import asdict, dataclass +from datetime import datetime from pathlib import Path from typing import Any from urllib.parse import quote @@ -20,6 +21,7 @@ if not sys.version_info >= (3, 12): INDEX_HTML_TEMPLATE = """ + {items} @@ -90,7 +92,7 @@ def parse_from_filename(file: str) -> WheelFileInfo: ) -def generate_project_list(subdir_names: list[str]) -> str: +def generate_project_list(subdir_names: list[str], comment: str = "") -> str: """ Generate project list HTML content linking to each project & variant sub-directory. """ @@ -98,11 +100,14 @@ def generate_project_list(subdir_names: list[str]) -> str: for name in sorted(subdir_names): name = name.strip("/").strip(".") href_tags.append(f' {name}/
') - return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags)) + return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment) def generate_package_index_and_metadata( - wheel_files: list[WheelFileInfo], wheel_base_dir: Path, index_base_dir: Path + wheel_files: list[WheelFileInfo], + wheel_base_dir: Path, + index_base_dir: Path, + comment: str = "", ) -> tuple[str, str]: """ Generate package index HTML content for a specific package, linking to actual wheel files. @@ -120,7 +125,7 @@ def generate_package_index_and_metadata( file_meta = asdict(file) file_meta["path"] = file_path_quoted metadata.append(file_meta) - index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags)) + index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment) metadata_str = json.dumps(metadata, indent=2) return index_str, metadata_str @@ -131,6 +136,7 @@ def generate_index_and_metadata( index_base_dir: Path, default_variant: str | None = None, alias_to_default: str | None = None, + comment: str = "", ): """ Generate index for all wheel files. @@ -141,6 +147,7 @@ def generate_index_and_metadata( index_base_dir (Path): Base directory to store index files. default_variant (str | None): The default variant name, if any. alias_to_default (str | None): Alias variant name for the default variant, if any. + comment (str | None): Optional comment to include in the generated HTML files. First, parse all wheel files to extract metadata. We need to collect all wheel files for each variant, and generate an index for it (in a sub-directory). @@ -234,6 +241,10 @@ def generate_index_and_metadata( variant_to_files[alias_to_default] = variant_to_files["default"].copy() print(f"Alias variant '{alias_to_default}' created for default variant.") + # Generate comment in HTML header + comment_str = f" ({comment})" if comment else "" + comment_tmpl = f"Generated on {datetime.now().isoformat()}{comment_str}" + # Generate index for each variant subdir_names = set() for variant, files in variant_to_files.items(): @@ -253,7 +264,7 @@ def generate_index_and_metadata( subdir_names = subdir_names.union(packages) else: # generate project list for this variant directly - project_list_str = generate_project_list(sorted(packages)) + project_list_str = generate_project_list(sorted(packages), comment_tmpl) with open(variant_dir / "index.html", "w") as f: f.write(project_list_str) @@ -263,7 +274,7 @@ def generate_index_and_metadata( package_dir = variant_dir / package package_dir.mkdir(parents=True, exist_ok=True) index_str, metadata_str = generate_package_index_and_metadata( - package_files, wheel_base_dir, package_dir + package_files, wheel_base_dir, package_dir, comment ) with open(package_dir / "index.html", "w") as f: f.write(index_str) @@ -271,7 +282,7 @@ def generate_index_and_metadata( f.write(metadata_str) # Generate top-level project list index - project_list_str = generate_project_list(sorted(subdir_names)) + project_list_str = generate_project_list(sorted(subdir_names), comment_tmpl) with open(index_base_dir / "index.html", "w") as f: f.write(project_list_str) @@ -283,6 +294,7 @@ if __name__ == "__main__": --current-objects : path to JSON file containing current S3 objects listing in this version directory --output-dir : directory to store generated index files --alias-to-default : (optional) alias variant name for the default variant + --comment : (optional) comment string to include in generated HTML files """ parser = argparse.ArgumentParser( @@ -312,6 +324,12 @@ if __name__ == "__main__": default=None, help="Alias variant name for the default variant", ) + parser.add_argument( + "--comment", + type=str, + default="", + help="Optional comment string to include in generated HTML files", + ) args = parser.parse_args() @@ -366,5 +384,6 @@ if __name__ == "__main__": index_base_dir=index_base_dir, default_variant=None, alias_to_default=args.alias_to_default, + comment=args.comment.strip(), ) print(f"Successfully generated index and metadata in {output_dir}") diff --git a/.buildkite/scripts/upload-wheels.sh b/.buildkite/scripts/upload-wheels.sh index 0ac8fdd45bd0a..8e38ace0bfbc2 100644 --- a/.buildkite/scripts/upload-wheels.sh +++ b/.buildkite/scripts/upload-wheels.sh @@ -81,7 +81,10 @@ else alias_arg="" fi -$PYTHON pip install regex && .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" $alias_arg +# HACK: we do not need regex module here, but it is required by pre-commit hook +# To avoid any external dependency, we simply replace it back to the stdlib re module +sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py +$PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "commit $BUILDKITE_COMMIT" $alias_arg # copy indices to // unconditionally echo "Uploading indices to $S3_COMMIT_PREFIX"