From b33843feb41f424b905b8424e742848b45c73323 Mon Sep 17 00:00:00 2001 From: oxmc7769 Date: Sat, 4 Jul 2026 02:37:14 -0700 Subject: [PATCH] Support merged remove_projects manifests and add repo.yml docs local_manifests values are normally a single URL. The remove_projects key may instead be a list of URLs, merged/de-duplicated into a single manifest fragment rather than one file per key. Document repo.yml structure in README and add a fully commented example.repo.yml. --- README.md | 47 ++++++++++++++++++++++------ environ.py | 73 ++++++++++++++++++++++++++++++++++++++----- example.repo.yml | 58 ++++++++++++++++++++++++++++++++++ repo.yml | 19 ----------- rpivanilla-ensetup.sh | 13 -------- setup-ccache.sh | 33 ------------------- 6 files changed, 160 insertions(+), 83 deletions(-) create mode 100644 example.repo.yml delete mode 100644 repo.yml delete mode 100644 rpivanilla-ensetup.sh delete mode 100755 setup-ccache.sh diff --git a/README.md b/README.md index 45235d6..e982aea 100644 --- a/README.md +++ b/README.md @@ -19,17 +19,51 @@ python3 environ.py --sourceDir my-source --applyPatches ### Configuration -Create `repo.yml`: +Create `repo.yml` (or copy [`example.repo.yml`](example.repo.yml), which has this +same structure with full inline comments): + ```yaml repo: + # Upstream manifest repo, passed to `repo init -u`. url: "https://android.googlesource.com/platform/manifest" + # Tag/branch, passed to `repo init -b`. Must match an AOSP release tag. branch: "android-14.0.0_r1" + # Extra flags appended verbatim to `repo init`. options: - "--depth=1" - "--no-clone-bundle" + # Local manifest XML fragments. Downloaded with `requests` and written to + # /.repo/local_manifests/.xml after `repo init`, before + # `repo sync` — `repo` merges every *.xml in that directory into the manifest + # (adding projects, removing projects, overriding revisions, etc). + # + # Each value is either: + # - a single URL string -> downloaded as-is to ".xml" + # - a YAML list of URLs -> all downloaded and merged into one ".xml", + # combining child elements and de-duplicating ones that share the same + # tag + name attribute (first occurrence wins). Handy for splitting a + # long list across a common file and a device-specific one. local_manifests: device_manifest: "https://raw.githubusercontent.com/example/device-manifest/main/manifest.xml" vendor_manifest: "https://raw.githubusercontent.com/example/vendor-manifest/main/manifest.xml" + remove_projects: + - "https://raw.githubusercontent.com/example/manifests/main/common/remove_projects.xml" + - "https://raw.githubusercontent.com/example/manifests/main/device/remove_projects.xml" + +# Used with --applyPatches / --forceApplyPatches. +patches: + # Directories (relative to sourceDir) to search recursively for *.patch files, + # resolved after repo sync. Subdirectory layout mirrors AOSP project paths: + # patches/frameworks/base/*.patch -> applied at frameworks/base/ + # patches/packages/apps/Settings/*.patch -> applied at packages/apps/Settings/ + dirs: + - patches + + # Optional: maps a patch subdirectory to a different AOSP project path, for + # layouts that don't/shouldn't mirror the AOSP tree directly (e.g. grouping + # patches by Android version instead of by project path). + mappings: + android/36: frameworks/base ``` ### Command Line Options @@ -50,18 +84,11 @@ repo: | `--config ` | Path to repo configuration file (default: repo.yml) | | `--no-git-lfs` | Disable Git LFS setup (enabled by default) | -### Raspberry Pi Setup - -```bash -chmod +x rpivanilla-ensetup.sh -./rpivanilla-ensetup.sh -``` - ## Files - `environ.py` - Main setup script -- `rpivanilla-ensetup.sh` - Raspberry Pi environment setup -- `repo.yml` - Repository configuration +- `repo.yml` - Repository configuration (not tracked; create your own, see below) +- `example.repo.yml` - Fully commented example/reference for `repo.yml` - `scripts/better-patch.py` - Patch management - `patches/` - Patch files diff --git a/environ.py b/environ.py index 9749466..9c7010d 100755 --- a/environ.py +++ b/environ.py @@ -7,6 +7,7 @@ import argparse from pathlib import Path import yaml import requests +import xml.etree.ElementTree as ET # Import the patch handler from scripts.better_patch import GitPatchHandler @@ -105,25 +106,81 @@ def repo_init(source_dir, config): download_local_manifests(source_dir, repo_config['local_manifests']) def download_local_manifests(source_dir, manifests): - """Download local manifest files using requests.""" + """Download local manifest files using requests. + + Every key is normally a single URL, downloaded as-is to ".xml". The + "remove_projects" key is the one exception: it may instead be a YAML list + of URLs, all downloaded and merged into a single "remove_projects.xml" + fragment (their child elements combined, de-duplicated by name), e.g.: + + remove_projects: + - ".../common/remove_projects.xml" + - ".../rpi/extra_remove_projects.xml" + """ manifests_dir = os.path.join(source_dir, ".repo", "local_manifests") os.makedirs(manifests_dir, exist_ok=True) - - for manifest_name, manifest_url in manifests.items(): + + for manifest_name, manifest_value in manifests.items(): + if manifest_name == "remove_projects" and isinstance(manifest_value, list): + download_and_merge_manifest_fragments(manifests_dir, manifest_name, manifest_value) + continue + manifest_path = os.path.join(manifests_dir, f"{manifest_name}.xml") - print(f"Downloading local manifest: {manifest_name} from {manifest_url}") - + print(f"Downloading local manifest: {manifest_name} from {manifest_value}") + try: - response = requests.get(manifest_url, timeout=30) + response = requests.get(manifest_value, timeout=30) response.raise_for_status() - + with open(manifest_path, 'w') as f: f.write(response.text) print(f"Successfully downloaded {manifest_name}.xml") - + except requests.exceptions.RequestException as e: print(f"Failed to download {manifest_name}.xml: {e}") +def download_and_merge_manifest_fragments(manifests_dir, manifest_name, urls): + """Download multiple manifest fragments for one local_manifests key and merge + them into a single ".xml", de-duplicating child elements that + share the same tag + name attribute (first occurrence wins).""" + combined_root = ET.Element("manifest") + seen = set() + any_success = False + + for url in urls: + print(f"Downloading local manifest: {manifest_name} from {url}") + try: + response = requests.get(url, timeout=30) + response.raise_for_status() + fragment_root = ET.fromstring(response.text) + except requests.exceptions.RequestException as e: + print(f"Failed to download {manifest_name}.xml ({url}): {e}") + continue + except ET.ParseError as e: + print(f"Failed to parse {manifest_name}.xml ({url}): {e}") + continue + + any_success = True + for elem in fragment_root: + name = elem.get("name") + if name is not None: + key = (elem.tag, name) + if key in seen: + print(f" Skipping duplicate {elem.tag}: {name}") + continue + seen.add(key) + combined_root.append(elem) + + if not any_success: + print(f"Failed to download any fragments for {manifest_name}.xml") + return + + manifest_path = os.path.join(manifests_dir, f"{manifest_name}.xml") + tree = ET.ElementTree(combined_root) + ET.indent(tree, space=" ") + tree.write(manifest_path, encoding="unicode", xml_declaration=True) + print(f"Successfully wrote combined {manifest_name}.xml ({len(combined_root)} entries from {len(urls)} source(s))") + def setup_git_lfs(source_dir): """Set up Git LFS in the repository.""" print("Setting up Git LFS...") diff --git a/example.repo.yml b/example.repo.yml new file mode 100644 index 0000000..511e8b6 --- /dev/null +++ b/example.repo.yml @@ -0,0 +1,58 @@ +# Example repo.yml for environ.py. +# +# Copy this to repo.yml (or point --config at your own copy) and fill in real +# URLs/branches. This example is based on the PawletOS Raspberry Pi manifest: +# android_local_manifest/rpi/repo-shallow.yml + +repo: + # Upstream manifest repo passed to `repo init -u`. + url: "https://android.googlesource.com/platform/manifest" + # Tag/branch passed to `repo init -b`. Must match an AOSP release tag. + branch: "android-16.0.0_r4" + # Extra flags appended verbatim to `repo init`. + options: + - "--depth=1" # Shallow clone: history-less checkout, faster/smaller sync. + # Local manifest XML fragments, fetched with `requests` and written into + # /.repo/local_manifests/.xml after `repo init`, before `repo sync`. + # `repo` reads every *.xml file in that directory and layers them on top of + # the upstream manifest (adding projects, removing projects, overriding revisions). + # + # Each value is either: + # - a single URL string -> downloaded as-is to ".xml" + # - a YAML list of URLs -> all downloaded and merged into one ".xml", + # combining their child elements and de-duplicating entries that share the + # same tag + name attribute (first occurrence wins). Useful for splitting a + # long list across a common file and a device-specific one. + local_manifests: + # Adds PawletOS's own repo/project definitions common to all devices. + manifest_common_pawletos: "https://git.oxmc.me/PawletOS/android_local_manifest/raw/branch/android-16.0/common/pawletos.xml" + # Adds Broadcom vendor projects needed for Raspberry Pi (VideoCore/graphics, etc.). + manifest_brcm_rpi: "https://git.oxmc.me/PawletOS/android_local_manifest/raw/branch/android-16.0/rpi/vendor/brcm_rpi.xml" + # Adds LineageOS projects PawletOS builds on top of. + manifest_lineageos: "https://git.oxmc.me/PawletOS/android_local_manifest/raw/branch/android-16.0/common/vendor/lineageos.xml" + # Adds PawletOS's Raspberry Pi-specific device/project definitions. + manifest_pawletos: "https://git.oxmc.me/PawletOS/android_local_manifest/raw/branch/android-16.0/rpi/pawletos.xml" + # entries: AOSP projects unused on this target that should + # NOT be synced (saves time/disk; e.g. other-vendor devices, darwin prebuilts). + # Listed as a YAML list so a device-specific removal file can be appended + # alongside the common one, e.g.: + # remove_projects: + # - ".../common/remove_projects.xml" + # - ".../rpi/remove_projects.xml" + remove_projects: + - "https://git.oxmc.me/PawletOS/android_local_manifest/raw/branch/android-16.0/common/remove_projects.xml" + +patches: + # Directories (relative to source root) to search for patches, used with + # `environ.py --applyPatches`. Each entry is resolved inside the AOSP source + # directory after repo sync. Subdirectory layout mirrors AOSP project paths: + # vendor/pawlet/patches/frameworks/base/*.patch -> applied at frameworks/base/ + # vendor/pawlet/patches/packages/apps/Settings/*.patch -> applied at packages/apps/Settings/ + dirs: + - vendor/pawlet/patches + + # Optional: maps a patch subdirectory name to a different AOSP project path, + # for cases where the patch directory layout can't/shouldn't mirror the AOSP + # tree directly (e.g. grouping patches by Android version instead). + # mappings: + # android/36: frameworks/base diff --git a/repo.yml b/repo.yml deleted file mode 100644 index 54fd040..0000000 --- a/repo.yml +++ /dev/null @@ -1,19 +0,0 @@ -repo: - url: "https://android.googlesource.com/platform/manifest" - branch: "android-16.0.0_r4" - options: - - "--depth=1" - local_manifests: - manifest_brcm_rpi: "https://git.oxmc.me/PawletOS/android_local_manifest/raw/branch/android-16.0/vendors/brcm_rpi.xml" - manifest_lineageos: "https://git.oxmc.me/PawletOS/android_local_manifest/raw/branch/android-16.0/vendors/lineageos.xml" - manifest_pawletos: "https://git.oxmc.me/PawletOS/android_local_manifest/raw/branch/android-16.0/pawletos.xml" - remove_projects: "https://git.oxmc.me/PawletOS/android_local_manifest/raw/branch/android-16.0/remove_projects.xml" - -patches: - # Directories (relative to source root) to search for patches. - # Each entry is resolved inside the AOSP source directory after repo sync. - # Subdirectory layout mirrors AOSP project paths: - # vendor/pawlet/patches/frameworks/base/*.patch -> applied at frameworks/base/ - # vendor/pawlet/patches/packages/apps/Settings/*.patch -> applied at packages/apps/Settings/ - dirs: - - vendor/pawlet/patches \ No newline at end of file diff --git a/rpivanilla-ensetup.sh b/rpivanilla-ensetup.sh deleted file mode 100644 index f52a3bc..0000000 --- a/rpivanilla-ensetup.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Change dir -cd ~/RaspberryVanilla - -# sourch files -source ~/RaspberryVanilla/build/envsetup.sh - -# export -#export SOONG_ALLOW_MISSING_DEPENDENCIES=true # Used for multi disk build -#export OUT_DIR=/mnt/256gb-ssd/aosp_build # Used for multi disk build - -echo "Env setup, lunch than build" diff --git a/setup-ccache.sh b/setup-ccache.sh deleted file mode 100755 index e8f54d7..0000000 --- a/setup-ccache.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Configure ccache directory and size. -# NOTE: Android 13+ build system blocks CC_WRAPPER/CXX_WRAPPER — ccache cannot -# wrap the compiler. CCACHE_DIR is set for reference only; Soong handles caching. -# $1 = cache size limit (default: 30G) - -CCACHE_PATH=/mnt/ssd/ccache -CACHE_SIZE="${1:-30G}" -BASHRC="$HOME/.bashrc" - -mkdir -p "$CCACHE_PATH" - -CCACHE_DIR="$CCACHE_PATH" ccache -M "$CACHE_SIZE" -CCACHE_DIR="$CCACHE_PATH" ccache --set-config=compression=true -CCACHE_DIR="$CCACHE_PATH" ccache --set-config=compression_level=1 - -export CCACHE_DIR="$CCACHE_PATH" - -if ! grep -q 'CCACHE_DIR' "$BASHRC"; then - cat >> "$BASHRC" << 'EOF' - -# ccache dir (Android 16 — compiler wrapping not supported, Soong handles caching) -export CCACHE_DIR=/mnt/ssd/ccache -EOF - echo "CCACHE_DIR written to $BASHRC" -else - # Remove any CC_WRAPPER/CXX_WRAPPER/USE_CCACHE/CCACHE_EXEC lines if present - sed -i '/CC_WRAPPER\|CXX_WRAPPER\|USE_CCACHE\|CCACHE_EXEC/d' "$BASHRC" - echo "Cleaned up unsupported ccache wrapper vars from $BASHRC" -fi - -echo "" -CCACHE_DIR="$CCACHE_PATH" ccache -s