diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml new file mode 100644 index 00000000..15fca6dc --- /dev/null +++ b/.github/workflows/build-images.yaml @@ -0,0 +1,113 @@ +permissions: + actions: none + checks: none + contents: none + deployments: none + issues: write + packages: none + pull-requests: none + repository-projects: none + security-events: none + statuses: none + +on: + workflow_dispatch: + inputs: + version: + description: 'chromium version' + required: false + default: '' + schedule: + - cron: '0 1 * * *' + +env: + VERSION: ${{ github.event.inputs.version }} + NEW_VERSION: ${{ null }} + +name: Builds and pushes tagged image to DockerHub +jobs: + generate_build_deps: + name: Generate Build Deps + runs-on: ubuntu-latest + steps: + - name: Reclaiming disk space on / by removing dotnet/android/ghc + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo apt-get remove google-cloud-sdk azure-cli google-chrome-stable \ + firefox mysql-server-core-8.0 mono-devel podman \ + powershell + sudo apt-get autoremove + + - name: Checkout repo + uses: actions/checkout@v2 + with: + path: cromite + fetch-depth: 1 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Check versions + shell: bash + run: | + if [ -z $VERSION ]; then + VERSION=$(curl -s https://omahaproxy.appspot.com/all.json | jq '.[] | select(.os | contains("win64")) | .versions[] | select(.channel | contains("stable")) | .current_version' | xargs) + echo "VERSION=$VERSION" >> $GITHUB_ENV + fi + + - name: Building build-deps container ${{ env.VERSION }} + shell: bash + run: | + IS_PRESENT=$(docker inspect --type=image uazo/build-deps:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + IS_PRESENT=$(docker manifest inspect uazo/build-deps:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + DOCKER_BUILDKIT=1 docker build -t uazo/build-deps:$VERSION \ + --progress plain \ + --build-arg VERSION=$VERSION \ + --build-arg HTTP_PROXY="$PROXY_ADDR" \ + --no-cache \ + cromite/tools/images/build-deps/. + + docker push uazo/build-deps:$VERSION + echo "NEW_VERSION=$VERSION" >> $GITHUB_ENV + fi + fi + + - name: Building chromium container ${{ env.VERSION }} + shell: bash + run: | + IS_PRESENT=$(docker inspect --type=image uazo/chromium:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + IS_PRESENT=$(docker manifest inspect uazo/chromium:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + DOCKER_BUILDKIT=1 docker build -t uazo/chromium:$VERSION \ + --progress plain \ + --build-arg VERSION=$VERSION \ + --build-arg HTTP_PROXY="$PROXY_ADDR" \ + cromite/tools/images/chr-source/. + + docker push uazo/chromium:$VERSION + echo "NEW_VERSION=$VERSION" >> $GITHUB_ENV + fi + fi + + - name: Create issue ${{ env.VERSION }} + if: "${{ env.NEW_VERSION != '' }}" + shell: bash + run: | + GH=../../gh_2.18.1_linux_amd64/bin/gh + wget https://github.com/cli/cli/releases/download/v2.18.1/gh_2.18.1_linux_amd64.tar.gz + tar xfz gh_2.18.1_linux_amd64.tar.gz + + cd cromite/tools + echo ${{ secrets.GITHUB_TOKEN }} | $GH auth login --with-token + $GH issue create -t "$VERSION: new chromium version" -b "" + + + diff --git a/.github/workflows/build_bromite_dev.yaml b/.github/workflows/build_bromite_dev.yaml new file mode 100644 index 00000000..5aa81b7b --- /dev/null +++ b/.github/workflows/build_bromite_dev.yaml @@ -0,0 +1,347 @@ +name: Build Bromite +permissions: + actions: none + checks: none + contents: none + deployments: none + issues: none + packages: none + pull-requests: none + repository-projects: none + security-events: none + statuses: none + +on: + workflow_dispatch: + inputs: + sha: + description: 'uazo/bromite SHA' + required: true + default: '' + target_os: + description: 'targetos [android/win/all]' + required: true + default: 'all' + build: + description: 'android arch [arm64/x64/all]' + required: true + default: 'all' + type: + description: 'runner? [dev/ci]' + required: true + default: 'ci' + debug: + description: 'debug? [true/false]' + required: true + default: 'false' + clangd: + description: 'clangd? [true/false]' + required: true + default: 'false' + +env: + BROMITE_SHA: ${{ github.event.inputs.sha }} + REMOVEDOCKERSUPPORT: true + USELOCALIMAGE: true + +jobs: + check_images: + runs-on: ${{ github.event.inputs.type }} + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + path: bromite-buildtools + fetch-depth: 1 + + - name: Enable proxy on container + shell: bash + run: | + if ! [[ -z "${HTTP_PROXY}" ]]; then + PROXY_ADDR=http://$(hostname -I | cut -d' ' -f1 | xargs):8118 + echo "PROXY_ADDR=$PROXY_ADDR" >> $GITHUB_ENV + sudo iptables -D INPUT -p tcp -s localhost --dport 8118 -j ACCEPT + sudo iptables -D INPUT -p tcp --dport 8118 -j DROP + fi + + - name: Get current chromium version + shell: bash + run: | + mkdir bromite + cd bromite + git init + git remote add origin https://github.com/uazo/bromite + git fetch origin $BROMITE_SHA + git reset --hard FETCH_HEAD + cd .. + + export VERSION=$( cat ./bromite/build/RELEASE ) + rm -rf bromite + + echo Current version is $VERSION + echo "VERSION=$VERSION" >> $GITHUB_ENV + + cd bromite-buildtools + + - name: Building build-deps container ${{ env.VERSION }} + shell: bash + run: | + IS_PRESENT=$(docker inspect --type=image uazo/build-deps:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + IS_PRESENT=$(docker manifest inspect uazo/build-deps:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + DOCKER_BUILDKIT=1 docker build -t uazo/build-deps:$VERSION \ + --progress plain \ + --build-arg VERSION=$VERSION \ + --build-arg HTTP_PROXY="$PROXY_ADDR" \ + --no-cache \ + bromite-buildtools/images/build-deps/. + fi + fi + + - name: Building chromium container ${{ env.VERSION }} + shell: bash + run: | + IS_PRESENT=$(docker inspect --type=image uazo/chromium:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + IS_PRESENT=$(docker manifest inspect uazo/chromium:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + DOCKER_BUILDKIT=1 docker build -t uazo/chromium:$VERSION \ + --progress plain \ + --build-arg VERSION=$VERSION \ + --build-arg HTTP_PROXY="$PROXY_ADDR" \ + bromite-buildtools/images/chr-source/. + fi + fi + + - name: Building bromite container ${{ env.VERSION }}-${{ env.BROMITE_SHA }} + shell: bash + run: | + IS_PRESENT=$(docker inspect --type=image uazo/bromite:$VERSION-$BROMITE_SHA > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + IS_PRESENT=$(docker manifest inspect uazo/bromite:$VERSION-$BROMITE_SHA > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + DOCKER_BUILDKIT=1 docker build -t uazo/bromite:$VERSION-$BROMITE_SHA --progress plain \ + --build-arg BROMITE_SHA=$BROMITE_SHA \ + --build-arg VERSION=$VERSION \ + --build-arg HTTP_PROXY="$PROXY_ADDR" \ + bromite-buildtools/images/bromite-source/. + fi + fi + + - name: Building bromite-build container ${{ env.VERSION }}-${{ env.BROMITE_SHA }} + shell: bash + run: | + IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:$VERSION-$BROMITE_SHA > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + IS_PRESENT=$(docker manifest inspect uazo/bromite-build:$VERSION-$BROMITE_SHA > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + DOCKER_BUILDKIT=1 docker build -t uazo/bromite-build:$VERSION-$BROMITE_SHA --progress plain \ + --build-arg BROMITE_SHA=$BROMITE_SHA \ + --build-arg VERSION=$VERSION \ + --build-arg HTTP_PROXY="$PROXY_ADDR" \ + --no-cache \ + bromite-buildtools/images/bromite-build/. + fi + fi + + - name: Mark image to build + shell: bash + run: | + IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:build > /dev/null ; echo $?) + if [ $IS_PRESENT -eq "0" ]; then + docker rmi uazo/bromite-build:build + fi + docker tag uazo/bromite-build:$VERSION-$BROMITE_SHA uazo/bromite-build:build + + build: + runs-on: ${{ github.event.inputs.type }} + needs: check_images + if: success() + timeout-minutes: 1200 + + container: + image: uazo/bromite-build:build + env: + REMOVEDOCKERSUPPORT: true # CUSTOM RUNNER: remove sharing of docker socket + USELOCALIMAGE: true # CUSTOM RUNNER: permit use of local images + USEINTERNALNETWORK: true # CUSTOM RUNNER: create the docker network as internal + WORKSPACE: /home/lg/working_dir + # kythe + KYTHE_CORPUS: chromium.googlesource.com/chromium/src + KYTHE_ROOT_DIRECTORY: /home/lg/working_dir/chromium/src + KYTHE_OUTPUT_DIRECTORY: /home/lg/working_dir/chromium/src/out/bromite/kythe + # cross build + DEPOT_TOOLS_WIN_TOOLCHAIN_BASE_URL: /win_sdk/10.0.22621.0/ + WINDOWSSDKDIR: "/win_sdk/10.0.22621.0/Windows Kits/10/" + GYP_MSVS_OVERRIDE_PATH: /win_sdk/10.0.22621.0/ + # compile in debug mode + TARGET_ISDEBUG: ${{ github.event.inputs.debug }} + TARGET_OS: ${{ github.event.inputs.target_os }} + volumes: + - /storage/images/android/${{ github.event.inputs.sha }}/${{ github.event.inputs.debug }}/arm64:/home/lg/working_dir/chromium/src/out/bromite + - /storage/images/android/${{ github.event.inputs.sha }}/${{ github.event.inputs.debug }}/x64:/home/lg/working_dir/chromium/src/out/bromite_x64 + - /storage/images/win/x64/${{ github.event.inputs.sha }}:/home/lg/working_dir/chromium/src/out/bromite_win + - /tmp/proxy:/tmp/proxy + - /win_sdk:/win_sdk + + steps: + - name: Prepare Build Container + shell: bash + run: | + # set workspace paths + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE + + # reset proxy env + HTTP_PROXY= + HTTPS_PROXY= + http_proxy= + https_proxy= + + # set out folder permissions + test -d chromium/src/out/bromite || sudo mkdir -p chromium/src/out/bromite && \ + sudo chown lg chromium/src/out && + sudo chown lg chromium/src/out/bromite + + test -d chromium/src/out/bromite_win || sudo mkdir -p chromium/src/out/bromite_win && \ + sudo chown lg chromium/src/out && + sudo chown lg chromium/src/out/bromite_win + + test -d chromium/src/out/bromite_x64 || sudo mkdir -p chromium/src/out/bromite_x64 && \ + sudo chown lg chromium/src/out && + sudo chown lg chromium/src/out/bromite_x64 + + # make kythe output directory + test -d $KYTHE_OUTPUT_DIRECTORY || mkdir -p $KYTHE_OUTPUT_DIRECTORY + + sudo mkdir -p /run/user/1000/ + sudo chown lg /run/user/1000/ + sudo chmod g-rxw /run/user/1000/ + sudo chmod o-rxw /run/user/1000/ + + - name: Build Bromite Android arm64 + if: ${{ (github.event.inputs.target_os == 'android' || github.event.inputs.target_os == 'all') && (github.event.inputs.build == 'arm64' || github.event.inputs.build == 'all') }} + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + + echo "::group::-------- gn gen" + gn gen --args="import(\"/home/lg/working_dir/bromite/build/bromite.gn_args\") $(cat ../../build_args.gni) target_cpu = \"arm64\" " out/bromite + echo "::endgroup::" + + echo "::group::-------- gn args" + gn args out/bromite/ --list --short + gn args out/bromite/ --list >out/bromite/gn_list + echo "::endgroup::" + + autoninja -C out/bromite chrome_public_apk + + cp ../../bromite/build/RELEASE out/bromite + + - name: Get ninja logs Android arm64 + if: ${{ (github.event.inputs.target_os == 'android' || github.event.inputs.target_os == 'all') && (github.event.inputs.build == 'arm64' || github.event.inputs.build == 'all') }} + shell: bash + run: | + cd $WORKSPACE + $WORKSPACE/ninjatracing/ninjatracing $WORKSPACE/chromium/src/out/bromite/.ninja_log >$WORKSPACE/chromium/src/out/bromite/ninja_log_trace.json + python3 $WORKSPACE/chromium/src/third_party/catapult/tracing/bin/trace2html $WORKSPACE/chromium/src/out/bromite/ninja_log_trace.json + + - name: Build Bromite Windows + if: ${{ github.event.inputs.target_os == 'win' || github.event.inputs.target_os == 'all' }} + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + + echo "::group::-------- gn gen" + gn gen --args="import(\"/home/lg/working_dir/bromite/build/bromite.gn_args\") target_os = \"win\" $(cat ../../build_args.gni) target_cpu = \"x64\" " out/bromite_win + echo "::endgroup::" + + echo "::group::-------- gn args" + gn args out/bromite_win/ --list --short + gn args out/bromite_win/ --list >out/bromite_win/gn_list + echo "::endgroup::" + + autoninja -C out/bromite_win chrome + + cp ../../bromite/build/RELEASE out/bromite_win + + - name: Get ninja logs Windows + if: ${{ github.event.inputs.target_os == 'win' || github.event.inputs.target_os == 'all' }} + shell: bash + run: | + cd $WORKSPACE + $WORKSPACE/ninjatracing/ninjatracing $WORKSPACE/chromium/src/out/bromite_win/.ninja_log >$WORKSPACE/chromium/src/out/bromite_win/ninja_log_trace.json + python3 $WORKSPACE/chromium/src/third_party/catapult/tracing/bin/trace2html $WORKSPACE/chromium/src/out/bromite_win/ninja_log_trace.json + + - name: Build Bromite Android x64 + if: ${{ (github.event.inputs.target_os == 'android' || github.event.inputs.target_os == 'all') && (github.event.inputs.build == 'x64' || github.event.inputs.build == 'all') }} + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + + echo "::group::-------- gn gen" + gn gen --args="import(\"/home/lg/working_dir/bromite/build/bromite.gn_args\") $(cat ../../build_args.gni) target_cpu = \"x64\" " out/bromite_x64 + echo "::endgroup::" + + echo "::group::-------- gn args" + gn args out/bromite_x64/ --list --short + gn args out/bromite_x64/ --list >out/bromite_x64/gn_list + echo "::endgroup::" + + autoninja -C out/bromite_x64 chrome_public_apk + + cp ../../bromite/build/RELEASE out/bromite_x64 + + - name: Get ninja logs Android x64 + if: ${{ (github.event.inputs.target_os == 'android' || github.event.inputs.target_os == 'all') && (github.event.inputs.build == 'x64' || github.event.inputs.build == 'all') }} + shell: bash + run: | + cd $WORKSPACE + $WORKSPACE/ninjatracing/ninjatracing $WORKSPACE/chromium/src/out/bromite_x64/.ninja_log >$WORKSPACE/chromium/src/out/bromite_x64/ninja_log_trace.json + python3 $WORKSPACE/chromium/src/third_party/catapult/tracing/bin/trace2html $WORKSPACE/chromium/src/out/bromite_x64/ninja_log_trace.json + + + - name: Generate breakpad symbols arm64 + if: ${{ (github.event.inputs.target_os == 'android' || github.event.inputs.target_os == 'all') && (github.event.inputs.build == 'arm64' || github.event.inputs.build == 'all') }} + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + + echo "::group::-------- generating breakpad symbols" + autoninja -C out/bromite minidump_stackwalk dump_syms + components/crash/content/tools/generate_breakpad_symbols.py --build-dir=out/bromite \ + --symbols-dir=out/bromite/symbols/ --binary=out/bromite/lib.unstripped/libchrome.so \ + --platform=android --clear --verbose + cp out/bromite/lib.unstripped/libchrome.so out/bromite/symbols/libchrome.lib.so + cp out/bromite/minidump_stackwalk out/bromite/symbols + cp out/bromite/dump_syms out/bromite/symbols + echo "::endgroup::" + + - name: Generate Supersize data + if: ${{ github.event.inputs.debug == 'false' && (github.event.inputs.target_os == 'android' || github.event.inputs.target_os == 'all') && (github.event.inputs.build == 'arm64' || github.event.inputs.build == 'all') }} + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + + tools/binary_size/supersize archive out/bromite/chrome.size --apk-file out/bromite/apks/ChromePublic.apk -v + + - name: Generate clangd index + if: ${{ github.event.inputs.clangd == 'true' }} + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + + test -f out/bromite/bromite.idx || \ + cp -r out/bromite out/clangd && \ + gn gen --args="import(\"/home/lg/working_dir/bromite/build/bromite.gn_args\") $(cat ../../build_args.gni) skip_clangd_unsupported_options = true" out/clangd && \ + $WORKSPACE/ninja/ninja -C $WORKSPACE/chromium/src/out/clangd -a chrome_public_apk \ + -t compdb cc cxx objc objcxx >$WORKSPACE/chromium/src/out/clangd/compile_commands.json && \ + /home/lg/working_dir/clangd_snapshot_20211205/bin/clangd-indexer --executor=all-TUs out/clangd/compile_commands.json >out/bromite/bromite.idx && \ + rm -rf out/clangd diff --git a/.github/workflows/check-git-apply.yaml b/.github/workflows/check-git-apply.yaml new file mode 100644 index 00000000..1dcbb64a --- /dev/null +++ b/.github/workflows/check-git-apply.yaml @@ -0,0 +1,139 @@ +name: Check git apply +#permissions: +# actions: none +# checks: none +# contents: none +# deployments: none +# issues: none +# packages: none +# pull-requests: write +# repository-projects: none +# security-events: none +# statuses: none + +on: + push: + branches-ignore: + - 'ci' + + workflow_dispatch: + inputs: + rtag: + description: 'uazo/cromite TAG or COMMIT' + required: true + default: '' + +env: + RTAG: ${{ github.event.inputs.rtag }} + +jobs: + build: + runs-on: ubuntu-latest + steps: + + - name: "Get value from dispatch" + run: | + if [ -z "${RTAG}" ]; then + echo "RTAG=$GITHUB_SHA" >> $GITHUB_ENV + echo "RTAG=$RTAG" + fi + + - name: Prepare container + run: | + sudo apt update + sudo apt install -y wget unzip tar sed dos2unix patchutils wiggle curl + + wget https://github.com/uazo/superpatch/releases/latest/download/SuperPatchUtils.tar.gz + tar xfz SuperPatchUtils.tar.gz + rm SuperPatchUtils.tar.gz + + wget https://github.com/ericchiang/pup/releases/download/v0.4.0/pup_v0.4.0_linux_amd64.zip + unzip pup_v0.4.0_linux_amd64.zip + rm pup_v0.4.0_linux_amd64.zip + + BRANCH=$(curl https://github.com/uazo/cromite/branch_commits/$RTAG | ./pup -p li.branch:last-child a text{}) + if [ -z "${BRANCH}" ]; then + BRANCH=$(curl https://github.com/uazo/cromite/branch_commits/$RTAG a | ./pup -p li.branch:first-child a text{}) + fi + if [ -z "${BRANCH}" ]; then + echo "Can not recover the base commit" + exit 1 + fi + echo "BRANCH=$BRANCH" >> $GITHUB_ENV + + - name: Checkout 'uazo/cromite' ${{ env.BRANCH }} + uses: actions/checkout@v2 + with: + repository: 'uazo/cromite' + ref: ${{ github.event.inputs.rtag }} + path: 'cromite' + fetch-depth: 1 + + - name: Check chromium version + run: | + VERSION=$(cat cromite/build/RELEASE) + echo "VERSION=$VERSION" >> $GITHUB_ENV + + - name: Download chromium ${{ env.VERSION }} sources + run: | + ./bin/SuperPatchUtils bromite $RTAG chromium/src + cd chromium/src + git init + git config user.email "you@example.com" + git config user.name "Your Name" + git add . + git commit -m $VERSION + git tag -a $VERSION -m $VERSION + cd ../.. + + - name: Apply patches ${{ github.event.inputs.rtag }} + run: | + export HOME=$(pwd) + cd ~/chromium/src + + export SILENT=true + export SKIPAUTOGENERATED=true + bash ~/cromite/tools/apply-all-patch.sh || exit 1 + + rm -rf ~/cromite/build/patches-new/ + rm ~/cromite/build/bromite_patches_list_new.txt + + - name: Export patches + run: | + export HOME=$(pwd) + + cd ~/chromium/src + bash ~/cromite/tools/export-all-patch.sh + + cd ~/cromite + rm -rf build/patches/* + mv build/patches-new/* build/patches + rm -rf build/patches-new/ + + - name: Check differences CHANGES=${{ env.CHANGES }} + run: | + cd cromite + CHANGES=0 && git diff --quiet || CHANGES=1 + echo "CHANGES=$CHANGES" >> $GITHUB_ENV + + if [[ CHANGES -eq 1 ]]; then + git add build/patches/*.patch + git diff --name-only --staged + # MESSAGE=$(git diff --name-only --staged) + # echo "MESSAGE='$MESSAGE'" >> $GITHUB_ENV + fi + + - name: Create Pull Request + uses: peter-evans/create-pull-request@dcd5fd746d53dd8de555c0f10bca6c35628be47a #v3.12.0 + if: env.CHANGES == '1' + with: + path: cromite + base: ${{ env.BRANCH }} + #push-to-fork: uazo/cromite + add-paths: | + build/patches/*.patch + commit-message: 'AUTOMATED - git apply results' + title: Git apply result for ${{ env.BRANCH }} branch + body: ${{ env.MESSAGE }} + delete-branch: true + branch-suffix: short-commit-hash diff --git a/.github/workflows/execute-tests.yaml b/.github/workflows/execute-tests.yaml new file mode 100644 index 00000000..04efa0f8 --- /dev/null +++ b/.github/workflows/execute-tests.yaml @@ -0,0 +1,130 @@ +name: Execute Tests +permissions: + actions: none + checks: none + contents: none + deployments: none + issues: none + packages: none + pull-requests: none + repository-projects: none + security-events: none + statuses: none + +on: + workflow_dispatch: + inputs: + sha: + description: 'uazo/bromite SHA' + required: true + default: 'ce2f07034a3d3b257e9bada65d8293a6786bb95f' + +env: + BROMITE_SHA: ${{ github.event.inputs.sha }} + REMOVEDOCKERSUPPORT: true + USELOCALIMAGE: true + +jobs: + get_image: + runs-on: self-hosted + steps: + - name: Mark image to test + shell: bash + run: | + docker tag uazo/bromite-build:$BROMITE_SHA uazo/bromite-build:test + + exec-test: + runs-on: self-hosted + needs: get_image + if: success() + + container: + image: uazo/bromite-build:test + env: + REMOVEDOCKERSUPPORT: true # CUSTOM RUNNER: remove sharing of docker socket + USELOCALIMAGE: true # CUSTOM RUNNER: permit use of local images + USEINTERNALNETWORK: true # CUSTOM RUNNER: create the docker network as internal + WORKSPACE: /home/lg/working_dir + ARTIFACS_DIR: /home/lg/working_dir/artifacs + volumes: + - /storage/images/${{ github.event.inputs.sha }}/out:/home/lg/working_dir/chromium/src/out/bromite:r + - /storage/images/${{ github.event.inputs.sha }}/out/tests:/home/lg/working_dir/chromium/src/out/tests + options: --device=/dev/kvm + + steps: + - name: Prepare Test Container + shell: bash + run: | + # set workspace paths + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE + + # copy artifacts + #cd chromium/src + #mkdir -p out/bromite + #cp -arp $ARTIFACS_DIR/out/* out/bromite + + rm -rf out/tests/* + #mkdir -p out/tests + + # reset proxy env + #HTTP_PROXY= + #HTTPS_PROXY= + #http_proxy= + #https_proxy= + +# - name: Wait forever +# shell: bash +# run: | +# sleep infinity + + - name: Execute chrome junit test + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + out/bromite/bin/run_chrome_junit_tests --gtest_filter=*Bromite* || KO=1 + + - name: Start Android Emulator + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + + sudo mkdir -p /github/home/.android + sudo chmod 666 /github/home/.android/ + tools/android/avd/avd.py install --avd-config ../../generic_android31.textpb -v + tools/android/avd/avd.py start --avd-config ../../generic_android31.textpb -v + + - name: Execute unit tests + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + out/bromite/unit_tests -v --fast-local-dev --gtest_filter=*Bromite* || KO=1 + + - name: Restart Android Emulator + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + pkill -f qemu-system-x86 + tools/android/avd/avd.py start --avd-config ../../generic_android31.textpb -v + + - name: Execute instrumentation tests + shell: bash + run: | + PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + cd $WORKSPACE/chromium/src + out/bromite/bin/run_chrome_public_test_apk -v --num-retries=1 --gtest_filter=*Bromite* || KO=1 + + #- name: Wait forever + # shell: bash + # run: | + # sleep infinity + + - name: Copy results + shell: bash + run: | + sudo cp -r $WORKSPACE/chromium/src/out/bromite/TEST_RESULTS* $WORKSPACE/chromium/src/out/tests/ + diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000..e7bb963b --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,135 @@ +name: Release Bromite CI +permissions: + contents: write + +on: + workflow_dispatch: + inputs: + sha: + description: 'uazo/bromite SHA' + required: true + default: '0a8ff322d6e9d738f7b2fa52618b178081bd100d' + type: + description: 'runner? [dev/ci]' + required: true + default: 'ci' + +env: + BROMITE_SHA: ${{ github.event.inputs.sha }} + REMOVEDOCKERSUPPORT: true + USELOCALIMAGE: true + +jobs: + release: + runs-on: ${{ github.event.inputs.type }} + env: + OUTPUTFILE_ARM64: /storage/images/android/${{ github.event.inputs.sha }}/false/arm64/ + OUTPUTFILE_X64: /storage/images/android/${{ github.event.inputs.sha }}/false/x64/ + + steps: + - name: Prepare container + run: | + wget https://github.com/ericchiang/pup/releases/download/v0.4.0/pup_v0.4.0_linux_amd64.zip + unzip pup_v0.4.0_linux_amd64.zip && rm pup_v0.4.0_linux_amd64.zip + BRANCH=$(curl https://github.com/uazo/bromite/branch_commits/$BROMITE_SHA | ./pup -p li.branch:last-child a text{} | xargs) + echo "BRANCH=$BRANCH" >> $GITHUB_ENV + + wget https://github.com/cli/cli/releases/download/v2.18.1/gh_2.18.1_linux_amd64.tar.gz + tar xfz gh_2.18.1_linux_amd64.tar.gz + + - name: Checkout 'uazo/buildtools' + uses: actions/checkout@v2 + with: + repository: 'uazo/bromite-buildtools' + path: 'bromite' + fetch-depth: 1 + + - name: Create release + shell: bash + run: | + GH=gh_2.18.1_linux_amd64/bin/gh + WINOUT=/storage/images/win/x64/$BROMITE_SHA/ + VERSION=v$(cat ${{ env.OUTPUTFILE_ARM64 }}/RELEASE) + + cd bromite + + GH=gh_2.21.1_linux_amd64/bin/gh + wget https://github.com/cli/cli/releases/download/v2.21.1/gh_2.21.1_linux_amd64.tar.gz + tar xfz gh_2.21.1_linux_amd64.tar.gz + + echo ${{ secrets.GITHUB_TOKEN }} | $GH auth login --with-token + + echo "\`\`\`" >note + echo "this is not the official release of bromite but a test version." >>note + echo "you can try it at your own risk." >>note + echo "\`\`\`" >>note + + $GH release create $VERSION-$BROMITE_SHA --notes-file note -d + + # android arm64 + sudo cp ${{ env.OUTPUTFILE_ARM64 }}/apks/ChromePublic.apk arm64_ChromePublic.apk + sudo chown runner arm64_ChromePublic.apk + echo Uploading arm64_ChromePublic + $GH release upload $VERSION-$BROMITE_SHA arm64_ChromePublic.apk + + # android x64 + sudo cp ${{ env.OUTPUTFILE_X64 }}/apks/ChromePublic.apk x64_ChromePublic.apk + sudo chown runner x64_ChromePublic.apk + echo Uploading x64_ChromePublic + $GH release upload $VERSION-$BROMITE_SHA x64_ChromePublic.apk + + echo Uploading chrome.size + test $VERSION-$BROMITE_SHA $OUTPUTFILE_ARM64/chrome.size && \ + $GH release upload $VERSION-$BROMITE_SHA $OUTPUTFILE_ARM64/chrome.size + + echo Uploading arm64_ninja_log_trace.html + test $OUTPUTFILE_ARM64/ninja_log_trace.html && \ + cp $OUTPUTFILE_ARM64/ninja_log_trace.html arm64_ninja_log_trace.html && \ + $GH release upload $VERSION-$BROMITE_SHA arm64_ninja_log_trace.html + + # windows + mkdir chrome-win/ + cp $WINOUT/*.manifest chrome-win/ + cp $WINOUT/chrome.dll chrome-win/ + cp $WINOUT/chrome.exe chrome-win/ + cp $WINOUT/chrome_100_percent.pak chrome-win/ + cp $WINOUT/chrome_200_percent.pak chrome-win/ + cp $WINOUT/chrome_elf.dll chrome-win/ + cp $WINOUT/chrome_proxy.exe chrome-win/ + cp $WINOUT/chrome_pwa_launcher.exe chrome-win/ + cp $WINOUT/chrome_wer.dll chrome-win/ + cp "/win_sdk/10.0.22621.0/Windows Kits/10/Redist/D3D/x64/d3dcompiler_47.dll" chrome-win/ + cp $WINOUT/elevation_service.exe chrome-win/ + cp "$WINOUT/First Run" chrome-win/ + cp $WINOUT/headless_lib_data.pak chrome-win/ + cp $WINOUT/icudtl.dat chrome-win/ + cp $WINOUT/libEGL.dll chrome-win/ + cp $WINOUT/libGLESv2.dll chrome-win/ + cp $WINOUT/Logo.png chrome-win/ + cp $WINOUT/mojo_core.dll chrome-win/ + cp $WINOUT/notification_helper.exe chrome-win/ + cp $WINOUT/resources.pak chrome-win/ + cp $WINOUT/SmallLogo.png chrome-win/ + cp $WINOUT/snapshot_blob.bin chrome-win/ + cp $WINOUT/VkICD_mock_icd.dll chrome-win/ + cp $WINOUT/VkLayer_khronos_validation.dll chrome-win/ + cp $WINOUT/vk_swiftshader.dll chrome-win/ + cp $WINOUT/vulkan-1.dll chrome-win/ + cp -r $WINOUT/locales chrome-win/locales + + #test chrome-win.zip && rm chrome-win.zip + zip -r chrome-win.zip chrome-win/ + + echo Uploading chrome-win.zip + $GH release upload $VERSION-$BROMITE_SHA chrome-win.zip + + # workaround for https://github.com/cli/cli/issues/6599 + sleep 30s + + TIMESTAMP=$(date +%s -r chrome-win.zip) + echo >updateurl.txt "browser=chromium;os=windows;architecture=64-bit;timestamp=$TIMESTAMP;editor=uazo;channel=stable;repository=https://github.com/uazo/bromite-buildtools/releases;download=https://github.com/uazo/bromite-buildtools/releases/latest/download/chrome-win.zip;version=$(cat $WINOUT/RELEASE);revision=1;commit=$BROMITE_SHA" + $GH release upload $VERSION-$BROMITE_SHA updateurl.txt + + $GH release edit $VERSION-$BROMITE_SHA -t $VERSION-$BROMITE_SHA + $GH release edit $VERSION-$BROMITE_SHA --draft=false + diff --git a/.gitignore b/.gitignore index 18b4cac4..0e285062 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,5 @@ # This .gitignore file was automatically created by Microsoft(R) Visual Studio. ################################################################################ -/.vs/bromite-uazo/config -/.vs/bromite-uazo/FileContentIndex /.vs +/temp diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 00000000..251ae84d --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1,15 @@ +################################################################################ +# This .gitignore file was automatically created by Microsoft(R) Visual Studio. +################################################################################ + +/.vs +images/github-runner/.env +images/privoxy/privoxy.conf.save +images/remote-index/bromite.idx +/images/terraform-buildgrid/.terraform +/images/terraform-buildgrid/.vs +/images/terraform-buildgrid/.terraform.lock.hcl +/images/terraform-buildgrid/terraform.auto.tfvars +/images/terraform-buildgrid/ssh-key-2021-06-01.key.pub +/images/terraform-buildgrid/ssh-key-2021-06-01.key +/images/terraform-buildgrid/private_key.pem diff --git a/tools/LICENSE b/tools/LICENSE new file mode 100644 index 00000000..f288702d --- /dev/null +++ b/tools/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000..16519fbf --- /dev/null +++ b/tools/README.md @@ -0,0 +1,93 @@ +# bromite-buildtools + +this repo contains my build machine and some scripts I use for Bromite development. the ci uses a modified version of the gihub runner (avaiable [here](https://github.com/uazo/runner)) and use [sysbox](https://github.com/nestybox/sysbox) to improve security. it also contains everything you need to use a self-hosted modified version of [goma](https://github.com/uazo/goma-server) for a multi-machine build + +### Setting-up + +1. Prepare folders + +``` +cd ~ && mkdir gh-runner +cd gh-runner && mkdir docker-inner +SYSBOX_UID=$(cat /etc/subuid | grep sysbox | cut -d : -f 2) +sudo chown $SYSBOX_UID:$SYSBOX_UID docker-inner/ + +mkdir /storage +sudo chown $SYSBOX_UID:$SYSBOX_UID /storage +``` + +2. Clone this repo +3. Prepare `.env` + +``` +cd bromite-buildtools/images/github-runner/ +cp .env.example .env +``` + +4. Edit `.env` file + +``` +RUNNER_NAME=pd-gh-runner +GITHUB_PERSONAL_TOKEN= +GITHUB_OWNER=uazo +GITHUB_REPOSITORY=bromite-buildtools +RUNNER_LABELS=dev +ALLOWEDAUTHORSLIST=uazo +``` + +5. Prepare for windows cross build +Follow the [instructions](https://chromium.googlesource.com/chromium/src.git/+/HEAD/docs/win_cross.md#if-you_re-not-at-google) to create the zip with the toolchain + +example: +``` +cd path\to\depot_tools\win_toolchain +D:\Downloads\depot_tools\win_toolchain> package_from_installed.py --allow_multiple_vs_installs -w 10.0.20348.0 2019 +``` + +create the `/casefold` in the unix host with [casefold attribute](https://unix.stackexchange.com/questions/558977/how-to-enable-new-in-kernel-5-2-case-insensitivity-for-ext4-on-a-given-directory) and unzip the contents into. +``` +~$ ls /casefold/10.0.20348.0/ -la +total 36 +drwxr-xr-x 8 root root 4096 Oct 5 13:20 . +drwxr-xr-x 5 root root 4096 Oct 5 13:17 .. +drwxr-xr-x 6 root root 4096 Oct 5 13:19 'DIA SDK' +drwxr-xr-x 2 root root 4096 Oct 5 13:20 sys32 +drwxr-xr-x 2 root root 4096 Oct 5 13:20 sys64 +drwxr-xr-x 2 root root 4096 Oct 5 13:20 sysarm64 +drwxr-xr-x 5 root root 4096 Oct 5 13:20 VC +-rw-rw-rw- 1 root root 5 Sep 26 17:05 VS_VERSION +drwxr-xr-x 3 root root 4096 Oct 5 13:20 'Windows Kits' +``` + +6. Start the runner +``` +cd bromite-buildtools/images/github-runner/ +./start-runner.sh +``` + +### Test Android Version + +Simply download latest build on https://github.com/uazo/bromite-buildtools/releases/latest + +### Test Windows Version + +1. Download https://github.com/henrypp/chrlauncher/releases +2. Create a `chrlauncher.ini` + +``` +[chrlauncher] + +# Custom Chromium update URL (string): +ChromiumUpdateUrl=https://github.com/uazo/bromite-buildtools/releases/latest/download/updateurl.txt + +# Command line for Chromium (string): +# See here: http://peter.sh/experiments/chromium-command-line-switches/ +ChromiumCommandLine=--user-data-dir=".\User Data" --no-default-browser-check + +# Chromium executable file name (string): +ChromiumBinary=chrome.exe + +# Chromium binaries directory (string): +# Relative (to chrlauncher directory) or full path (env. variables supported). +ChromiumDirectory=.\bin +``` diff --git a/tools/apply-all-patch.sh b/tools/apply-all-patch.sh new file mode 100644 index 00000000..041c7f6a --- /dev/null +++ b/tools/apply-all-patch.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +git am --abort + +PATCH_OLD_PATH=~/cromite/build/patches +PATCH_NEW_PATH=~/cromite/build/patches-new + +DESTINATION=~/cromite/build/bromite_patches_list_ok.txt +DESTINATION_FAILED=~/cromite/build/bromite_patches_list_failed.txt + +rm $DESTINATION +rm $DESTINATION_FAILED +mkdir $PATCH_NEW_PATH + +IFS=$'\n' + +PATCH_LIST=~/cromite/build/bromite_patches_list_new.txt +if [ ! -f $PATCH_LIST ]; then + cp ~/cromite/build/bromite_patches_list.txt $PATCH_LIST +fi + +echo "Phase 1: check clean" +for current_file in $(cat $PATCH_LIST); do + + if [[ $current_file =~ ^#.* ]]; then + + echo "Executing $current_file" + eval "${current_file:1}" + echo $current_file >>$DESTINATION + + elif [[ $current_file =~ ^-.* ]]; then + + echo "Skipping $current_file" + + elif [[ $current_file =~ ^=.* ]]; then + + echo "Adding $current_file" + echo "Executing bash ~/create-from-patch.sh $PATCH_OLD_PATH/${current_file:1} $PATCH_NEW_PATH" + + bash ~/cromite/tools/create-from-patch.sh $PATCH_OLD_PATH/${current_file:1} $PATCH_NEW_PATH + + #echo "Press return" + #read -n 1 + + + elif [[ $current_file =~ ^1.* ]]; then + + echo "Using new path $current_file" + + bash ~/cromite/tools/apply-single-patch.sh $PATCH_NEW_PATH/${current_file:1} $PATCH_NEW_PATH + + echo "" + LAST_COMMIT=$(git rev-parse HEAD) + echo "Last Commit " $LAST_COMMIT + bash ~/cromite/tools/export-single-patch.sh $LAST_COMMIT + + else + + if [ -n "$SKIPAUTOGENERATED" ]; then + if [[ "$current_file" == *"Automated-domain-substitution"* ]]; then + echo "" + echo -e ${RED} " -> Excluding $current_file" ${NC} + continue + fi + fi + + bash ~/cromite/tools/apply-single-patch.sh $PATCH_OLD_PATH/$current_file $PATCH_NEW_PATH || exit 1 + + if [ -z "$SILENT" ]; then + echo $current_file >>$DESTINATION + echo $PATCH_FILE + + echo "" + LAST_COMMIT=$(git rev-parse HEAD) + echo "Last Commit " $LAST_COMMIT + bash ~/cromite/tools/export-single-patch.sh $LAST_COMMIT || exit 1 + fi + + fi + +done diff --git a/tools/apply-single-patch.sh b/tools/apply-single-patch.sh new file mode 100644 index 00000000..110d58d2 --- /dev/null +++ b/tools/apply-single-patch.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +PATCH=$1 +LOG_FILE=~/build.log + +dos2unix $PATCH + +OK=0 +DOBUILD=1 +DOEXPORT=1 + +TEST=0 +git log | grep FILE:$(basename -- $PATCH) && TEST=1 +if [[ TEST -eq 1 ]]; then + + echo "Patch $PATCH already exists. skipped." | tee -a ${LOG_FILE} + exit 0 + +fi + +echo "" | tee -a ${LOG_FILE} +echo "Applying patch $PATCH" | tee -a ${LOG_FILE} +git apply --reject --whitespace=fix $PATCH && OK=1 + +if [[ OK -eq 0 ]]; then + OK=1 + for file in $(grep '+++ b/' $PATCH | sed -e 's#+++ [ab]/##'); do + test -f $file || OK=0 + done + + for file in $(find . -name *.rej); do + echo " -> Check $file" | tee -a ${LOG_FILE}; + wiggle --no-ignore --replace ${file::-4} $file && rm $file && rm ${file::-4}.porig && echo " OK" || OK=0 + done + + for file in $(find . -name *.rej); do + echo " Found: $file" | tee -a ${LOG_FILE}; + git add ${file::-4} + OK=0 + done + + if [[ OK -eq 0 ]]; then + DOBUILD=1 + echo "Current patch $PATCH" | tee -a ${LOG_FILE} + echo " Patch not apply cleanly. Please fix..." | tee -a ${LOG_FILE} + if [ -n "$SILENT" ]; then + exit 1 + fi + echo " Press return" + read -n 1 + else + echo " Patch not apply cleanly. Wiggle done!" | tee -a ${LOG_FILE} + fi + + echo " Deleting rej" + find . -type f -name '*.rej' -delete + find . -type f -name '*.porig' -delete + +else + echo "Patch apply cleanly." | tee -a ${LOG_FILE} + DOBUILD=0 +fi + +if [[ DOBUILD -eq 1 ]]; then + OK=0 + #echo "Building ${PATCH}: chrome_public_apk" | tee -a ${LOG_FILE} + #date "+%Y%m%d %H.%M.%S" | tee -a ${LOG_FILE} + #autoninja -C out/arm64 chrome_public_apk && OK=1 + #date "+%Y%m%d %H.%M.%S" | tee -a ${LOG_FILE} + + DOEXPORT=1 + + if [[ OK -eq 1 ]]; then + if [ -z "$SILENT" ]; then + echo "Read to add $PATCH. Press return" + read -n 1 + fi + fi +fi + +if [[ DOEXPORT -eq 1 ]]; then + until false + do + bash ~/cromite/tools/create-from-patch.sh $PATCH $2 || exit 1 + + rm /tmp/1 /tmp/2 || true + lsdiff $PATCH >/tmp/1 + lsdiff ../../cromite/build/patches-new/$(basename -- $PATCH) >/tmp/2 + STATUS="$(cmp --silent /tmp/1 /tmp/2; echo $?)" + if [[ $STATUS -ne 0 ]]; then + git reset HEAD^ + diff /tmp/1 /tmp/2 + echo "Some files are missing. Check again." | tee -a ${LOG_FILE} + if [ -n "$SILENT" ]; then + exit 1 + fi + + read -r -p "y to continue [y/N] " response + if [[ "$response" =~ ^([yY][eE][sS]|[yY])$ ]]; then + bash ~/cromite/tools/create-from-patch.sh $PATCH $2 || exit 1 + break + fi + else + break + fi + done +fi diff --git a/tools/create-from-patch.sh b/tools/create-from-patch.sh new file mode 100644 index 00000000..1679cdba --- /dev/null +++ b/tools/create-from-patch.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +PATCH=$1 + +PLEASEWAIT=0 +if [[ $PATCH =~ ^+.* ]]; then + PLEASEWAIT=1 + PATCH=${PATCH:1} +fi + +if [ -z "$2" ] +then + PATCH_NEW_PATH=~/cromite/build/patches-new +else + PATCH_NEW_PATH=$2 +fi + +echo " Creating new patch" +git add . + +HEAD=$(sed -n '1,/---/ p' $PATCH | sed '/^---/d') +CONTENT=$(git -C ~/chromium/src/ diff --cached --binary) + +PATCH_FILE=$PATCH_NEW_PATH/$(basename $PATCH) +test -f $PATCH_FILE && rm $PATCH_FILE +echo "$HEAD" >$PATCH_FILE + +NEWLINE=$(tail -n 1 "$PATCH_FILE") +#echo $NEWLINE +if [[ "$NEWLINE" == Subject:* ]]; then + echo "" >>$PATCH_FILE +else + NEWLINE=$(tail -n 2 "$PATCH_FILE" | head -n 1) + if [[ "$NEWLINE" == Subject:* ]]; then + echo "" >>$PATCH_FILE + fi +fi + +echo "FILE:$(basename $PATCH)" >>$PATCH_FILE +echo "---" >>$PATCH_FILE +echo "$CONTENT" >>$PATCH_FILE + +#echo press return +#read -n 1 + +git reset --hard +git clean -f -d + +echo " Applying new patch" +OK=1 +git am $PATCH_FILE || OK=0 + +if [[ OK -eq 0 ]]; then + if [ -n "$SILENT" ]; then + exit 1 + fi + echo "---> Failed to apply. Press return" + read -n 1 +fi diff --git a/tools/depot_tools.diff b/tools/depot_tools.diff new file mode 100644 index 00000000..0422a729 --- /dev/null +++ b/tools/depot_tools.diff @@ -0,0 +1,60 @@ +diff --git a/gclient_scm.py b/gclient_scm.py +--- a/gclient_scm.py ++++ b/gclient_scm.py +@@ -1012,28 +1012,35 @@ class GitWrapper(SCMWrapper): + gclient_utils.safe_makedirs(parent_dir) + + template_dir = None ++ use_fetch = False ++ ++ tmp_dir = tempfile.mkdtemp( ++ prefix='_gclient_%s_' % os.path.basename(self.checkout_path), ++ dir=parent_dir) ++ + if hasattr(options, 'no_history') and options.no_history: + if gclient_utils.IsGitSha(revision): + # In the case of a subproject, the pinned sha is not necessarily the + # head of the remote branch (so we can't just use --depth=N). Instead, + # we tell git to fetch all the remote objects from SHA..HEAD by means of + # a template git dir which has a 'shallow' file pointing to the sha. +- template_dir = tempfile.mkdtemp( +- prefix='_gclient_gittmp_%s' % os.path.basename(self.checkout_path), +- dir=parent_dir) +- self._Run(['init', '--bare', template_dir], options, cwd=self._root_dir) +- with open(os.path.join(template_dir, 'shallow'), 'w') as template_file: +- template_file.write(revision) +- clone_cmd.append('--template=' + template_dir) ++ self._Run(['init', tmp_dir], options, cwd=self._root_dir) ++ ++ self._Run(['-C', tmp_dir, 'remote', 'add', 'origin', url], options, cwd=self._root_dir) ++ ++ clone_cmd = cfg + ['-C', tmp_dir, 'fetch', '--progress'] ++ clone_cmd.append('--depth=1') ++ clone_cmd.append(url) ++ clone_cmd.append(revision) ++ use_fetch = True + else: + # Otherwise, we're just interested in the HEAD. Just use --depth. + clone_cmd.append('--depth=1') + +- tmp_dir = tempfile.mkdtemp( +- prefix='_gclient_%s_' % os.path.basename(self.checkout_path), +- dir=parent_dir) + try: +- clone_cmd.append(tmp_dir) ++ if use_fetch == False: ++ clone_cmd.append(tmp_dir) ++ + if self.print_outbuf: + print_stdout = True + filter_fn = None +@@ -1328,6 +1335,9 @@ class GitWrapper(SCMWrapper): + if refspec: + fetch_cmd.append(refspec) + ++ if hasattr(options, 'no_history') and options.no_history: ++ fetch_cmd.append('--depth=1') ++ + if prune: + fetch_cmd.append('--prune') + if options.verbose: diff --git a/tools/export-all-patch.sh b/tools/export-all-patch.sh new file mode 100644 index 00000000..1ce1ba4b --- /dev/null +++ b/tools/export-all-patch.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +VERSION=$(cat ~/cromite/build/RELEASE) +CURRENT_RELEASE=$(git -C ~/chromium/src/ rev-parse --verify refs/tags/$VERSION) + +ALLPATCHS_E=$(git -C ~/chromium/src/ rev-list HEAD...$CURRENT_RELEASE) + +mkdir ~/cromite/build/patches-new + +NO_NAME=1 + +for patch in $ALLPATCHS_E; do + + PATCH_FILE=$(git -C ~/chromium/src/ show -s $patch | grep FILE: | sed 's/FILE://g' | sed 's/^[ \t]*//;s/[ \t]*$//') + if [[ "$PATCH_FILE" == *"Automated-domain-substitution"* ]]; then + continue + fi + + if [ -z "$PATCH_FILE" ] + then + PATCH_FILE=$(git -C ~/chromium/src/ show -s $patch | tail -n 1) + if [[ "$PATCH_FILE" != *".patch" ]]; then + PATCH_FILE=00$(git -C ~/chromium/src/ show -s $patch | head -n 5 | tail -n 1 | xargs | tr " " - | tr [:punct:] -).patch + echo New Patch: ${PATCH_FILE} + fi + fi + + bash ~/cromite/tools/export-single-patch.sh $patch $PATCH_FILE + +done + +PATCH_LIST=~/cromite/build/bromite_patches_list.txt +mkdir ~/cromite/build/patches-new/changed +mkdir ~/cromite/build/patches-new/contrib +for current_file in $(cat $PATCH_LIST); do + if [[ "$current_file" == *".patch" ]]; then + if [[ $current_file =~ ^changed/.* ]]; then + mv ~/cromite/build/patches-new/$(basename $current_file) ~/cromite/build/patches-new/changed + elif [[ $current_file =~ ^contrib/.* ]]; then + mv ~/cromite/build/patches-new/$(basename $current_file) ~/cromite/build/patches-new/contrib || true + fi + fi +done diff --git a/tools/export-patch-list.sh b/tools/export-patch-list.sh new file mode 100644 index 00000000..88a90213 --- /dev/null +++ b/tools/export-patch-list.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +VERSION=$(cat ~/cromite/build/RELEASE) +CURRENT_RELEASE=$(git -C ~/chromium/src/ rev-parse --verify refs/tags/$VERSION) + +ALLPATCHS_E=$(git -C ~/chromium/src/ rev-list HEAD...$CURRENT_RELEASE) + +mkdir ~/cromite/build/patches-new +rm ~/cromite/build/patches-new/patch-list + +NO_NAME=1 + +for patch in $ALLPATCHS_E; do + + PATCH_FILE=$(git -C ~/chromium/src/ show -s $patch | grep FILE: | sed 's/FILE://g' | sed 's/^[ \t]*//;s/[ \t]*$//') + if [[ "$PATCH_FILE" == *"Automated-domain-substitution"* ]]; then + continue + fi + PATCH_MESSAGE=$(git -C ~/chromium/src/ show -s $patch) + if [[ $PATCH_MESSAGE == *NOEXPORT:* ]] ; + then + continue + fi + if [[ -z "$PATCH_FILE" ]]; then + PATCH_FILE=00$(git -C ~/chromium/src/ show -s $patch | head -n 5 | tail -n 1 | xargs | tr " " - | tr [:punct:] -).patch + fi + + echo $PATCH_FILE >>~/cromite/build/patches-new/patch-list + +done + +tac ~/cromite/build/patches-new/patch-list >~/cromite/build/patches-new/zz-patch-list.txt +rm ~/cromite/build/patches-new/patch-list diff --git a/tools/export-single-patch.sh b/tools/export-single-patch.sh new file mode 100644 index 00000000..895012c0 --- /dev/null +++ b/tools/export-single-patch.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +patch=$1 +output=$2 + +PATCH_FILE=$(git -C ~/chromium/src/ show -s $patch | grep FILE: | sed 's/FILE://g' | sed 's/^[ \t]*//;s/[ \t]*$//') +if [ -z "$output" ] +then + PATCH_FILE=$(git -C ~/chromium/src/ show -s $patch | tail -n 1 | xargs) + echo Exporting $patch ~/cromite/build/patches-new/$PATCH_FILE +else + PATCH_FILE=$output + echo Exporting new $patch ~/cromite/build/patches-new/$PATCH_FILE +fi + +PATCH_MESSAGE=$(git -C ~/chromium/src/ show -s $patch) +if [[ $PATCH_MESSAGE == *NOEXPORT:* ]] ; +then + echo Request NO export + exit 0 +fi + +git -C ~/chromium/src/ format-patch -1 --keep-subject --stdout --full-index --zero-commit --no-signature $patch >~/cromite/build/patches-new/$PATCH_FILE +echo " exported" + +CHANGE_REF="" +while read line; do + for i in {1..5} + do + if [[ "$line" == index* ]]; then + read next_line + if [[ "$next_line" != "GIT binary patch" ]]; then + CHANGE_REF=${CHANGE_REF}"/^${line}/d;" + break + else + line=$next_line + continue + fi + else + break + fi + done +done <~/cromite/build/patches-new/$PATCH_FILE + +if [ "$CHANGE_REF" ] +then + sed -i "$CHANGE_REF" ~/cromite/build/patches-new/$PATCH_FILE +fi +sed -i '/^From 0000000000000000000000000000000000000000/d' ~/cromite/build/patches-new/$PATCH_FILE +sed -i '/^FILE:/d' ~/cromite/build/patches-new/$PATCH_FILE +sed -i '/^ mode change/d' ~/cromite/build/patches-new/$PATCH_FILE +sed -i '/^old mode /d' ~/cromite/build/patches-new/$PATCH_FILE +sed -i '/^new mode /d' ~/cromite/build/patches-new/$PATCH_FILE + +echo "--" >> ~/cromite/build/patches-new/$PATCH_FILE +echo "2.25.1" >> ~/cromite/build/patches-new/$PATCH_FILE +#echo "" >> ~/cromite/build/patches-new/$PATCH_FILE + +echo " done." +echo "" diff --git a/tools/goma_auth.py b/tools/goma_auth.py new file mode 100644 index 00000000..05f417d1 --- /dev/null +++ b/tools/goma_auth.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +def main(): + return 0 diff --git a/tools/images/bromite-build/Dockerfile b/tools/images/bromite-build/Dockerfile new file mode 100644 index 00000000..8f946105 --- /dev/null +++ b/tools/images/bromite-build/Dockerfile @@ -0,0 +1,32 @@ +ARG BROMITE_SHA +ARG VERSION + +FROM uazo/bromite:$VERSION-$BROMITE_SHA + +ARG HTTP_PROXY +ENV HTTP_PROXY=$HTTP_PROXY +ENV HTTPS_PROXY=$HTTP_PROXY +ENV https_proxy=$HTTP_PROXY + +USER lg +WORKDIR /home/lg/working_dir + +COPY pre-start.sh . +COPY start-build.sh . +COPY goma_auth.py . +COPY casupload . +COPY build_args.gni . + +COPY generic_android31.textpb . + +ENV CIPD_CACHE_DIR=/home/lg/working_dir/.cipd_cache +ENV VPYTHON_VIRTUALENV_ROOT=/home/lg/vpython_root + +RUN sudo chmod +x ./start-build.sh \ + && \ + sudo chmod +x ./pre-start.sh \ + && \ + sudo chmod 775 ./goma_auth.py \ + && \ + ./pre-start.sh + diff --git a/tools/images/bromite-build/action.yaml b/tools/images/bromite-build/action.yaml new file mode 100644 index 00000000..434da444 --- /dev/null +++ b/tools/images/bromite-build/action.yaml @@ -0,0 +1,34 @@ +name: 'Prepare Bromite Builder Image' +description: 'Check and build Bromite builder image by sha' + +inputs: + sha: + description: 'Bromite sha ref' + required: true + +runs: + using: "composite" + steps: + - name: Checking image for "${{ inputs.sha }}" + shell: bash + run: | + BROMITE_SHA="${{ inputs.sha }}" + + IS_PRESENT=$(docker inspect --type=image uazo/bromite-build:$BROMITE_SHA > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + IS_PRESENT=$(docker manifest inspect uazo/bromite-build:$BROMITE_SHA > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + echo "Image not found" + + mkdir bromite-source + pushd bromite-source/ + git clone https://github.com/uazo/bromite-buildtools + + cd bromite-buildtools/images/bromite-build/ + DOCKER_BUILDKIT=1 docker build -t uazo/bromite-build:$BROMITE_SHA --progress plain \ + --build-arg BROMITE_SHA=$BROMITE_SHA \ + . + + popd + fi + fi diff --git a/tools/images/bromite-build/build_args.gni b/tools/images/bromite-build/build_args.gni new file mode 100644 index 00000000..79014e11 --- /dev/null +++ b/tools/images/bromite-build/build_args.gni @@ -0,0 +1,50 @@ + +declare_args() { + _is_debug_build = "" +} + +not_needed(["_target_build", "_is_debug_build"]) + +if(getenv("TARGET_CPU") != "") { + target_cpu = getenv("TARGET_CPU") +} + +if (target_os == "android") { + target_os = "android" # fix traffic annotation auditor + #enable_kythe_annotations = true + chrome_public_manifest_package = "org.bromite.bromite.dev" + + _is_debug_build = getenv("TARGET_ISDEBUG") + if(_is_debug_build == "true") { + # print("Debug build on") + is_debug = true + is_official_build = false + dcheck_always_on = true + symbol_level = 1 + strip_debug_info = false + generate_linker_map = false + + # since is_cfi require use_thin_lto + # but not work in debug mode + is_cfi = false # disable it + use_cfi_cast = false # disable it + } else { + generate_linker_map = true + use_relative_vtables_abi = false + } +} + +if (target_os == "win") { + target_os = "win" # fix traffic annotation auditor + target_cpu = "x64" + symbol_level = 0 + use_large_pdbs = true + + enable_pdf = true + pdf_is_complete_lib = true + enable_plugins = true + enable_ppapi = false + + is_cfi = false # disable it + use_cfi_cast = false # disable it +} diff --git a/tools/images/bromite-build/casupload b/tools/images/bromite-build/casupload new file mode 100644 index 00000000..a181f102 Binary files /dev/null and b/tools/images/bromite-build/casupload differ diff --git a/tools/images/bromite-build/generic_android31.textpb b/tools/images/bromite-build/generic_android31.textpb new file mode 100644 index 00000000..0c365514 --- /dev/null +++ b/tools/images/bromite-build/generic_android31.textpb @@ -0,0 +1,25 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Configuration for a generic x86_64 android-12 AVD (userdebug build). + +emulator_package { + package_name: "chromium/third_party/android_sdk/public/emulator" + version: "gMHhUuoQRKfxr-MBn3fNNXZtkAVXtOwMwT7kfx8jkIgC" # 30.7.5 + dest_path: ".emulator_sdk" +} + +system_image_package { + package_name: "chromium/third_party/android_sdk/public/system-images/android-31/google_apis/x86_64" + version: "R6Jh5_P21Euu-kdb11zcNjdJKN4vV1mdQTb8t4gph4IC" # 6 + dest_path: ".emulator_sdk" +} +system_image_name: "system-images;android-31;google_apis;x86_64" + +avd_package { + package_name: "chromium/third_party/android_sdk/public/avds/android-31/google_apis/x86_64" + version: "Ur_zl6_BRKRkf_9X3SMZ3eH2auoOyJ2kLslpTZZwi3gC" # created in bb_id 8841388797862621664 + dest_path: ".android" +} +avd_name: "android_31_google_apis_x86_64" diff --git a/tools/images/bromite-build/goma_auth.py b/tools/images/bromite-build/goma_auth.py new file mode 100644 index 00000000..05f417d1 --- /dev/null +++ b/tools/images/bromite-build/goma_auth.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +def main(): + return 0 diff --git a/tools/images/bromite-build/pre-start.sh b/tools/images/bromite-build/pre-start.sh new file mode 100644 index 00000000..3f17a902 --- /dev/null +++ b/tools/images/bromite-build/pre-start.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +RED='\033[0;31m' +NC='\033[0m' # No Color + +WORKSPACE=/home/lg/working_dir +PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$PATH + +sudo apt-get install -y lsof libgoogle-glog-dev libprotobuf17 libgrpc++1 parallel golang-go + +echo -e ${RED} -------- download mtool ${NC} +git clone https://github.com/bromite/mtool +cd mtool +make +cd .. + +echo -e ${RED} -------- download ninjatracing ${NC} +git clone https://github.com/nico/ninjatracing + +mkdir $CIPD_CACHE_DIR +mkdir $VPYTHON_VIRTUALENV_ROOT + +echo -e ${RED} -------- download goma client ${NC} +cd $WORKSPACE +cipd install infra/goma/client/linux-amd64 -root $WORKSPACE/goma + +echo "nomatter" >$WORKSPACE/.debug_auth_file +sudo cp $WORKSPACE/goma_auth.py $WORKSPACE/goma/ + +echo -e ${RED} -------- prepare vpython virtual environment ${NC} +rm -rf /tmp/vpython* +cd $WORKSPACE/chromium/src +vpython -vpython-spec .vpython -vpython-root $VPYTHON_VIRTUALENV_ROOT -vpython-log-level debug -vpython-tool install +vpython3 -vpython-spec .vpython3 -vpython-root $VPYTHON_VIRTUALENV_ROOT -vpython-log-level debug -vpython-tool install + +echo -e ${RED} -------- download x86_64 android image ${NC} +#echo -e "\$ParanoidMode CheckIntegrity\n\nchromium/third_party/android_sdk/public/avds/android-31/google_apis/x86_64 Ur_zl6_BRKRkf_9X3SMZ3eH2auoOyJ2kLslpTZZwi3gC" | .cipd_client ensure -ensure-file - -root $WORKSPACE/chromium/src/.android +#echo -e "\$ParanoidMode CheckIntegrity\n\nchromium/third_party/android_sdk/public/emulator gMHhUuoQRKfxr-MBn3fNNXZtkAVXtOwMwT7kfx8jkIgC\nchromium/third_party/android_sdk/public/system-images/android-31/google_apis/x86_64 R6Jh5_P21Euu-kdb11zcNjdJKN4vV1mdQTb8t4gph4IC" | .cipd_client ensure -ensure-file - -root $WORKSPACE/chromium/src/.emulator_sdk + +echo -e ${RED} -------- download kythe resources ${NC} +wget https://chromium.googlesource.com/chromium/tools/build/+/main/recipes/recipe_modules/codesearch/resources/add_kythe_metadata.py?format=TEXT -O ~/add_kythe_metadata.py.base64 +base64 -d ~/add_kythe_metadata.py.base64 >~/add_kythe_metadata.py +echo -e "infra/tools/package_index/linux-amd64 latest" | .cipd_client ensure -ensure-file - -root ~/package_index/latest + +cd $WORKSPACE/ +wget https://github.com/kythe/kythe/releases/download/v0.0.55/kythe-v0.0.55.tar.gz +tar xfz kythe-v0.0.55.tar.gz + +# removed since fail download with +# https://commondatastorage.9oo91eapis.qjz9zk/chromium-browser-clang/Linux_x64/translation_unit-llvmorg-14-init-5759-g02895eed-1.tgz +python tools/clang/scripts/update.py --package=translation_unit + +echo -e ${RED} -------- download bromite-buildtools ${NC} +cd $WORKSPACE/ +git clone https://github.com/uazo/bromite-buildtools + +echo -e ${RED} -------- compile modified ninja ${NC} +cd $WORKSPACE/ +git clone https://github.com/ninja-build/ninja.git -b v1.8.2 +cd ninja +git apply $WORKSPACE/bromite-buildtools/ninja-one-target-for-compdb.patch +CXX=clang++ ./configure.py --bootstrap + +echo -e ${RED} -------- download clang indexer ${NC} +cd $WORKSPACE/ +wget https://github.com/clangd/clangd/releases/download/snapshot_20211205/clangd_indexing_tools-linux-snapshot_20211205.zip +unzip clangd_indexing_tools-linux-snapshot_20211205.zip +rm clangd_indexing_tools-linux-snapshot_20211205.zip diff --git a/tools/images/bromite-build/start-build.sh b/tools/images/bromite-build/start-build.sh new file mode 100644 index 00000000..92b48a2e --- /dev/null +++ b/tools/images/bromite-build/start-build.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +RED='\033[0;31m' +NC='\033[0m' # No Color + +WORKSPACE=/home/lg/working_dir + +PATH=$WORKSPACE/chromium/src/third_party/llvm-build/Release+Asserts/bin:$WORKSPACE/depot_tools/:/usr/local/go/bin:$WORKSPACE/mtool/bin:$PATH + +export GOMA_SERVER_HOST=$SERVER_HOST_GOMA +export GOMA_SERVER_PORT=5050 +export GOMA_USE_SSL=false +export GOMA_HTTP_AUTHORIZATION_FILE=$WORKSPACE/.debug_auth_file +export GOMA_HERMETIC=error +export GOMA_USE_LOCAL=false +export GOMA_FALLBACK=true +export GOMA_ARBITRARY_TOOLCHAIN_SUPPORT=true + +$WORKSPACE/goma/goma_ctl.py ensure_stop +$WORKSPACE/goma/goma_ctl.py ensure_start + +cd chromium/src + +OUT_PRESENT=0 +test -d out/bromite && OUT_PRESENT=1 +if [[ OUT_PRESENT -eq 0 ]]; then + + echo -e ${RED} -------- sync out folder ${NC} + test -d ../../artifacs/out/bromite && \ + mkdir -p out/bromite && \ + cp -arp ../../artifacs/out/bromite/* out/bromite/ + + echo -e ${RED} -------- gn gen ${NC} + gn gen --args="import(\"/home/lg/working_dir/bromite/build/GN_ARGS\") use_goma=true goma_dir=\"$WORKSPACE/goma\" $(cat ../../build_args.gni) " out/bromite + + echo -e ${RED} -------- gn args ${NC} + gn args out/bromite/ --list --short + gn args out/bromite/ --list >$WORKSPACE/artifacs/gn_list + + echo -e ${RED} -------- apply .mtool ${NC} + test -f out/bromite/.mtool && \ + cp out/bromite/.mtool .mtool && \ + $WORKSPACE/mtool/chromium/mtime.sh --restore + +fi + +if [[ -z "${GOMAJOBS}" ]]; then + GOMAJOBS=40 +fi + +echo -e ${RED} -------- pre-cache toolchain ${NC} +sudo ../../casupload --cas-server=unix:/tmp/proxy/bots.sock --instance=default_instance \ + third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include \ + third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/include \ + third_party/llvm-build/Release+Asserts/lib \ + third_party/llvm-build/Release+Asserts/bin \ + buildtools/third_party/libc++ \ + chrome/android/profiles/afdo.prof + + +echo -e ${RED} -------- start build ${NC} +autoninja -j $GOMAJOBS -C out/bromite chrome_public_apk +echo -e ${RED} -------- end build ${NC} + +wget http://127.0.0.1:8088/logz?INFO -O ../../artifacs/goma-client.log +cp out/bromite/apks/* $WORKSPACE/artifacs/ + +echo -e ${RED} -------- generating breakpad symbols ${NC} +autoninja -j $GOMAJOBS -C out/bromite minidump_stackwalk dump_syms +components/crash/content/tools/generate_breakpad_symbols.py --build-dir=out/bromite \ + --symbols-dir=$WORKSPACE/artifacs/symbols/ --binary=out/bromite/lib.unstripped/libchrome.so --clear --verbose +cp out/bromite/lib.unstripped/libchrome.so $WORKSPACE/artifacs/symbols/libchrome.lib.so +cp out/bromite/minidump_stackwalk $WORKSPACE/artifacs/symbols +cp out/bromite/dump_syms $WORKSPACE/artifacs/symbols + +echo -e ${RED} -------- sync out folder ${NC} +$WORKSPACE/mtool/chromium/mtime.sh --backup +mv .mtool out/bromite/ +cp -arp out/bromite $WORKSPACE/artifacs/out + +echo -e ${RED} -------- stop goma ${NC} +$WORKSPACE/goma/goma_ctl.py ensure_stop diff --git a/tools/images/bromite-source/Dockerfile b/tools/images/bromite-source/Dockerfile new file mode 100644 index 00000000..42609d47 --- /dev/null +++ b/tools/images/bromite-source/Dockerfile @@ -0,0 +1,32 @@ +ARG VERSION + +FROM uazo/chromium:$VERSION + +ARG BROMITE_SHA +ENV BROMITE_SHA=$BROMITE_SHA + +ARG HTTP_PROXY +ENV HTTP_PROXY=$HTTP_PROXY +ENV HTTPS_PROXY=$HTTP_PROXY +ENV https_proxy=$HTTP_PROXY + +#USER lg +WORKDIR /home/lg/working_dir + +RUN sudo chown lg /etc/apt/apt.conf.d/proxy.conf +RUN sudo echo "Acquire::http::Proxy \"$HTTP_PROXY\";" >/etc/apt/apt.conf.d/proxy.conf +RUN sudo apt-get update + +COPY apply-bromite-patches.sh . + +RUN sudo chmod +x ./apply-bromite-patches.sh && \ + mkdir bromite && \ + cd bromite && \ + git init && \ + git remote add origin https://github.com/uazo/bromite && \ + git fetch origin $BROMITE_SHA && \ + git reset --hard FETCH_HEAD && \ + cd .. + +RUN ./apply-bromite-patches.sh + diff --git a/tools/images/bromite-source/action.yaml b/tools/images/bromite-source/action.yaml new file mode 100644 index 00000000..f0211365 --- /dev/null +++ b/tools/images/bromite-source/action.yaml @@ -0,0 +1,41 @@ +name: 'Prepare Bromite Source Image' +description: 'Check and build Bromite sources image by version' + +inputs: + version: + description: 'Chromium Version (example 91.0.4472.146)' + required: true + + sha: + description: 'Bromite sha ref' + required: true + +runs: + using: "composite" + steps: + - name: Checking image for "${{ inputs.sha }}" + shell: bash + run: | + BROMITE_SHA="${{ inputs.sha }}" + VERSION="${{ inputs.version }}" + + IS_PRESENT=$(docker inspect --type=image uazo/bromite:$BROMITE_SHA > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + IS_PRESENT=$(docker manifest inspect uazo/bromite:$BROMITE_SHA > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + echo "Image not found" + + mkdir bromite-source + pushd bromite-source/ + + git clone https://github.com/uazo/bromite-buildtools + + cd bromite-buildtools/images/bromite-source/ + DOCKER_BUILDKIT=1 docker build -t uazo/bromite:$BROMITE_SHA --progress plain \ + --build-arg BROMITE_SHA=$BROMITE_SHA \ + --build-arg VERSION=$VERSION \ + . + + popd + fi + fi diff --git a/tools/images/bromite-source/apply-bromite-patches.sh b/tools/images/bromite-source/apply-bromite-patches.sh new file mode 100644 index 00000000..3a2096ee --- /dev/null +++ b/tools/images/bromite-source/apply-bromite-patches.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +RED='\033[0;31m' +NC='\033[0m' # No Color + +sudo apt install sed + +cd chromium/src + +echo -e ${RED} ------- apply patchs ${NC} +for file in $(cat ../../bromite/build/bromite_patches_list.txt) ; do + + if [[ "$file" == *".patch" ]]; then + #if [[ "$file" == *"Automated-domain-substitution"* ]]; then + # echo -e ${RED} " -> Excluding $file" ${NC} + # continue + #fi + + echo -e ${RED} " -> Apply $file" ${NC} + + REPL="0,/^---/s//FILE:"$(basename $file)"\n---/" + cat ../../bromite/build/patches/$file | sed $REPL | git am + + if [ $? -ne 0 ] + then + echo -e "Error on ../../bromite/build/patches/${file}" + exit 1 + fi + + echo " " + fi + +done diff --git a/tools/images/build-deps/Dockerfile b/tools/images/build-deps/Dockerfile new file mode 100644 index 00000000..69430647 --- /dev/null +++ b/tools/images/build-deps/Dockerfile @@ -0,0 +1,44 @@ +FROM ubuntu:20.04 +ARG VERSION + +ARG HTTP_PROXY + +ENV HTTP_PROXY=$HTTP_PROXY +ENV http_proxy=$HTTP_PROXY + +ENV HTTPS_PROXY=$HTTP_PROXY +ENV https_proxy=$HTTP_PROXY + +RUN touch /etc/apt/apt.conf.d/proxy.conf && \ + echo "Acquire::http::Proxy \"$HTTP_PROXY\";" >/etc/apt/apt.conf.d/proxy.conf + +RUN dpkg --add-architecture i386 + +RUN apt-get update &&\ + DEBIAN_FRONTEND=noninteractive \ + apt-get -f -y install sudo lsb-release cl-base64 bash wget apt-utils python sed tzdata build-essential lib32gcc-9-dev g++-multilib dos2unix wiggle + +ENV user lg + +RUN useradd -m -d /home/${user} ${user} && \ + chown -R ${user} /home/${user} && \ + adduser ${user} sudo && \ + chmod 4755 /usr/bin/sudo && \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +USER ${user} + +#RUN mkdir -p /home/${user} && \ +# echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +WORKDIR /home/${user} + +RUN wget https://raw.githubusercontent.com/chromium/chromium/$VERSION/build/install-build-deps.sh && \ + wget https://raw.githubusercontent.com/chromium/chromium/$VERSION/build/install-build-deps.py && \ + sed -i 's/snapcraft/wget/' install-build-deps.sh && \ + chmod +x ./install-build-deps.sh && \ + chmod +x ./install-build-deps.py && \ + sudo ./install-build-deps.sh --no-prompt --lib32 --no-chromeos-fonts && \ + sudo ./install-build-deps.sh --android --no-prompt --no-chromeos-fonts && \ + sudo mkdir -p /github/home/.vscode-server && \ + sudo chown lg /github/home/.vscode-server + diff --git a/tools/images/build-deps/action.yaml b/tools/images/build-deps/action.yaml new file mode 100644 index 00000000..7702ae81 --- /dev/null +++ b/tools/images/build-deps/action.yaml @@ -0,0 +1,32 @@ +name: 'Prepare Build Deps Image' +description: 'Check and build build deps image by version' + +inputs: + version: + description: 'Chromium Version (example 91.0.4472.146)' + required: true + +runs: + using: "composite" + steps: + - name: Checking image for "${{ inputs.version }}" + shell: bash + run: | + VERSION="${{ inputs.version }}" + IS_PRESENT=$(docker inspect --type=image uazo/build-deps:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + IS_PRESENT=$(docker manifest inspect uazo/build-deps:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + echo "Image not found" + + mkdir build-repo + pushd build-repo/ + + git clone https://github.com/uazo/bromite-buildtools + cd bromite-buildtools/images/build-deps/ + DOCKER_BUILDKIT=1 docker build -t uazo/build-deps:$VERSION --progress plain . + + popd + rm -rf build-repo + fi + fi diff --git a/tools/images/buildstack.env.example b/tools/images/buildstack.env.example new file mode 100644 index 00000000..7b6ed7dc --- /dev/null +++ b/tools/images/buildstack.env.example @@ -0,0 +1 @@ +REMOTEEXEC_ADDR=:50051 diff --git a/tools/images/buildstack.yml b/tools/images/buildstack.yml new file mode 100644 index 00000000..87729add --- /dev/null +++ b/tools/images/buildstack.yml @@ -0,0 +1,117 @@ +version: "3.9" + +services: + buildboxcasd: + image: uazo/buildboxcasd + networks: + - bridge-ext + env_file: + - buildstack.env + volumes: + - /cache:/wrk-cache:rw + deploy: + mode: global + placement: + max_replicas_per_node: 1 + + buildboxrunner-32: + image: uazo/buildboxrunner + networks: + - bridge-int + depends_on: + - buildboxcasd + volumes: + - /cache:/wrk-cache:rw + deploy: + replicas: 40 + placement: + max_replicas_per_node: 30 + constraints: [node.labels.gomarunners == 32] + + buildboxrunner-12: + image: uazo/buildboxrunner + networks: + - bridge-int + depends_on: + - buildboxcasd + volumes: + - /cache:/wrk-cache:rw + deploy: + replicas: 60 + placement: + max_replicas_per_node: 12 + constraints: [node.labels.gomarunners == 12] + + buildboxrunner-8: + image: uazo/buildboxrunner + networks: + - bridge-int + depends_on: + - buildboxcasd + volumes: + - /cache:/wrk-cache:rw + deploy: + replicas: 60 + placement: + max_replicas_per_node: 8 + constraints: [node.labels.gomarunners == 8] + + buildboxrunner-6: + image: uazo/buildboxrunner + networks: + - bridge-int + depends_on: + - buildboxcasd + volumes: + - /cache:/wrk-cache:rw + deploy: + replicas: 60 + placement: + max_replicas_per_node: 6 + constraints: [node.labels.gomarunners == 6] + + buildboxrunner-4: + image: uazo/buildboxrunner + networks: + - bridge-int + depends_on: + - buildboxcasd + volumes: + - /cache:/wrk-cache:rw + deploy: + replicas: 60 + placement: + max_replicas_per_node: 4 + constraints: [node.labels.gomarunners == 4] + + buildboxrunner-2: + image: uazo/buildboxrunner + networks: + - bridge-int + depends_on: + - buildboxcasd + volumes: + - /cache:/wrk-cache:rw + deploy: + replicas: 60 + placement: + max_replicas_per_node: 2 + constraints: [node.labels.gomarunners == 2] + +networks: + bridge-int: + driver: overlay + driver_opts: + com.docker.network.bridge.name: build-bridge-int + com.docker.network.bridge.enable_icc: "false" + com.docker.network.bridge.enable_ip_masquerade: "false" + com.docker.network.bridge.default_bridge: "false" + name: build-bridge-int + internal: true + + bridge-ext: + driver: overlay + driver_opts: + com.docker.network.bridge.name: build-bridge-ext + com.docker.network.bridge.enable_icc: "false" + name: build-bridge-ext diff --git a/tools/images/chr-source/Dockerfile b/tools/images/chr-source/Dockerfile new file mode 100644 index 00000000..702c2691 --- /dev/null +++ b/tools/images/chr-source/Dockerfile @@ -0,0 +1,17 @@ +ARG VERSION + +FROM uazo/build-deps:$VERSION +ARG VERSION +ENV VERSION=$VERSION + +#USER lg +WORKDIR /home/lg/working_dir + +COPY prepare-build.sh . +COPY depot_tools.diff . +COPY remove_ninja_uploader.diff . + +RUN sudo chmod +x ./prepare-build.sh \ + && \ + ./prepare-build.sh + diff --git a/tools/images/chr-source/action.yaml b/tools/images/chr-source/action.yaml new file mode 100644 index 00000000..8e7230d0 --- /dev/null +++ b/tools/images/chr-source/action.yaml @@ -0,0 +1,32 @@ +name: 'Prepare Chromium Sources Image' +description: 'Check and build chromium source image by version' + +inputs: + version: + description: 'Chromium Version (example 91.0.4472.146)' + required: true + +runs: + using: "composite" + steps: + - name: Checking image for "${{ inputs.version }}" + shell: bash + run: | + VERSION="${{ inputs.version }}" + IS_PRESENT=$(docker inspect --type=image uazo/chromium:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + IS_PRESENT=$(docker manifest inspect uazo/chromium:$VERSION > /dev/null ; echo $?) + if [ $IS_PRESENT -ne "0" ]; then + echo "Image not found" + + mkdir chr-source + pushd chr-sourced/ + + git clone https://github.com/uazo/bromite-buildtools + cd bromite-buildtools/images/chr-source/ + DOCKER_BUILDKIT=1 docker build -t uazo/chromium:$VERSION --progress plain --build-arg VERSION=$VERSION . + + popd + rm -rf chr-source + fi + fi diff --git a/tools/images/chr-source/depot_tools.diff b/tools/images/chr-source/depot_tools.diff new file mode 100644 index 00000000..41dc7513 --- /dev/null +++ b/tools/images/chr-source/depot_tools.diff @@ -0,0 +1,63 @@ +diff --git a/gclient_scm.py b/gclient_scm.py +index f58c07a8..c8409726 100644 +--- a/gclient_scm.py ++++ b/gclient_scm.py +@@ -1137,31 +1137,29 @@ class GitWrapper(SCMWrapper): + clone_cmd.append(url) + + template_dir = None ++ use_fetch = False ++ ++ tmp_dir = tempfile.mkdtemp( ++ prefix='_gclient_%s_' % os.path.basename(self.checkout_path), ++ dir=parent_dir) ++ + if hasattr(options, 'no_history') and options.no_history: + if gclient_utils.IsGitSha(revision): +- # In the case of a subproject, the pinned sha is not necessarily the +- # head of the remote branch (so we can't just use --depth=N). Instead, +- # we tell git to fetch all the remote objects from SHA..HEAD by means +- # of a template git dir which has a 'shallow' file pointing to the +- # sha. +- template_dir = tempfile.mkdtemp(prefix='_gclient_gittmp_%s' % +- os.path.basename(self.checkout_path), +- dir=parent_dir) +- self._Run(['init', '--bare', template_dir], +- options, +- cwd=self._root_dir) +- with open(os.path.join(template_dir, 'shallow'), +- 'w') as template_file: +- template_file.write(revision) +- clone_cmd.append('--template=' + template_dir) ++ self._Run(['init', tmp_dir], options, cwd=self._root_dir) ++ ++ self._Run(['-C', tmp_dir, 'remote', 'add', 'origin', url], options, cwd=self._root_dir) ++ ++ clone_cmd = cfg + ['-C', tmp_dir, 'fetch', '--progress'] ++ clone_cmd.append('--depth=1') ++ clone_cmd.append(url) ++ clone_cmd.append(revision) ++ use_fetch = True + else: + # Otherwise, we're just interested in the HEAD. Just use --depth. + clone_cmd.append('--depth=1') + +- tmp_dir = tempfile.mkdtemp(prefix='_gclient_%s_' % +- os.path.basename(self.checkout_path), +- dir=parent_dir) +- clone_cmd.append(tmp_dir) ++ if use_fetch == False: ++ clone_cmd.append(tmp_dir) + + try: + self._Run(clone_cmd, +@@ -1452,6 +1450,9 @@ class GitWrapper(SCMWrapper): + if refspec: + fetch_cmd.append(refspec) + ++ if hasattr(options, 'no_history') and options.no_history: ++ fetch_cmd.append('--depth=1') ++ + if prune: + fetch_cmd.append('--prune') + if options.verbose: diff --git a/tools/images/chr-source/prepare-build.sh b/tools/images/chr-source/prepare-build.sh new file mode 100644 index 00000000..fc2ff4b5 --- /dev/null +++ b/tools/images/chr-source/prepare-build.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +RED='\033[0;31m' +NC='\033[0m' # No Color + +echo -e ${RED} -------- chromium version is: $VERSION ${NC} + +echo -e ${RED} -------- cloning depot_tools ${NC} +git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + +echo -e ${RED} -------- apply depot_tools patch ${NC} +cd depot_tools/ +git apply ../depot_tools.diff +git apply ../remove_ninja_uploader.diff +cd .. + +echo -e ${RED} -------- set envs ${NC} +PATH=/home/lg/working_dir/depot_tools:$PATH +echo $PATH + +echo -e ${RED} -------- download chromium repo ${NC} +mkdir ./chromium +cd ./chromium + +export DEPOT_TOOLS_UPDATE=0 + +gclient root + +mkdir ./src +cd ./src + +#CHR_SOURCE=https://chromium.googlesource.com/chromium/src.git +CHR_SOURCE=https://github.com/chromium/chromium.git + +git init +git remote add origin $CHR_SOURCE + +git fetch --depth 2 $CHR_SOURCE +refs/tags/$VERSION:chromium_$VERSION +git checkout $VERSION +VERSION_SHA=$( git show-ref -s $VERSION | head -n1 ) + +echo >../.gclient "solutions = [" +echo >>../.gclient " { \"name\" : 'src'," +echo >>../.gclient " \"url\" : '$CHR_SOURCE@$VERSION_SHA'," +echo >>../.gclient " \"deps_file\" : 'DEPS'," +echo >>../.gclient " \"managed\" : True," +echo >>../.gclient " \"custom_deps\" : {" +echo >>../.gclient " }," +echo >>../.gclient " \"custom_vars\": {" +echo >>../.gclient " \"checkout_android_prebuilts_build_tools\": True," +echo >>../.gclient " \"checkout_telemetry_dependencies\": False," +echo >>../.gclient " \"codesearch\": 'Debug'," +echo >>../.gclient " }," +echo >>../.gclient " }," +echo >>../.gclient "]" +echo >>../.gclient "target_os=['android']" + +git submodule foreach git config -f ./.git/config submodule.$name.ignore all +git config --add remote.origin.fetch '+refs/tags/*:refs/tags/*' +#git config diff.ignoreSubmodules all + +echo -e ${RED} -------- sync third_party repos ${NC} +gclient sync -D --no-history --nohooks + +git config user.email "you@example.com" +git config user.name "Your Name" + +# remove origin for chromium +# git remote remove origin + +echo -e ${RED} -------- running hooks ${NC} +gclient runhooks + +echo -e ${RED} -------- download objdump ${NC} +tools/clang/scripts/update.py --package=objdump + +echo -e ${RED} -------- build rc ${NC} +cd build/toolchain/win/rc +git clone -q https://github.com/nico/hack +cd hack/res + +../../../../../../third_party/android_ndk/toolchains/llvm/prebuilt/linux-x86_64/bin/clang++ \ + -std=c++14 rc.cc -Wall \ + -Wno-c++11-narrowing -O2 -fno-rtti -fno-exceptions -DNDEBUG \ + -o rc-linux64 -fuse-ld=lld -target x86_64-unknown-linux-gnu +cd ../../../../../../ +cp build/toolchain/win/rc/hack/res/rc-linux64 build/toolchain/win/rc/linux64/rc + diff --git a/tools/images/chr-source/remove_ninja_uploader.diff b/tools/images/chr-source/remove_ninja_uploader.diff new file mode 100644 index 00000000..b3432b8f --- /dev/null +++ b/tools/images/chr-source/remove_ninja_uploader.diff @@ -0,0 +1,12 @@ +diff --git a/ninjalog_uploader_wrapper.py b/ninjalog_uploader_wrapper.py +index fbfca3f2..3cd9e9b1 100755 +--- a/ninjalog_uploader_wrapper.py ++++ b/ninjalog_uploader_wrapper.py +@@ -75,6 +75,7 @@ You can find a more detailed explanation in + + + def main(): ++ return 0 + config = LoadConfig() + + if len(sys.argv) == 2 and sys.argv[1] == 'opt-in': diff --git a/tools/images/docker-host/Dockerfile b/tools/images/docker-host/Dockerfile new file mode 100644 index 00000000..6ceb4868 --- /dev/null +++ b/tools/images/docker-host/Dockerfile @@ -0,0 +1,62 @@ +#FROM nestybox/ubuntu-focal-systemd-docker +FROM ubuntu:20.04 + +RUN set -xe && \ + echo '#!/bin/sh' > /usr/sbin/policy-rc.d && \ + echo 'exit 101' >> /usr/sbin/policy-rc.d && \ + chmod +x /usr/sbin/policy-rc.d && \ + dpkg-divert --local --rename --add /sbin/initctl && \ + cp -a /usr/sbin/policy-rc.d /sbin/initctl && \ + sed -i 's/^exit.*/exit 0/' /sbin/initctl && \ + echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup && \ + echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean && \ + echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean && \ + echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean && \ + echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages && \ + echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes && \ + echo 'Apt::AutoRemove::SuggestsImportant "false";' > /etc/apt/apt.conf.d/docker-autoremove-suggests + +RUN mkdir -p /run/systemd && echo 'docker' > /run/systemd/container + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + systemd systemd-sysv libsystemd0 ca-certificates dbus \ + iptables iproute2 kmod locales sudo udev && \ + echo "ReadKMsg=no" >> /etc/systemd/journald.conf && \ + apt-get clean -y && \ + rm -rf /var/cache/debconf/* /var/lib/apt/lists/* /var/log/* /tmp/* /var/tmp/* \ + /usr/share/doc/* /usr/share/man/* /usr/share/local/* && \ + useradd --create-home --shell /bin/bash admin && \ + echo "admin:admin" | chpasswd && \ + adduser admin sudo + +STOPSIGNAL SIGRTMIN+3 + +RUN apt-get update && \ + apt-get install --no-install-recommends -y apt-transport-https ca-certificates \ + curl gnupg-agent software-properties-common && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + apt-key fingerprint 0EBFCD88 && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + apt-get update && apt-get install --no-install-recommends -y docker-ce docker-ce-cli containerd.io=1.4.4-1 && \ + apt-get clean -y && \ + rm -rf /var/cache/debconf/* /var/lib/apt/lists/* /var/log/* /tmp/* /var/tmp/* \ + /usr/share/doc/* /usr/share/man/* /usr/share/local/* && \ + usermod -a -G docker admin + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + openssh-server && \ + mkdir /home/admin/.ssh && \ + chown admin:admin /home/admin/.ssh + +# Extra deps +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y \ + sudo \ + pigz \ + && rm -rf /var/lib/apt/list/* + +ENTRYPOINT ["/sbin/init", "--log-level=err"] +CMD ["/bin/bash"] diff --git a/tools/images/github-runner/.env.example b/tools/images/github-runner/.env.example new file mode 100644 index 00000000..737fd47b --- /dev/null +++ b/tools/images/github-runner/.env.example @@ -0,0 +1,6 @@ +RUNNER_NAME=pd-gh-runner +GITHUB_PERSONAL_TOKEN= +GITHUB_OWNER= +GITHUB_REPOSITORY= +RUNNER_LABELS=