tor-browser-build/projects/browser/build

#!/bin/bash
[% c("var/set_default_env") -%]
distdir=/var/tmp/dist/[% project %]
export TORBROWSER_VERSION='[% c("version") %]'

mkdir -p $distdir

scripts_dir=/var/tmp/build_scripts
mkdir -p "$scripts_dir"

OUTDIR='[% dest_dir _ "/" _ c("filename") %]'
mkdir -p $OUTDIR

# When we build with MULTI_LINGUAL=1, the browser will be packaged inside a
# directory named tor-browser (instead of tor-browser_en-US). Therefore we
# stage everything under tor-browser-stage to avoid a conflict.
TB_STAGE_DIR=$distdir/tor-browser-stage
GENERATEDPREFSPATH=$rootdir/Bundle-Data/PTConfigs/generated-prefs.js
# Create initially empty prefs file where we can dump our conditionally included/genetered prefs
touch "$GENERATEDPREFSPATH"

[% IF c("var/macos") %]
  TBDIR="$TB_STAGE_DIR/[% c("var/Project_Name") %].app"
  [% IF c("var/macos_universal") %]
    TBDIR_AARCH64="$TB_STAGE_DIR/[% c("var/Project_Name") %]-aarch64.app"
    TBDIRS=("$TBDIR" "$TBDIR_AARCH64")
  [% ELSE -%]
    TBDIRS=("$TBDIR")
  [% END -%]

  DOCSPATH=Contents/Resources/[% c('var/ProjectName') %]/Docs
  EXTSPATH=Contents/Resources/distribution/extensions
  TORBINPATH=Contents/MacOS/Tor
  TORCONFIGPATH=Contents/Resources/TorBrowser/Tor

  tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/libdmg') %]
  export PATH=/var/tmp/dist/libdmg-hfsplus:$PATH
[% ELSE %]
  TBDIR=$TB_STAGE_DIR/Browser
  TBDIRS=("$TBDIR")

  DOCSPATH=[% c('var/ProjectName') %]/Docs
  EXTSPATH=[% c('var/ProjectName') %]/Data/Browser/profile.default/extensions
  TORBINPATH=TorBrowser/Tor
  TORCONFIGPATH=TorBrowser/Data/Tor

  mkdir -p "$TBDIR/[% c('var/ProjectName') %]/Data/Browser/Caches"
[% END %]

for tbdir in "${TBDIRS[@]}"
do
  mkdir -p "$tbdir/$EXTSPATH"
done

# Extract the MAR tools.
unzip -d $rootdir $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip
MARTOOLS=$rootdir/mar-tools

mv [% c('input_files_by_name/noscript') %] "$TBDIR/$EXTSPATH/{73a6fe31-595d-460b-a920-fcc0f8843232}.xpi"

[% IF c("var/tor-browser") -%]
  function mv_tbdir {
    declare args=($@)
    local dest="${args[-1]}"
    unset args[-1]
    mv ${args[@]} "$TBDIR/$dest"
    [% IF c("var/macos_universal") -%]
      for src in ${args[@]}
      do
        mv "aarch64/$src" "$TBDIR_AARCH64/$dest"
      done
    [% END -%]
  }

  tar -xvf [% c('input_files_by_name/tor-expert-bundle') %]
  [% IF c("var/macos_universal") -%]
    mkdir aarch64
    tar -C aarch64 -xf [% c('input_files_by_name/tor-expert-bundle-aarch64') %]
  [% END -%]

  # geoip(6) and anything else that belongs in the data dir from the expert bundle
  mkdir -p "$TBDIR/$TORCONFIGPATH" [% IF c("var/macos_universal") %]"$TBDIR_AARCH64/$TORCONFIGPATH"[% END %]
  mv_tbdir data/{geoip,geoip6} "$TORCONFIGPATH"

  # Bridge lists will be consumed later in `bridges_conf`
  mv tor/pluggable_transports/bridges_list.*.txt $rootdir
  [% IF c("var/macos_universal") %]
    rm -f aarch64/tor/pluggable_transports/bridges_list.*.txt
  [% END -%]

  # Snowflake's README is the only documentation shipped in tor-expert-bundle
  mkdir -p "$TBDIR/$DOCSPATH/snowflake" [% IF c("var/macos_universal") %]"$TBDIR_AARCH64/$DOCSPATH/snowflake"[% END %]
  mv_tbdir tor/pluggable_transports/README.SNOWFLAKE.md "$DOCSPATH/snowflake/README.md"

  # Move the PTs to where TB expects them
  mkdir -p "$TBDIR/$TORBINPATH" [% IF c("var/macos_universal") %]"$TBDIR_AARCH64/$TORBINPATH"[% END %]
  mv_tbdir tor/pluggable_transports "$TORBINPATH/PluggableTransports"

  # Move tor and dependencies to where TB expects them
  mv_tbdir tor/* "$TORBINPATH"

  # on linux, libstdc++ lives in it's own directory
  [% IF c("var/linux") %]
    find $TBDIR/$TORBINPATH
    mkdir -p "$TBDIR/$TORBINPATH/libstdc++"
    mv "$TBDIR/$TORBINPATH"/libstdc++.so.* "$TBDIR/$TORBINPATH/libstdc++"
  [% END %]

  # the expert bundle includes tor-gencert, which isn't needed for browser releases
  [% IF c("var/windows") %]
    rm "$TBDIR/$TORBINPATH/tor-gencert.exe"
  [% END %]

[% END -%]

for tbdir in "${TBDIRS[@]}"
do
  [% IF c("var/macos") -%]
    tbdir="$tbdir/Contents/Resources"
  [% END -%]
  tar -C "$tbdir" -xf [% c('input_files_by_name/fonts') %]
done

[% IF c("var/linux") %]
  cat > ${TB_STAGE_DIR}/start-[% c("var/project-name") %].desktop << 'RBM_TB_EOF'
[% INCLUDE 'RelativeLink/start-browser.desktop' -%]
RBM_TB_EOF
  cat > ${TB_STAGE_DIR}/Browser/start-[% c("var/project-name") %] << 'RBM_TB_EOF'
[% INCLUDE 'RelativeLink/start-browser' -%]
RBM_TB_EOF
  cat > ${TB_STAGE_DIR}/Browser/execdesktop << 'RBM_TB_EOF'
[% INCLUDE 'RelativeLink/execdesktop' -%]
RBM_TB_EOF
  chmod +x ${TB_STAGE_DIR}/start-[% c("var/project-name") %].desktop \
           ${TB_STAGE_DIR}/Browser/start-[% c("var/project-name") %] \
           ${TB_STAGE_DIR}/Browser/execdesktop
  cp ${TB_STAGE_DIR}/start-[% c("var/project-name") %].desktop \
     ${TB_STAGE_DIR}/Browser
  [% IF c("var/namecoin") %]
    pushd ${TB_STAGE_DIR}/Browser/
    patch -p1 < $rootdir/namecoin.patch
    popd
  [% END %]
  # Make sure we get the desired scrollbar behavior with Gtk3, see bug 27546.
  GTK_SETTINGS_DIR=${TB_STAGE_DIR}/Browser/.config/gtk-3.0
  mkdir -p $GTK_SETTINGS_DIR
  cp $rootdir/gtk3-settings.ini $GTK_SETTINGS_DIR/settings.ini
[% END %]

tar -C ${TB_STAGE_DIR} -xf [% c('input_files_by_name/firefox') %]/browser.tar.gz

[% IF c("var/macos_universal") -%]
  # Use symlink in tmp dir to extract "Tor Browser.app" to "Tor Browser-aarch64.app"
  ff_aarch64=$(mktemp -d)
  ln -s "$TBDIR_AARCH64" "$ff_aarch64/[% c("var/Project_Name") %].app"
  tar -C $ff_aarch64 -xf [% c('input_files_by_name/firefox-aarch64') %]/browser.tar.gz
[% END %]

[% IF c("var/tor-browser") -%]
  # Include the Tor Browser manual
  TMP_MANUAL_PATH=$rootdir/tmp_manual/
  mkdir $TMP_MANUAL_PATH
  pushd $TMP_MANUAL_PATH
  tar -xf $rootdir/[% c('input_files_by_name/manual') %]
  find . -exec [% c("touch") %] {} \;
  for tbdir in "${TBDIRS[@]}"
  do
    find chrome/ | sort | zip -X -@ "$tbdir[% IF c('var/macos') %]/Contents/Resources[% END %]/browser/omni.ja"
  done
  popd
  rm -rf $TMP_MANUAL_PATH
[% END -%]

[% IF c("var/namecoin") %]
  # Extract Electrum-NMC
  tar -C "$TBDIR/TorBrowser" -xf [% c('input_files_by_name/electrum-nmc') %]

  # Extract ncprop279
  mkdir "$TBDIR/TorBrowser/ncprop279"
  tar -C "$TBDIR/TorBrowser/ncprop279" -xf [% c('input_files_by_name/ncprop279') %]

  # Extract StemNS
  tar -C "$TBDIR/TorBrowser" -xf [% c('input_files_by_name/stemns') %]
[% END %]

[% IF c("var/linux");
     SET bundledata_osname = 'linux';
   ELSIF c("var/macos");
     SET bundledata_osname = 'mac';
   ELSIF c("var/windows");
     SET bundledata_osname = 'windows';
   END; %]

[% IF c("var/macos") %]
  # The Bundle-Data is designed for embedded data, so we need to modify
  # the structure when we want the data to be outside the app directory.
  # We also create an override.ini file to disable the profile migrator.
  SKELETON_TMP=$rootdir/Bundle-Data/mac-skeleton-tmp
  SKELETON_TMP_RESOURCES=$SKELETON_TMP/Contents/Resources
  mkdir -p $SKELETON_TMP_RESOURCES/browser
  echo "[XRE]" > $SKELETON_TMP_RESOURCES/browser/override.ini
  echo "EnableProfileMigrator=0" >> $SKELETON_TMP_RESOURCES/browser/override.ini
  [% IF c("var/tor-browser") -%]
    mkdir -p $SKELETON_TMP_RESOURCES/TorBrowser/Tor
    cp -p Bundle-Data/mac/TorBrowser/Data/Tor/torrc-defaults $SKELETON_TMP_RESOURCES/TorBrowser/Tor/
  [% END -%]
  rm -Rf Bundle-Data/mac
  mv $SKELETON_TMP Bundle-Data/mac

  [% IF c("var/tor-browser") -%]
    # Install a "tor" shim that sets the working directory. See #10030.
    for tbdir in "${TBDIRS[@]}"
    do
      mv "$tbdir/$TORBINPATH/tor" "$tbdir/$TORBINPATH/tor.real"
      cp Bundle-Data/mac-tor.sh "$tbdir/$TORCONFIGPATH/tor"
    done
  [% END -%]

  tar -C Bundle-Data/mac-applications.dmg -c . | tar -C $TB_STAGE_DIR -x
[% END %]

for tbdir in "${TBDIRS[@]}"
do
  mkdir -p "$tbdir/$DOCSPATH"
  cp -a Bundle-Data/Docs/* "$tbdir/$DOCSPATH"
  tar -C Bundle-Data/[% bundledata_osname %] \
    [% IF ! c("var/namecoin") %]--exclude=*Electrum-NMC* --exclude=*ncprop279*[% END %] \
    -c . | tar -C "$tbdir[% IF ! c("var/macos") %]/[% c('var/ProjectName') %][% END %]" -x
done

[% IF c("var/tor-browser") -%]
  for tbdir in "${TBDIRS[@]}"
  do
    cat Bundle-Data/PTConfigs/[% bundledata_osname %]/torrc-defaults-appendix >> "$tbdir/$TORCONFIGPATH/torrc-defaults"
  done
  cat Bundle-Data/PTConfigs/bridge_prefs.js >> "$GENERATEDPREFSPATH"

  function bridges_conf {
    local pt="$1"
    local i=1
    while read -r line
    do
      echo "pref(\"extensions.torlauncher.default_bridge.$pt.$i\", \"$line\");" >> "$GENERATEDPREFSPATH"
      i=$((i + 1))
    done < "$rootdir/bridges_list.$pt.txt"
  }
  bridges_conf obfs4
  bridges_conf meek-azure
  bridges_conf snowflake
[% END -%]

[% IF c("var/linux") %]
  chmod 700 ${TB_STAGE_DIR}/Browser/[% c('var/ProjectName') %]/Data/Browser
  [% IF c("var/tor-browser") -%]
    chmod 700 ${TB_STAGE_DIR}/Browser/TorBrowser/Data/Tor
  [% END -%]
[% END %]

# With multi-lingual builds, we use "ALL" in the package name as the locale for
# update purposes. But we do not include "ALL" in the name of the directory that
# is inside the package (in other words, users will not see tor-browser_ALL
# after they install Tor Browser).
PKG_LOCALE="ALL"
PKG_DIR='[% c("var/project-name") %]'

for tbdir in "${TBDIRS[@]}"
do
  tbdir="$tbdir[% IF c('var/macos') %]/Contents/Resources[% END %]/browser/"
  pushd "$tbdir"
  unzip omni.ja defaults/preferences/[% c("var/prefs_file") %] || [ $? -lt 3 ]
  # Append our built extension-overrides.js to the preferences file
  cat "$GENERATEDPREFSPATH" >> defaults/preferences/[% c("var/prefs_file") %]
  cp defaults/preferences/[% c("var/prefs_file") %] $rootdir
  [% c("touch") %] defaults/preferences/[% c("var/prefs_file") %]
  zip -Xm omni.ja defaults/preferences/[% c("var/prefs_file") %]
  rm -rf defaults
  # create tbb_version.json file for #25020
  echo '{"version":"[% c("var/torbrowser_version") %]","architecture":"[% c("var/mar_osname") %]","channel":"[% c("var/channel") %]","locale":"en-US"}' > ../tbb_version.json
  popd
done

[% IF c("var/windows") %]
  tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/nsis') %]
  export PATH="/var/tmp/dist/nsis/bin:$PATH"

  mv $rootdir/windows-installer $distdir/windows-installer
  mv ${TB_STAGE_DIR} $distdir/windows-installer/"[% c('var/Project_Name') %]"
  mv $distdir/windows-installer ${TB_STAGE_DIR}
[% END %]

[% IF c("var/macos_universal") -%]
  # unify.py requires lipo, so we need to add cctools in the PATH
  tar -C /var/tmp/dist -xf $rootdir/[% c('input_files_by_name/macosx-toolchain') %]
  export PATH="/var/tmp/dist/macosx-toolchain/cctools/bin:$PATH"
  firefox_src=/var/tmp/dist/firefox-src
  mkdir -p $firefox_src
  tar -C $firefox_src -xf $rootdir/[% c('input_files_by_name/src-firefox') %]
  firefox_src=$(echo $firefox_src/firefox-*)
  # Temporarily move noscript outside of $TBDIR to avoid error when running unify.py:
  #   File "/var/tmp/dist/firefox-src/firefox-69721baf14f0/python/mozbuild/mozpack/mozjar.py", line 452, in _getreader
  #     raise JarReaderError(
  # mozpack.mozjar.JarReaderError: Central directory and file header mismatch. Corrupted archive?
  #
  # See https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/issues/40439#note_2838724
  mv "$TBDIR/$EXTSPATH/{73a6fe31-595d-460b-a920-fcc0f8843232}.xpi" $rootdir/ns.xpi
  MOZ_AUTOMATION=1 $firefox_src/mach python $firefox_src/toolkit/mozapps/installer/unify.py "$TBDIR" "$TBDIR_AARCH64"
  mkdir -p "$TBDIR/$EXTSPATH"
  mv $rootdir/ns.xpi "$TBDIR/$EXTSPATH/{73a6fe31-595d-460b-a920-fcc0f8843232}.xpi"
  rm -Rf "$TBDIR_AARCH64"
[% END -%]

[% IF c("var/windows") %]
  TBDIR="$distdir/$PKG_DIR/[% c("var/Project_Name") %]/Browser"
[% ELSIF c("var/macos") %]
  TBDIR="$distdir/$PKG_DIR/[% c("var/Project_Name") %].app"
[% ELSE %]
  TBDIR="$distdir/$PKG_DIR/Browser"
[% END %]

cat > "$scripts_dir/create-$PKG_DIR" << SCRIPT_EOF
#!/bin/bash
set -e
cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR

pushd "$TBDIR[% IF c("var/macos") %]/Contents/Resources/[% END %]"
rm -f precomplete
python $MARTOOLS/createprecomplete.py
popd

cd $distdir

[% IF c("var/build_mar") -%]
  # Create full MAR file and compressed package.
  [% SET mar_file = c("var/project-name") _ '-' _ c("var/mar_osname") _ '-' _ c("var/torbrowser_version") _ '_${PKG_LOCALE}.mar' %]
  MAR=$MARTOOLS/mar \
  MOZ_PRODUCT_VERSION=[% c("var/torbrowser_version") %] \
  MAR_CHANNEL_ID=torbrowser-torproject-[% c("var/channel") %] \
  $MARTOOLS/make_full_update.sh -q $OUTDIR/[% mar_file %] "$TBDIR"
[% END -%]

[% IF c("var/linux") %]
  [% c('tar', {
        tar_src => [ '$PKG_DIR' ],
        tar_args => '-cJf $OUTDIR/' _ c("var/project-name") _ '-' _ c("var/mar_osname") _ '-' _ c("var/torbrowser_version") _ '_${PKG_LOCALE}.tar.xz',
    }) %]
[% ELSIF c("var/macos") %]
  [% c('var/ddmg', {
        dmg_src => '"$PKG_DIR"',
        dmg_out => '$OUTDIR/' _ c('var/ProjectName') _ '-' _ c("var/torbrowser_version") _ '-' _ c("var/mar_osname") _ '_${PKG_LOCALE}.dmg',
    }) %]
[% ELSIF c("var/windows") %]
  find "$PKG_DIR" -exec [% c("touch") %] {} \;
  pushd "$PKG_DIR"
  makensis [% c("var/projectname") %].nsi
  # Working around NSIS braindamage
  mv [% c("var/projectname") %]-install.exe browser-install-tmp.exe
  python3 $rootdir/pe_checksum_fix.py
  mv browser-install-tmp2.exe [% c("var/projectname") %]-install.exe
  rm browser-install-tmp.exe
  mv [% c("var/projectname") %]-install.exe $OUTDIR/[% c("var/projectname") %]-install[% IF c("var/windows-x86_64") %]-win64[% END %]-[% c("var/torbrowser_version") %]_${PKG_LOCALE}.exe
  popd
[% END %]
rm -rf $distdir/${PKG_DIR}
SCRIPT_EOF

cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
[% IF c("var/linux-x86_64") -%]
  [% IF c("var/tor-browser") -%]
    cp $rootdir/[% c('input_files_by_name/firefox') %]/browser-debug.tar.xz "$OUTDIR"/[% c("var/project-name") %]-[% c("var/mar_osname") %]-debug.tar.xz
  [% END -%]
  [% IF !c("var/asan") -%]
    cp $rootdir/[% c('input_files_by_name/firefox') %]/geckodriver-linux64.tar.xz "$OUTDIR"/
  [% END -%]
[% END -%]
[%IF c("var/tor-browser") -%]
  tor_expert_bundle_src="[% c("input_files_by_name/tor-expert-bundle") %]"
  # strip off trailing "$buildid.tar.gz"
  tor_expert_bundle_dest=${tor_expert_bundle_src:0:-14}.tar.gz
  cp $rootdir/[% c("input_files_by_name/tor-expert-bundle") %] "$OUTDIR"/$tor_expert_bundle_dest
  [% IF c("var/macos_universal") %]
    tor_expert_bundle_src="[% c('input_files_by_name/tor-expert-bundle-aarch64') %]"
    # strip off trailing "$buildid.tar.gz"
    tor_expert_bundle_dest=${tor_expert_bundle_src:0:-14}.tar.gz
    cp $rootdir/[% c('input_files_by_name/tor-expert-bundle-aarch64') %] "$OUTDIR"/$tor_expert_bundle_dest
  [% END -%]
[% END -%]
[% IF c("var/build_infos_json") -%]
  cp $rootdir/[% c('input_files_by_name/firefox') %]/build-infos.json "$OUTDIR"/build-infos-[% c("var/mar_osname") %].json
[% END -%]

chmod 775 $rootdir/run_scripts "$scripts_dir"/*
$rootdir/run_scripts [% c("num_procs") %] "$scripts_dir"