From 4f6ea3ccab40dd02c9d9eae8e3225a28876930a3 Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Tue, 8 Jun 2021 11:49:59 +0200 Subject: [PATCH 01/11] updates --- common | 2 +- settings | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common b/common index 5bce528..bf95af6 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 5bce5285fa7046e6987ec3e5a8931ac17ca6c7c0 +Subproject commit bf95af6af7ccf96fccb58d553ff552c41558e5d3 diff --git a/settings b/settings index 5cbb8b8..0532d6c 160000 --- a/settings +++ b/settings @@ -1 +1 @@ -Subproject commit 5cbb8b8b5401188f6e92f7880d33963ee0dcd8ca +Subproject commit 0532d6cb861db42f3ccfbb3a90908e7cc9f31013 From 779a4bc1d08da0617e01e875c6e16c4652b2f6a9 Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Tue, 8 Jun 2021 13:28:39 +0200 Subject: [PATCH 02/11] updates --- pybuild.py | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 pybuild.py diff --git a/pybuild.py b/pybuild.py new file mode 100644 index 0000000..6aa58ea --- /dev/null +++ b/pybuild.py @@ -0,0 +1,85 @@ +#!/usr/bin/python3 + +# +# pybuild.py - try move functionality away from that too big of a script. +# + +import optparse +import sys + +parser = optparse.OptionParser() + +#parser.add_option('-o', '--output', dest="output_filename", default="default.out") +#parser.add_option('-v', '--verbose',dest="verbose",default=False,action="store_true") +#parser.add_option('--version',dest="version",default=1.0,type="float") +parser.add_option('-x', '--cross', dest='cross_compile', default=False, action="store_true") +parser.add_option('-s', '--src', dest='src', default='release') +parser.add_option('-t', '--distro', dest='distro', default='win') + +options, remainder = parser.parse_args() + +#if len(sys.argv[1:])>=1: +# print('ARGV :', sys.argv[1:]) +# print('VERSION :', options.version) +# print('VERBOSE :', options.verbose) +# print('OUTPUT :', options.output_filename) +# print('REMAINING :', remainder) +# + + +for arg in remainder: + if arg == 'all': + print("[debug] all") + elif arg == 'clean': + print("[debug] clean") + else: + print("[debug] unknown command: ", arg) + + + +# Print help message +if len(remainder)<1: + print("""# Use: + + pybuild [] clean | all | + +# Options: + + -x,--cross - build windows setup.exe from linux + -s,--src - release,nightly,tor-browser + (default=release) + -t,--distro - deb,rpm,win (default=win) + +# Targets: + + all - all steps from fetch to producing setup.exe + clean - clean everything, includeing extracted/fetched sources + + fetch - wget or hg clone or git pull + extract - nop if not wget + lw_do_patches - [librewolf] patch the source + build - build the browser + lw_post_build - [librewolf] insert our settings + package - package the browser into zip/apk + lw_artifacts - [librewolf] build setup.exe + +# Utilities: + + git_subs - git update submodules + + git_init - put the source folder in a .git reposity + git_reset - reset the source folder from the .git repo + + deps_deb - install dependencies with apt + deps_rpm - install dependencies with dnf + deps_pkg - install dependencies on freebsd + + rustup - update rust + mach_env - create mach environment +""") + sys.exit(1) +# eof + + + + From 312b272fd045e914add86a3f5e76a6e221d385bb Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Tue, 8 Jun 2021 14:21:10 +0200 Subject: [PATCH 03/11] commandline parsed --- pybuild.py | 143 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 119 insertions(+), 24 deletions(-) diff --git a/pybuild.py b/pybuild.py index 6aa58ea..7e0092f 100644 --- a/pybuild.py +++ b/pybuild.py @@ -9,36 +9,134 @@ import sys parser = optparse.OptionParser() -#parser.add_option('-o', '--output', dest="output_filename", default="default.out") -#parser.add_option('-v', '--verbose',dest="verbose",default=False,action="store_true") -#parser.add_option('--version',dest="version",default=1.0,type="float") parser.add_option('-x', '--cross', dest='cross_compile', default=False, action="store_true") parser.add_option('-s', '--src', dest='src', default='release') parser.add_option('-t', '--distro', dest='distro', default='win') options, remainder = parser.parse_args() -#if len(sys.argv[1:])>=1: -# print('ARGV :', sys.argv[1:]) -# print('VERSION :', options.version) -# print('VERBOSE :', options.verbose) -# print('OUTPUT :', options.output_filename) -# print('REMAINING :', remainder) -# - - -for arg in remainder: - if arg == 'all': - print("[debug] all") - elif arg == 'clean': - print("[debug] clean") - else: - print("[debug] unknown command: ", arg) -# Print help message -if len(remainder)<1: +# Targets: +def execute_fetch(): + print("[debug] doing target -> fetch") +def execute_extract(): + print("[debug] doing target -> extract") +def execute_lw_do_patches(): + print("[debug] doing target -> lw_do_patches") +def execute_build(): + print("[debug] doing target -> build") +def execute_lw_post_build(): + print("[debug] doing target -> lw_post_build") +def execute_package(): + print("[debug] doing target -> package") +def execute_lw_artifacts(): + print("[debug] doing target -> lw_artifacts") + +# Main targets: +def execute_all(): + execute_fetch() + execute_extract() + execute_lw_do_patches() + execute_build() + execute_lw_post_build() + execute_package() + execute_lw_artifacts() +def execute_clean(): + print("[debug] doing target -> clean") + +# Utilities: +def execute_git_subs(): + print("[debug] doing target -> git_subs") + +def execute_git_init(): + print("[debug] doing target -> git_init") +def execute_git_reset(): + print("[debug] doing target -> git_reset") + +def execute_deps_deb(): + print("[debug] doing target -> deps_deb") +def execute_deps_rpm(): + print("[debug] doing target -> deps_rpm") +def execute_deps_pkg(): + print("[debug] doing target -> deps_pkg") + +def execute_rustup(): + print("[debug] doing target -> rustup") +def execute_mach_env(): + print("[debug] doing target -> mach_env") + + + + + + +# main commandline interpretation +if len(remainder)>0: + if not options.src in ['release','nightly','tor-browser']: + print("error: option --src invalid value") + sys.exit(1) + if not options.distro in ['deb','rpm', 'win']: + print("error: option --distro invalid value") + sys.exit(1) + + print("[debug] ----------") + print("[debug] --cross = ", options.cross_compile) + print("[debug] --src = ", options.src) + print("[debug] --distro = ", options.distro) + print("[debug] ----------") + + for arg in remainder: + if arg == 'all': + execute_all() + elif arg == 'clean': + execute_clean() + + # Targets: + + elif arg == 'fetch': + execute_fetch() + elif arg == 'extract': + execute_extract() + elif arg == 'lw_do_patches': + execute_lw_do_patches() + elif arg == 'build': + execute_build() + elif arg == 'lw_post_build': + execute_lw_post_build() + elif arg == 'package': + execute_package() + elif arg == 'lw_artifacts': + execute_lw_artifacts() + + # Utilities + + elif arg == 'git_subs': + execute_git_subs() + + elif arg == 'git_init': + execute_git_init() + elif arg == 'git_reset': + execute_git_reset() + + elif arg == 'deps_deb': + execute_deps_deb() + elif arg == 'deps_rpm': + execute_deps_rpm() + elif arg == 'deps_pkg': + execute_deps_pkg() + + elif arg == 'rustup': + execute_rustup() + elif arg == 'mach_env': + execute_mach_env() + + else: + print("error: unknown command on command line: ", arg) + sys.exit(1) +else: + # Print help message print("""# Use: pybuild [] clean | all | @@ -77,9 +175,6 @@ if len(remainder)<1: rustup - update rust mach_env - create mach environment """) - sys.exit(1) -# eof - From 1ef647850132d48823d781703ab71bc8185d8e75 Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Tue, 8 Jun 2021 15:40:10 +0200 Subject: [PATCH 04/11] updates --- pybuild.py | 45 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/pybuild.py b/pybuild.py index 7e0092f..44ff117 100644 --- a/pybuild.py +++ b/pybuild.py @@ -6,6 +6,7 @@ import optparse import sys +import os parser = optparse.OptionParser() @@ -15,6 +16,28 @@ parser.add_option('-t', '--distro', dest='distro', default='win') options, remainder = parser.parse_args() + +#print("[debug] ----------") +#print("[debug] --cross = ", options.cross_compile) +#print("[debug] --src = ", options.src) +#print("[debug] --distro = ", options.distro) +#print("[debug] ----------") + + + + +def enter_srcdir(): + pass +def leave_srcdir(): + pass +def exec(cmd): + # print command on stdout and sys.exit(1) on errors + pass + + + + + @@ -26,14 +49,22 @@ def execute_extract(): def execute_lw_do_patches(): print("[debug] doing target -> lw_do_patches") def execute_build(): - print("[debug] doing target -> build") + enter_srcdir() + cmd = "./mach build" + exec(cmd) + leave_srcdir() def execute_lw_post_build(): print("[debug] doing target -> lw_post_build") def execute_package(): - print("[debug] doing target -> package") + enter_srcdir() + cmd = "./mach package" + exec(cmd) + leave_srcdir() def execute_lw_artifacts(): print("[debug] doing target -> lw_artifacts") + + # Main targets: def execute_all(): execute_fetch() @@ -45,6 +76,10 @@ def execute_all(): execute_lw_artifacts() def execute_clean(): print("[debug] doing target -> clean") + + + + # Utilities: def execute_git_subs(): @@ -80,12 +115,6 @@ if len(remainder)>0: if not options.distro in ['deb','rpm', 'win']: print("error: option --distro invalid value") sys.exit(1) - - print("[debug] ----------") - print("[debug] --cross = ", options.cross_compile) - print("[debug] --src = ", options.src) - print("[debug] --distro = ", options.distro) - print("[debug] ----------") for arg in remainder: if arg == 'all': From 147e5f7be8766ec559be10d8ebded302194fc996 Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Wed, 9 Jun 2021 18:57:51 +0200 Subject: [PATCH 05/11] Implemented most non-librewolf functionality --- common | 2 +- pybuild.py | 190 ++++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 145 insertions(+), 47 deletions(-) diff --git a/common b/common index bf95af6..febf04e 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit bf95af6af7ccf96fccb58d553ff552c41558e5d3 +Subproject commit febf04e63f6eb4428e573a23cb5433d3d969eeac diff --git a/pybuild.py b/pybuild.py index 44ff117..dd678d1 100644 --- a/pybuild.py +++ b/pybuild.py @@ -1,7 +1,9 @@ -#!/usr/bin/python3 +#!/bin/env python3 + +pkgver='89.0' # -# pybuild.py - try move functionality away from that too big of a script. +# pybuild.py - try move functionality away from that too big/horrible build script. # import optparse @@ -10,15 +12,18 @@ import os parser = optparse.OptionParser() -parser.add_option('-x', '--cross', dest='cross_compile', default=False, action="store_true") -parser.add_option('-s', '--src', dest='src', default='release') -parser.add_option('-t', '--distro', dest='distro', default='win') +parser.add_option('-x', '--cross', dest='cross_compile', default=False, action="store_true") +parser.add_option('-n', '--no-execute', dest='no_execute', default=False, action="store_true") +parser.add_option('-l', '--no-librewolf', dest='no_librewolf', default=False, action="store_true") +parser.add_option('-s', '--src', dest='src', default='release') +parser.add_option('-t', '--distro', dest='distro', default='win') options, remainder = parser.parse_args() #print("[debug] ----------") #print("[debug] --cross = ", options.cross_compile) +#print("[debug] --no-execute = ", options.no_execute) #print("[debug] --src = ", options.src) #print("[debug] --distro = ", options.distro) #print("[debug] ----------") @@ -26,45 +31,145 @@ options, remainder = parser.parse_args() +def beep(): + print('\a') + + def enter_srcdir(): - pass + dir = "firefox-{}".format(pkgver) + if options.src == 'nightly': + dir = 'mozilla-unified' + elif options.src == 'tor-browser': + dir = 'tor-browser' + print("cd {}".format(dir)) + if not options.no_execute: + os.chdir(dir) + def leave_srcdir(): - pass + print("cd ..") + if not options.no_execute: + os.chdir("..") + def exec(cmd): - # print command on stdout and sys.exit(1) on errors - pass + print("{}".format(cmd)) + if not options.no_execute: + retval = os.system(cmd) + if retval != 0: + beep() + sys.exit(1) + +# Utilities: +def execute_git_subs(): + exec("git submodule update --recursive") + exec("git submodule foreach git pull origin master") + exec("git submodule foreach git merge origin master") + +def execute_git_init(): + enter_srcdir() + exec("rm -rf .git") + exec("git init") + exec("git config core.safecrlf false") + exec("git config commit.gpgsign false") + exec("git add -f * .[a-z]*") + exec("git commit -am 'Initial commit'") + leave_srcdir() + + +def execute_deps_deb(): + deps1 = "python python-dev python3 python3-dev python3-distutils clang pkg-config libpulse-dev gcc" + deps2 = "curl wget nodejs libpango1.0-dev nasm yasm zip m4 libgtk-3-dev libgtk2.0-dev libdbus-glib-1-dev" + deps3 = "libxt-dev python3-pip mercurial automake autoconf libtool m4" + exec("apt install -y {} {} {}".format(deps1,deps2,deps3)) + +def execute_deps_rpm(): + deps1 = "python3 python3-distutils-extra clang pkg-config gcc curl wget nodejs nasm yasm zip m4" + deps2 = "python3-zstandard python-zstandard python-devel python3-devel gtk3-devel llvm gtk2-devel dbus-glib-devel libXt-devel pulseaudio-libs-devel" + exec("dnf -y install {} {}".format(deps1,deps2)) + +def execute_deps_pkg(): + deps = "wget gmake m4 python3 py37-sqlite3 pkgconf llvm node nasm zip unzip yasm" + exec("pkg install {}".format(deps)) + + +def execute_rustup(): + # rust needs special love: https://www.atechtown.com/install-rust-language-on-debian-10/ + exec("curl https://sh.rustup.rs -sSf | sh") + exec("cargo install cbindgen") + +def execute_mach_env(): + enter_srcdir() + exec("bash ./mach create-mach-environment") + leave_srcdir() + + + + + # Targets: def execute_fetch(): - print("[debug] doing target -> fetch") + if options.src == 'release': + exec("rm -f firefox-{}.source.tar.xz".format(pkgver)) + exec("wget -q https://archive.mozilla.org/pub/firefox/releases/{}/source/firefox-{}.source.tar.xz".format(pkgver, pkgver)) + elif options.src == 'nightly': + exec("rm -f bootstrap.py") + exec("rm -rf mozilla-unified") + exec("wget -q https://hg.mozilla.org/mozilla-central/raw-file/default/python/mozboot/bin/bootstrap.py") + exec("python3 bootstrap.py --no-interactive --application-choice=browser") + elif options.src == 'tor-browser': + exec("rm -rf tor-browser") + exec("git clone --no-checkout --recursive https://git.torproject.org/tor-browser.git") + enter_srcdir() + exec("git checkout tor-browser-89.0-10.5-1-build1") + exec("git submodule update --recursive") + exec("patch -p1 -i ../patches/tb-mozconfig-win10.patch") + leave_srcdir() + def execute_extract(): - print("[debug] doing target -> extract") + if options.src == 'release': + exec("rm -rf firefox-{}".format(pkgver)) + exec("tar xf firefox-{}.source.tar.xz".format(pkgver)) + def execute_lw_do_patches(): + if options.no_librewolf: + return print("[debug] doing target -> lw_do_patches") + def execute_build(): enter_srcdir() - cmd = "./mach build" - exec(cmd) + exec("bash ./mach build") leave_srcdir() + def execute_lw_post_build(): + if options.no_librewolf: + return print("[debug] doing target -> lw_post_build") + def execute_package(): enter_srcdir() - cmd = "./mach package" - exec(cmd) + exec("bash ./mach package") leave_srcdir() + def execute_lw_artifacts(): + if options.no_librewolf: + return print("[debug] doing target -> lw_artifacts") + + + + + + # Main targets: def execute_all(): execute_fetch() @@ -74,41 +179,35 @@ def execute_all(): execute_lw_post_build() execute_package() execute_lw_artifacts() + def execute_clean(): - print("[debug] doing target -> clean") + if options.src == 'release': + exec("rm -rf firefox-{}".format(pkgver)) + elif options.src == 'nightly': + exec("rm -rf mozilla-unified") + elif options.src == 'tor-browser': + exec("rm -rf tor-browser") + + exec("rm -rf librewolf firefox-{}.source.tar.xz mozconfig bootstrap.py".format(pkgver)) + exec("rm -f librewolf-{}.en-US.win64.zip librewolf-{}.en-US.win64-setup.exe".format(pkgver,pkgver)) + exec("rm -f tmp.nsi tmp-permissive.nsi tmp-strict.nsi".format()) + -# Utilities: -def execute_git_subs(): - print("[debug] doing target -> git_subs") - -def execute_git_init(): - print("[debug] doing target -> git_init") -def execute_git_reset(): - print("[debug] doing target -> git_reset") - -def execute_deps_deb(): - print("[debug] doing target -> deps_deb") -def execute_deps_rpm(): - print("[debug] doing target -> deps_rpm") -def execute_deps_pkg(): - print("[debug] doing target -> deps_pkg") - -def execute_rustup(): - print("[debug] doing target -> rustup") -def execute_mach_env(): - print("[debug] doing target -> mach_env") +# +# main commandline interface +# +if options.src == 'tor-browser': + options.no_librewolf = True - -# main commandline interpretation -if len(remainder)>0: +if len(remainder) > 0: if not options.src in ['release','nightly','tor-browser']: print("error: option --src invalid value") sys.exit(1) @@ -146,8 +245,6 @@ if len(remainder)>0: elif arg == 'git_init': execute_git_init() - elif arg == 'git_reset': - execute_git_reset() elif arg == 'deps_deb': execute_deps_deb() @@ -164,6 +261,7 @@ if len(remainder)>0: else: print("error: unknown command on command line: ", arg) sys.exit(1) + beep() else: # Print help message print("""# Use: @@ -172,7 +270,9 @@ else: # Options: - -x,--cross - build windows setup.exe from linux + -n,--no-execute - print commands, don't execute them + -l,--no-librewolf - skip LibreWolf specific stages. + -x,--cross - crosscompile from linux, implies -t win -s,--src - release,nightly,tor-browser (default=release) -t,--distro - deb,rpm,win (default=win) @@ -180,7 +280,7 @@ else: # Targets: all - all steps from fetch to producing setup.exe - clean - clean everything, includeing extracted/fetched sources + clean - clean everything, including extracted/fetched sources fetch - wget or hg clone or git pull extract - nop if not wget @@ -193,9 +293,7 @@ else: # Utilities: git_subs - git update submodules - - git_init - put the source folder in a .git reposity - git_reset - reset the source folder from the .git repo + git_init - put the source folder in a .git repository deps_deb - install dependencies with apt deps_rpm - install dependencies with dnf From 0bd27fb698033243378dab4913ee99224f6f6783 Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Thu, 10 Jun 2021 00:14:09 +0200 Subject: [PATCH 06/11] Added pybuild_lw.py --- .gitignore | 1 + pybuild.py | 51 +++------------------------------------------------ pybuild_lw.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 48 deletions(-) create mode 100644 pybuild_lw.py diff --git a/.gitignore b/.gitignore index 586bb29..a8af971 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ /tmp.nsi /tmp-permissive.nsi /tmp-strict.nsi +/__pycache__ \ No newline at end of file diff --git a/pybuild.py b/pybuild.py index dd678d1..070237d 100644 --- a/pybuild.py +++ b/pybuild.py @@ -10,6 +10,8 @@ import optparse import sys import os +import pybuild_lw + parser = optparse.OptionParser() parser.add_option('-x', '--cross', dest='cross_compile', default=False, action="store_true") @@ -19,22 +21,12 @@ parser.add_option('-s', '--src', dest='src', default='releas parser.add_option('-t', '--distro', dest='distro', default='win') options, remainder = parser.parse_args() - -#print("[debug] ----------") -#print("[debug] --cross = ", options.cross_compile) -#print("[debug] --no-execute = ", options.no_execute) -#print("[debug] --src = ", options.src) -#print("[debug] --distro = ", options.distro) -#print("[debug] ----------") - - def beep(): print('\a') - def enter_srcdir(): dir = "firefox-{}".format(pkgver) if options.src == 'nightly': @@ -264,44 +256,7 @@ if len(remainder) > 0: beep() else: # Print help message - print("""# Use: - - pybuild [] clean | all | - -# Options: - - -n,--no-execute - print commands, don't execute them - -l,--no-librewolf - skip LibreWolf specific stages. - -x,--cross - crosscompile from linux, implies -t win - -s,--src - release,nightly,tor-browser - (default=release) - -t,--distro - deb,rpm,win (default=win) - -# Targets: - - all - all steps from fetch to producing setup.exe - clean - clean everything, including extracted/fetched sources - - fetch - wget or hg clone or git pull - extract - nop if not wget - lw_do_patches - [librewolf] patch the source - build - build the browser - lw_post_build - [librewolf] insert our settings - package - package the browser into zip/apk - lw_artifacts - [librewolf] build setup.exe - -# Utilities: - - git_subs - git update submodules - git_init - put the source folder in a .git repository - - deps_deb - install dependencies with apt - deps_rpm - install dependencies with dnf - deps_pkg - install dependencies on freebsd - - rustup - update rust - mach_env - create mach environment -""") + print(pybuild_lw.help_message) diff --git a/pybuild_lw.py b/pybuild_lw.py new file mode 100644 index 0000000..b1e4be0 --- /dev/null +++ b/pybuild_lw.py @@ -0,0 +1,49 @@ + + + + + +# +# Large multiline strings +# + + + +help_message = """# Use: + + pybuild [] clean | all | | + +# Options: + + -n,--no-execute - print commands, don't execute them + -l,--no-librewolf - skip LibreWolf specific stages. + -x,--cross - crosscompile from linux, implies -t win + -s,--src - release,nightly,tor-browser + (default=release) + -t,--distro - deb,rpm,win (default=win) + +# Targets: + + all - all steps from fetch to producing setup.exe + clean - clean everything, including extracted/fetched sources + + fetch - wget or hg clone or git pull + extract - nop if not wget + lw_do_patches - [librewolf] patch the source + build - build the browser + lw_post_build - [librewolf] insert our settings + package - package the browser into zip/apk + lw_artifacts - [librewolf] build setup.exe + +# Utilities: + + git_subs - git update submodules + git_init - put the source folder in a .git repository + + deps_deb - install dependencies with apt + deps_rpm - install dependencies with dnf + deps_pkg - install dependencies on freebsd + + rustup - update rust + mach_env - create mach environment +""" From f2fa4ce26bd78a847c1b186d7337c4edf8aa1563 Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Thu, 10 Jun 2021 11:54:11 +0200 Subject: [PATCH 07/11] updates --- pybuild.py | 209 ++++++++++++++++++++++++++++++++------------------ pybuild_lw.py | 47 ------------ 2 files changed, 136 insertions(+), 120 deletions(-) diff --git a/pybuild.py b/pybuild.py index 070237d..a295197 100644 --- a/pybuild.py +++ b/pybuild.py @@ -53,8 +53,12 @@ def exec(cmd): - +# # Utilities: +# + + + def execute_git_subs(): exec("git submodule update --recursive") exec("git submodule foreach git pull origin master") @@ -102,10 +106,11 @@ def execute_mach_env(): - - - +# # Targets: +# + + def execute_fetch(): if options.src == 'release': exec("rm -f firefox-{}.source.tar.xz".format(pkgver)) @@ -129,26 +134,31 @@ def execute_extract(): exec("rm -rf firefox-{}".format(pkgver)) exec("tar xf firefox-{}.source.tar.xz".format(pkgver)) -def execute_lw_do_patches(): - if options.no_librewolf: - return - print("[debug] doing target -> lw_do_patches") - def execute_build(): enter_srcdir() exec("bash ./mach build") leave_srcdir() -def execute_lw_post_build(): - if options.no_librewolf: - return - print("[debug] doing target -> lw_post_build") - def execute_package(): enter_srcdir() exec("bash ./mach package") leave_srcdir() + +# +# LibreWolf specific: +# + +def execute_lw_do_patches(): + if options.no_librewolf: + return + print("[debug] doing target -> lw_do_patches") + +def execute_lw_post_build(): + if options.no_librewolf: + return + print("[debug] doing target -> lw_post_build") + def execute_lw_artifacts(): if options.no_librewolf: return @@ -162,7 +172,11 @@ def execute_lw_artifacts(): +# # Main targets: +# + + def execute_all(): execute_fetch() execute_extract() @@ -196,67 +210,116 @@ def execute_clean(): # main commandline interface # -if options.src == 'tor-browser': - options.no_librewolf = True +def main(): + if options.src == 'tor-browser': + options.no_librewolf = True -if len(remainder) > 0: - if not options.src in ['release','nightly','tor-browser']: - print("error: option --src invalid value") - sys.exit(1) - if not options.distro in ['deb','rpm', 'win']: - print("error: option --distro invalid value") - sys.exit(1) - - for arg in remainder: - if arg == 'all': - execute_all() - elif arg == 'clean': - execute_clean() - - # Targets: - - elif arg == 'fetch': - execute_fetch() - elif arg == 'extract': - execute_extract() - elif arg == 'lw_do_patches': - execute_lw_do_patches() - elif arg == 'build': - execute_build() - elif arg == 'lw_post_build': - execute_lw_post_build() - elif arg == 'package': - execute_package() - elif arg == 'lw_artifacts': - execute_lw_artifacts() - - # Utilities - - elif arg == 'git_subs': - execute_git_subs() - - elif arg == 'git_init': - execute_git_init() - - elif arg == 'deps_deb': - execute_deps_deb() - elif arg == 'deps_rpm': - execute_deps_rpm() - elif arg == 'deps_pkg': - execute_deps_pkg() - - elif arg == 'rustup': - execute_rustup() - elif arg == 'mach_env': - execute_mach_env() - - else: - print("error: unknown command on command line: ", arg) + if len(remainder) > 0: + if not options.src in ['release','nightly','tor-browser']: + print("error: option --src invalid value") sys.exit(1) - beep() -else: - # Print help message - print(pybuild_lw.help_message) + if not options.distro in ['deb','rpm', 'win']: + print("error: option --distro invalid value") + sys.exit(1) + + for arg in remainder: + if arg == 'all': + execute_all() + elif arg == 'clean': + execute_clean() + + # Targets: + + elif arg == 'fetch': + execute_fetch() + elif arg == 'extract': + execute_extract() + elif arg == 'lw_do_patches': + execute_lw_do_patches() + elif arg == 'build': + execute_build() + elif arg == 'lw_post_build': + execute_lw_post_build() + elif arg == 'package': + execute_package() + elif arg == 'lw_artifacts': + execute_lw_artifacts() + + # Utilities + + elif arg == 'git_subs': + execute_git_subs() + + elif arg == 'git_init': + execute_git_init() + + elif arg == 'deps_deb': + execute_deps_deb() + elif arg == 'deps_rpm': + execute_deps_rpm() + elif arg == 'deps_pkg': + execute_deps_pkg() + + elif arg == 'rustup': + execute_rustup() + elif arg == 'mach_env': + execute_mach_env() + + else: + print("error: unknown command on command line: ", arg) + sys.exit(1) + beep() + else: + # Print help message + print(help_message) + +# +# Large multiline strings +# + + + +help_message = """# Use: + + pybuild [] clean | all | | + +# Options: + + -n,--no-execute - print commands, don't execute them + -l,--no-librewolf - skip LibreWolf specific stages. + -x,--cross - crosscompile from linux, implies -t win + -s,--src - release,nightly,tor-browser + (default=release) + -t,--distro - deb,rpm,win (default=win) + +# Targets: + + all - all steps from fetch to producing setup.exe + clean - clean everything, including extracted/fetched sources + + fetch - wget or hg clone or git pull + extract - nop if not wget + lw_do_patches - [librewolf] patch the source + build - build the browser + lw_post_build - [librewolf] insert our settings + package - package the browser into zip/apk + lw_artifacts - [librewolf] build setup.exe + +# Utilities: + + git_subs - git update submodules + git_init - put the source folder in a .git repository + + deps_deb - install dependencies with apt + deps_rpm - install dependencies with dnf + deps_pkg - install dependencies on freebsd + + rustup - update rust + mach_env - create mach environment +""" + + +main() diff --git a/pybuild_lw.py b/pybuild_lw.py index b1e4be0..139597f 100644 --- a/pybuild_lw.py +++ b/pybuild_lw.py @@ -1,49 +1,2 @@ - - - -# -# Large multiline strings -# - - - -help_message = """# Use: - - pybuild [] clean | all | | - -# Options: - - -n,--no-execute - print commands, don't execute them - -l,--no-librewolf - skip LibreWolf specific stages. - -x,--cross - crosscompile from linux, implies -t win - -s,--src - release,nightly,tor-browser - (default=release) - -t,--distro - deb,rpm,win (default=win) - -# Targets: - - all - all steps from fetch to producing setup.exe - clean - clean everything, including extracted/fetched sources - - fetch - wget or hg clone or git pull - extract - nop if not wget - lw_do_patches - [librewolf] patch the source - build - build the browser - lw_post_build - [librewolf] insert our settings - package - package the browser into zip/apk - lw_artifacts - [librewolf] build setup.exe - -# Utilities: - - git_subs - git update submodules - git_init - put the source folder in a .git repository - - deps_deb - install dependencies with apt - deps_rpm - install dependencies with dnf - deps_pkg - install dependencies on freebsd - - rustup - update rust - mach_env - create mach environment -""" From f80ce7a5839f13a3c5d6d2c97e5b145bc144f3fb Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Thu, 10 Jun 2021 13:49:15 +0200 Subject: [PATCH 08/11] updates --- pybuild.py | 28 ++++++++++++++++++++++++---- pybuild_lw.py | 2 -- 2 files changed, 24 insertions(+), 6 deletions(-) delete mode 100644 pybuild_lw.py diff --git a/pybuild.py b/pybuild.py index a295197..3aa1446 100644 --- a/pybuild.py +++ b/pybuild.py @@ -10,8 +10,6 @@ import optparse import sys import os -import pybuild_lw - parser = optparse.OptionParser() parser.add_option('-x', '--cross', dest='cross_compile', default=False, action="store_true") @@ -35,7 +33,11 @@ def enter_srcdir(): dir = 'tor-browser' print("cd {}".format(dir)) if not options.no_execute: - os.chdir(dir) + try: + os.chdir(dir) + except: + print("fatal error: can't change to '{}' folder.".format(dir)) + sys.exit(1) def leave_srcdir(): print("cd ..") @@ -101,7 +103,22 @@ def execute_mach_env(): exec("bash ./mach create-mach-environment") leave_srcdir() - + +def execute_reset(): + if options.src == 'release': + print("error: cannot reset -release source as it's not under version control") + beep() + sys.exit(1) + elif options.src == 'nightly': + enter_srcdir() + exec("hg up -C") + exec("hg purge") + exec("hg pull -u") + leave_srcdir() + elif options.src == 'tor-browser': + enter_srcdir() + exec("git reset --hard") + leave_srcidr() @@ -264,6 +281,8 @@ def main(): execute_rustup() elif arg == 'mach_env': execute_mach_env() + elif arg == 'reset': + execute_reset() else: print("error: unknown command on command line: ", arg) @@ -311,6 +330,7 @@ help_message = """# Use: # Utilities: git_subs - git update submodules + reset - use git/mercurial to revert changes to a clean state git_init - put the source folder in a .git repository deps_deb - install dependencies with apt diff --git a/pybuild_lw.py b/pybuild_lw.py deleted file mode 100644 index 139597f..0000000 --- a/pybuild_lw.py +++ /dev/null @@ -1,2 +0,0 @@ - - From a7326a06803dfd64b21e986cdd937fcf934fbb26 Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Thu, 10 Jun 2021 15:27:33 +0200 Subject: [PATCH 09/11] updates --- pybuild.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pybuild.py b/pybuild.py index 3aa1446..58c7550 100644 --- a/pybuild.py +++ b/pybuild.py @@ -23,7 +23,7 @@ options, remainder = parser.parse_args() def beep(): - print('\a') + print('\a', end='') def enter_srcdir(): dir = "firefox-{}".format(pkgver) @@ -45,7 +45,7 @@ def leave_srcdir(): os.chdir("..") def exec(cmd): - print("{}".format(cmd)) + print(cmd) if not options.no_execute: retval = os.system(cmd) if retval != 0: @@ -73,7 +73,7 @@ def execute_git_init(): exec("git config core.safecrlf false") exec("git config commit.gpgsign false") exec("git add -f * .[a-z]*") - exec("git commit -am 'Initial commit'") + exec("git commit -am initial") leave_srcdir() @@ -106,9 +106,13 @@ def execute_mach_env(): def execute_reset(): if options.src == 'release': - print("error: cannot reset -release source as it's not under version control") - beep() - sys.exit(1) + path = "firefox-{}/.git/index".format(pkgver) + if not os.path.isfile(path): + print("fatal error: cannot reset '--src release' sources as it's not under version control.") + sys.exit(1) + enter_srcdir() + exec("git reset --hard") + leave_srcdir() elif options.src == 'nightly': enter_srcdir() exec("hg up -C") @@ -287,7 +291,7 @@ def main(): else: print("error: unknown command on command line: ", arg) sys.exit(1) - beep() + beep() else: # Print help message print(help_message) @@ -320,7 +324,7 @@ help_message = """# Use: clean - clean everything, including extracted/fetched sources fetch - wget or hg clone or git pull - extract - nop if not wget + extract - when using wget, extract the archive. lw_do_patches - [librewolf] patch the source build - build the browser lw_post_build - [librewolf] insert our settings From c9c01b9fdc522dbfb867b7b07970ee12b6c5f6b3 Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Fri, 11 Jun 2021 21:32:26 +0200 Subject: [PATCH 10/11] updates --- patches/package-manifest.patch | 14 ++++ pybuild.py | 120 +++++++++++++++++++++++++++++++-- 2 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 patches/package-manifest.patch diff --git a/patches/package-manifest.patch b/patches/package-manifest.patch new file mode 100644 index 0000000..ddc3a13 --- /dev/null +++ b/patches/package-manifest.patch @@ -0,0 +1,14 @@ +diff --git a/browser/installer/package-manifest.in b/browser/installer/package-manifest.in +index 2bdafc86b3..789bece8e6 100644 +--- a/browser/installer/package-manifest.in ++++ b/browser/installer/package-manifest.in +@@ -302,6 +302,9 @@ + ; Technically this is an app pref file, but we are keeping it in the original + ; gre location for now. + @RESPATH@/defaults/pref/channel-prefs.js ++@RESPATH@/defaults/pref/local-settings.js ++@RESPATH@/distribution/policies.json ++@RESPATH@/librewolf.cfg + + ; Background tasks-specific preferences. These are in the GRE + ; location since they apply to all tasks at this time. diff --git a/pybuild.py b/pybuild.py index 58c7550..7c8e147 100644 --- a/pybuild.py +++ b/pybuild.py @@ -9,6 +9,7 @@ pkgver='89.0' import optparse import sys import os +import glob parser = optparse.OptionParser() @@ -23,7 +24,8 @@ options, remainder = parser.parse_args() def beep(): - print('\a', end='') + if not options.no_execute: + print('\a', end='') def enter_srcdir(): dir = "firefox-{}".format(pkgver) @@ -49,10 +51,19 @@ def exec(cmd): if not options.no_execute: retval = os.system(cmd) if retval != 0: + print("fatal error: command '{}' failed".format(cmd)) beep() sys.exit(1) - +def patch(patchfile): + cmd = "patch -p1 -i {}".format(patchfile) + print(cmd) + if not options.no_execute: + retval = os.system(cmd) + if retval != 0: + print("fatal error: patch '{}' failed".format(patchfile)) + beep() + sys.exit(1) # @@ -147,7 +158,7 @@ def execute_fetch(): enter_srcdir() exec("git checkout tor-browser-89.0-10.5-1-build1") exec("git submodule update --recursive") - exec("patch -p1 -i ../patches/tb-mozconfig-win10.patch") + patch("../patches/tb-mozconfig-win10.patch") leave_srcdir() def execute_extract(): @@ -166,19 +177,71 @@ def execute_package(): leave_srcdir() + + + + + + + # # LibreWolf specific: # +def create_mozconfig(contents): + if not options.no_execute: + f = open('mozconfig', 'w') + f.write(contents) + f.close() + def execute_lw_do_patches(): if options.no_librewolf: return - print("[debug] doing target -> lw_do_patches") + if not options.src in ['release','nightly']: + return + + enter_srcdir() + # create the right mozconfig file.. + create_mozconfig(mozconfig_release) + + # copy branding files.. + exec("cp -vr ../common/source_files/* .") + exec("cp -v ../files/configure.sh browser/branding/librewolf") + + # patches.. + patch("../common/patches/context-menu.patch") + patch("../common/patches/remove_addons.patch") + patch("../common/patches/megabar.patch") + patch("../common/patches/mozilla-vpn-ad.patch") + + # sed patches.. + patch("../common/patches/sed-patches/allow-searchengines-non-esr.patch") + patch("../common/patches/sed-patches/disable-pocket.patch") + patch("../common/patches/sed-patches/remove-internal-plugin-certs.patch") + patch("../common/patches/sed-patches/stop-undesired-requests.patch") + + # local windows patches + patch("../patches/browser-confvars.patch") # not sure about this one yet! + patch("../patches/package-manifest.patch") # let ./mach package pick up our added files + leave_srcdir() def execute_lw_post_build(): if options.no_librewolf: return - print("[debug] doing target -> lw_post_build") + enter_srcdir() + + pattern = "obj-*" + retval = glob.glob(pattern) + if len(retval) != 1: + printf("fatal error: in execute_lw_post_build(): cannot glob build output folder '{}'".format(pattern)) + sys.exit(1) + + os.makedirs("{}/dist/bin/defaults/pref".format(retval[0]), exist_ok=True) + os.makedirs("{}/dist/bin/distribution".format(retval[0]), exist_ok=True) + exec("cp -v ../settings/defaults/pref/local-settings.js {}/dist/bin/defaults/pref/".format(retval[0])) + exec("cp -v ../settings/distribution/policies.json {}/dist/bin/distribution/".format(retval[0])) + exec("cp -v ../settings/librewolf.cfg {}/dist/bin/".format(retval[0])) + leave_srcdir() def execute_lw_artifacts(): if options.no_librewolf: @@ -215,7 +278,7 @@ def execute_clean(): elif options.src == 'tor-browser': exec("rm -rf tor-browser") - exec("rm -rf librewolf firefox-{}.source.tar.xz mozconfig bootstrap.py".format(pkgver)) + exec("rm -rf librewolf firefox-{}.source.tar.xz bootstrap.py".format(pkgver)) exec("rm -f librewolf-{}.en-US.win64.zip librewolf-{}.en-US.win64-setup.exe".format(pkgver,pkgver)) exec("rm -f tmp.nsi tmp-permissive.nsi tmp-strict.nsi".format()) @@ -345,5 +408,50 @@ help_message = """# Use: mach_env - create mach environment """ +# +# mozconfig files: +# + +mozconfig_release = """ +ac_add_options --enable-application=browser + +# This supposedly speeds up compilation (We test through dogfooding anyway) +ac_add_options --disable-tests +ac_add_options --disable-debug + +ac_add_options --enable-release +ac_add_options --enable-hardening +ac_add_options --enable-rust-simd +ac_add_options --enable-optimize + + +# Branding +ac_add_options --enable-update-channel=release +# theming bugs: ac_add_options --with-app-name=librewolf +# theming bugs: ac_add_options --with-app-basename=LibreWolf +ac_add_options --with-branding=browser/branding/librewolf +ac_add_options --with-distribution-id=io.gitlab.librewolf-community +ac_add_options --with-unsigned-addon-scopes=app,system +ac_add_options --allow-addon-sideload +#export MOZ_REQUIRE_SIGNING=0 + +# Features +ac_add_options --disable-crashreporter +ac_add_options --disable-updater + +# Disables crash reporting, telemetry and other data gathering tools +mk_add_options MOZ_CRASHREPORTER=0 +mk_add_options MOZ_DATA_REPORTING=0 +mk_add_options MOZ_SERVICES_HEALTHREPORT=0 +mk_add_options MOZ_TELEMETRY_REPORTING=0 + +# testing.. +# MOZ_APP_NAME=librewolf +# This gives the same theming issue as --with-app-name=librewolf +""" + + + + main() From 7c8be01b2998d813185efa875ae212ad6af64541 Mon Sep 17 00:00:00 2001 From: Bert van der Weerd Date: Sat, 12 Jun 2021 11:01:53 +0200 Subject: [PATCH 11/11] branch work complete --- pybuild.py | 65 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/pybuild.py b/pybuild.py index 7c8e147..7314864 100644 --- a/pybuild.py +++ b/pybuild.py @@ -224,38 +224,67 @@ def execute_lw_do_patches(): patch("../patches/browser-confvars.patch") # not sure about this one yet! patch("../patches/package-manifest.patch") # let ./mach package pick up our added files leave_srcdir() - -def execute_lw_post_build(): - if options.no_librewolf: - return - enter_srcdir() - + + +def get_objdir(): pattern = "obj-*" retval = glob.glob(pattern) if len(retval) != 1: printf("fatal error: in execute_lw_post_build(): cannot glob build output folder '{}'".format(pattern)) sys.exit(1) + return retval[0] + +def execute_lw_post_build(): + if options.no_librewolf: + return + enter_srcdir() + dirname = get_objdir() - os.makedirs("{}/dist/bin/defaults/pref".format(retval[0]), exist_ok=True) - os.makedirs("{}/dist/bin/distribution".format(retval[0]), exist_ok=True) - exec("cp -v ../settings/defaults/pref/local-settings.js {}/dist/bin/defaults/pref/".format(retval[0])) - exec("cp -v ../settings/distribution/policies.json {}/dist/bin/distribution/".format(retval[0])) - exec("cp -v ../settings/librewolf.cfg {}/dist/bin/".format(retval[0])) + os.makedirs("{}/dist/bin/defaults/pref".format(dirname), exist_ok=True) + os.makedirs("{}/dist/bin/distribution".format(dirname), exist_ok=True) + exec("cp -v ../settings/defaults/pref/local-settings.js {}/dist/bin/defaults/pref/".format(dirname)) + exec("cp -v ../settings/distribution/policies.json {}/dist/bin/distribution/".format(dirname)) + exec("cp -v ../settings/librewolf.cfg {}/dist/bin/".format(dirname)) leave_srcdir() def execute_lw_artifacts(): if options.no_librewolf: return - print("[debug] doing target -> lw_artifacts") - - - + enter_srcdir() + if options.distro == 'win': + exe = ".exe" + ospkg = "win64" + dirname = "{}/dist/firefox".format(get_objdir()) + elif options.distro == 'deb': + exe = "" + ospkg = "deb" + dirname = "{}/dist/firefox".format(get_objdir()) + elif options.distro == 'rpm': + exe = "" + ospkg = "rpm" + dirname = "{}/dist/firefox".format(get_objdir()) - - + exec("rm -rf ../firefox ../librewolf") + exec("cp -rv {} ..".format(dirname)) + leave_srcdir() + exec("mv firefox librewolf") + exec("mv -v librewolf/firefox{} librewolf/librewolf{}".format(exe,exe)); + exec("rm -rf librewolf/maintainanceservice* librewolf/pingsender* librewolf/firefox.*.xml librewolf/precomplete librewolf/removed-files librewolf/uninstall") + exec("cp -v common/source_files/browser/branding/librewolf/firefox.ico librewolf/librewolf.ico") + + # create zip file + zipname = "librewolf-{}.en-US.{}.zip".format(pkgver,ospkg) + exec("rm -f {}".format(zipname)) + exec("zip -qr9 {} librewolf".format(zipname)) + + # create installer + if options.distro == 'win': + exec("rm -f librewolf-{}.en-US.win64-setup.exe tmp.nsi".format(pkgver)) + exec("sed \"s/pkg_version/{}/g\" < artifacts_win.nsi > tmp.nsi".format(pkgver)) + exec("makensis-3.01.exe -V1 tmp.nsi") # # Main targets: # @@ -280,7 +309,7 @@ def execute_clean(): exec("rm -rf librewolf firefox-{}.source.tar.xz bootstrap.py".format(pkgver)) exec("rm -f librewolf-{}.en-US.win64.zip librewolf-{}.en-US.win64-setup.exe".format(pkgver,pkgver)) - exec("rm -f tmp.nsi tmp-permissive.nsi tmp-strict.nsi".format()) + exec("rm -f tmp.nsi")