ci.go 32 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046
  1. // Copyright 2016 The go-ethereum Authors
  2. // This file is part of the go-ethereum library.
  3. //
  4. // The go-ethereum library is free software: you can redistribute it and/or modify
  5. // it under the terms of the GNU Lesser General Public License as published by
  6. // the Free Software Foundation, either version 3 of the License, or
  7. // (at your option) any later version.
  8. //
  9. // The go-ethereum library is distributed in the hope that it will be useful,
  10. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. // GNU Lesser General Public License for more details.
  13. //
  14. // You should have received a copy of the GNU Lesser General Public License
  15. // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
  16. // +build none
  17. /*
  18. The ci command is called from Continuous Integration scripts.
  19. Usage: go run build/ci.go <command> <command flags/arguments>
  20. Available commands are:
  21. install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables
  22. test [ -coverage ] [ packages... ] -- runs the tests
  23. lint -- runs certain pre-selected linters
  24. archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
  25. importkeys -- imports signing keys from env
  26. debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package
  27. nsis -- creates a Windows NSIS installer
  28. aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive
  29. xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework
  30. xgo [ -alltools ] [ options ] -- cross builds according to options
  31. purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore
  32. For all commands, -n prevents execution of external programs (dry run mode).
  33. */
  34. package main
  35. import (
  36. "bufio"
  37. "bytes"
  38. "encoding/base64"
  39. "flag"
  40. "fmt"
  41. "go/parser"
  42. "go/token"
  43. "io/ioutil"
  44. "log"
  45. "os"
  46. "os/exec"
  47. "path/filepath"
  48. "regexp"
  49. "runtime"
  50. "strings"
  51. "time"
  52. "github.com/ethereum/go-ethereum/internal/build"
  53. )
  54. var (
  55. // Files that end up in the geth*.zip archive.
  56. gethArchiveFiles = []string{
  57. "COPYING",
  58. executablePath("geth"),
  59. }
  60. // Files that end up in the geth-alltools*.zip archive.
  61. allToolsArchiveFiles = []string{
  62. "COPYING",
  63. executablePath("abigen"),
  64. executablePath("bootnode"),
  65. executablePath("evm"),
  66. executablePath("geth"),
  67. executablePath("puppeth"),
  68. executablePath("rlpdump"),
  69. executablePath("swarm"),
  70. executablePath("wnode"),
  71. }
  72. // A debian package is created for all executables listed here.
  73. debExecutables = []debExecutable{
  74. {
  75. Name: "abigen",
  76. Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
  77. },
  78. {
  79. Name: "bootnode",
  80. Description: "Ethereum bootnode.",
  81. },
  82. {
  83. Name: "evm",
  84. Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
  85. },
  86. {
  87. Name: "geth",
  88. Description: "Ethereum CLI client.",
  89. },
  90. {
  91. Name: "puppeth",
  92. Description: "Ethereum private network manager.",
  93. },
  94. {
  95. Name: "rlpdump",
  96. Description: "Developer utility tool that prints RLP structures.",
  97. },
  98. {
  99. Name: "swarm",
  100. Description: "Ethereum Swarm daemon and tools",
  101. },
  102. {
  103. Name: "wnode",
  104. Description: "Ethereum Whisper diagnostic tool",
  105. },
  106. }
  107. // Distros for which packages are created.
  108. // Note: vivid is unsupported because there is no golang-1.6 package for it.
  109. // Note: wily is unsupported because it was officially deprecated on lanchpad.
  110. // Note: yakkety is unsupported because it was officially deprecated on lanchpad.
  111. // Note: zesty is unsupported because it was officially deprecated on lanchpad.
  112. debDistros = []string{"trusty", "xenial", "artful", "bionic"}
  113. )
  114. var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
  115. func executablePath(name string) string {
  116. if runtime.GOOS == "windows" {
  117. name += ".exe"
  118. }
  119. return filepath.Join(GOBIN, name)
  120. }
  121. func main() {
  122. log.SetFlags(log.Lshortfile)
  123. if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) {
  124. log.Fatal("this script must be run from the root of the repository")
  125. }
  126. if len(os.Args) < 2 {
  127. log.Fatal("need subcommand as first argument")
  128. }
  129. switch os.Args[1] {
  130. case "install":
  131. doInstall(os.Args[2:])
  132. case "test":
  133. doTest(os.Args[2:])
  134. case "lint":
  135. doLint(os.Args[2:])
  136. case "archive":
  137. doArchive(os.Args[2:])
  138. case "debsrc":
  139. doDebianSource(os.Args[2:])
  140. case "nsis":
  141. doWindowsInstaller(os.Args[2:])
  142. case "aar":
  143. doAndroidArchive(os.Args[2:])
  144. case "xcode":
  145. doXCodeFramework(os.Args[2:])
  146. case "xgo":
  147. doXgo(os.Args[2:])
  148. case "purge":
  149. doPurge(os.Args[2:])
  150. default:
  151. log.Fatal("unknown command ", os.Args[1])
  152. }
  153. }
  154. // Compiling
  155. func doInstall(cmdline []string) {
  156. var (
  157. arch = flag.String("arch", "", "Architecture to cross build for")
  158. cc = flag.String("cc", "", "C compiler to cross build with")
  159. )
  160. flag.CommandLine.Parse(cmdline)
  161. env := build.Env()
  162. // Check Go version. People regularly open issues about compilation
  163. // failure with outdated Go. This should save them the trouble.
  164. if !strings.Contains(runtime.Version(), "devel") {
  165. // Figure out the minor version number since we can't textually compare (1.10 < 1.9)
  166. var minor int
  167. fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
  168. if minor < 9 {
  169. log.Println("You have Go version", runtime.Version())
  170. log.Println("go-ethereum requires at least Go version 1.9 and cannot")
  171. log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
  172. os.Exit(1)
  173. }
  174. }
  175. // Compile packages given as arguments, or everything if there are no arguments.
  176. packages := []string{"./..."}
  177. if flag.NArg() > 0 {
  178. packages = flag.Args()
  179. }
  180. packages = build.ExpandPackagesNoVendor(packages)
  181. if *arch == "" || *arch == runtime.GOARCH {
  182. goinstall := goTool("install", buildFlags(env)...)
  183. goinstall.Args = append(goinstall.Args, "-v")
  184. goinstall.Args = append(goinstall.Args, packages...)
  185. build.MustRun(goinstall)
  186. return
  187. }
  188. // If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds
  189. if *arch == "arm" {
  190. os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm"))
  191. for _, path := range filepath.SplitList(build.GOPATH()) {
  192. os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm"))
  193. }
  194. }
  195. // Seems we are cross compiling, work around forbidden GOBIN
  196. goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...)
  197. goinstall.Args = append(goinstall.Args, "-v")
  198. goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...)
  199. goinstall.Args = append(goinstall.Args, packages...)
  200. build.MustRun(goinstall)
  201. if cmds, err := ioutil.ReadDir("cmd"); err == nil {
  202. for _, cmd := range cmds {
  203. pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly)
  204. if err != nil {
  205. log.Fatal(err)
  206. }
  207. for name := range pkgs {
  208. if name == "main" {
  209. gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...)
  210. gobuild.Args = append(gobuild.Args, "-v")
  211. gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...)
  212. gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name()))
  213. build.MustRun(gobuild)
  214. break
  215. }
  216. }
  217. }
  218. }
  219. }
  220. func buildFlags(env build.Environment) (flags []string) {
  221. var ld []string
  222. if env.Commit != "" {
  223. ld = append(ld, "-X", "main.gitCommit="+env.Commit)
  224. }
  225. if runtime.GOOS == "darwin" {
  226. ld = append(ld, "-s")
  227. }
  228. if len(ld) > 0 {
  229. flags = append(flags, "-ldflags", strings.Join(ld, " "))
  230. }
  231. return flags
  232. }
  233. func goTool(subcmd string, args ...string) *exec.Cmd {
  234. return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...)
  235. }
  236. func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd {
  237. cmd := build.GoTool(subcmd, args...)
  238. cmd.Env = []string{"GOPATH=" + build.GOPATH()}
  239. if arch == "" || arch == runtime.GOARCH {
  240. cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
  241. } else {
  242. cmd.Env = append(cmd.Env, "CGO_ENABLED=1")
  243. cmd.Env = append(cmd.Env, "GOARCH="+arch)
  244. }
  245. if cc != "" {
  246. cmd.Env = append(cmd.Env, "CC="+cc)
  247. }
  248. for _, e := range os.Environ() {
  249. if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
  250. continue
  251. }
  252. cmd.Env = append(cmd.Env, e)
  253. }
  254. return cmd
  255. }
  256. // Running The Tests
  257. //
  258. // "tests" also includes static analysis tools such as vet.
  259. func doTest(cmdline []string) {
  260. var (
  261. coverage = flag.Bool("coverage", false, "Whether to record code coverage")
  262. )
  263. flag.CommandLine.Parse(cmdline)
  264. env := build.Env()
  265. packages := []string{"./..."}
  266. if len(flag.CommandLine.Args()) > 0 {
  267. packages = flag.CommandLine.Args()
  268. }
  269. packages = build.ExpandPackagesNoVendor(packages)
  270. // Run analysis tools before the tests.
  271. build.MustRun(goTool("vet", packages...))
  272. // Run the actual tests.
  273. gotest := goTool("test", buildFlags(env)...)
  274. // Test a single package at a time. CI builders are slow
  275. // and some tests run into timeouts under load.
  276. gotest.Args = append(gotest.Args, "-p", "1")
  277. if *coverage {
  278. gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
  279. }
  280. gotest.Args = append(gotest.Args, packages...)
  281. build.MustRun(gotest)
  282. }
  283. // runs gometalinter on requested packages
  284. func doLint(cmdline []string) {
  285. flag.CommandLine.Parse(cmdline)
  286. packages := []string{"./..."}
  287. if len(flag.CommandLine.Args()) > 0 {
  288. packages = flag.CommandLine.Args()
  289. }
  290. // Get metalinter and install all supported linters
  291. build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2"))
  292. build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install")
  293. // Run fast linters batched together
  294. configs := []string{
  295. "--vendor",
  296. "--tests",
  297. "--disable-all",
  298. "--enable=goimports",
  299. "--enable=varcheck",
  300. "--enable=vet",
  301. "--enable=gofmt",
  302. "--enable=misspell",
  303. "--enable=goconst",
  304. "--min-occurrences=6", // for goconst
  305. }
  306. build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
  307. // Run slow linters one by one
  308. for _, linter := range []string{"unconvert", "gosimple"} {
  309. configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter}
  310. build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
  311. }
  312. }
  313. // Release Packaging
  314. func doArchive(cmdline []string) {
  315. var (
  316. arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging")
  317. atype = flag.String("type", "zip", "Type of archive to write (zip|tar)")
  318. signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`)
  319. upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
  320. ext string
  321. )
  322. flag.CommandLine.Parse(cmdline)
  323. switch *atype {
  324. case "zip":
  325. ext = ".zip"
  326. case "tar":
  327. ext = ".tar.gz"
  328. default:
  329. log.Fatal("unknown archive type: ", atype)
  330. }
  331. var (
  332. env = build.Env()
  333. base = archiveBasename(*arch, env)
  334. geth = "geth-" + base + ext
  335. alltools = "geth-alltools-" + base + ext
  336. )
  337. maybeSkipArchive(env)
  338. if err := build.WriteArchive(geth, gethArchiveFiles); err != nil {
  339. log.Fatal(err)
  340. }
  341. if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil {
  342. log.Fatal(err)
  343. }
  344. for _, archive := range []string{geth, alltools} {
  345. if err := archiveUpload(archive, *upload, *signer); err != nil {
  346. log.Fatal(err)
  347. }
  348. }
  349. }
  350. func archiveBasename(arch string, env build.Environment) string {
  351. platform := runtime.GOOS + "-" + arch
  352. if arch == "arm" {
  353. platform += os.Getenv("GOARM")
  354. }
  355. if arch == "android" {
  356. platform = "android-all"
  357. }
  358. if arch == "ios" {
  359. platform = "ios-all"
  360. }
  361. return platform + "-" + archiveVersion(env)
  362. }
  363. func archiveVersion(env build.Environment) string {
  364. version := build.VERSION()
  365. if isUnstableBuild(env) {
  366. version += "-unstable"
  367. }
  368. if env.Commit != "" {
  369. version += "-" + env.Commit[:8]
  370. }
  371. return version
  372. }
  373. func archiveUpload(archive string, blobstore string, signer string) error {
  374. // If signing was requested, generate the signature files
  375. if signer != "" {
  376. pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
  377. if err != nil {
  378. return fmt.Errorf("invalid base64 %s", signer)
  379. }
  380. if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
  381. return err
  382. }
  383. }
  384. // If uploading to Azure was requested, push the archive possibly with its signature
  385. if blobstore != "" {
  386. auth := build.AzureBlobstoreConfig{
  387. Account: strings.Split(blobstore, "/")[0],
  388. Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"),
  389. Container: strings.SplitN(blobstore, "/", 2)[1],
  390. }
  391. if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil {
  392. return err
  393. }
  394. if signer != "" {
  395. if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil {
  396. return err
  397. }
  398. }
  399. }
  400. return nil
  401. }
  402. // skips archiving for some build configurations.
  403. func maybeSkipArchive(env build.Environment) {
  404. if env.IsPullRequest {
  405. log.Printf("skipping because this is a PR build")
  406. os.Exit(0)
  407. }
  408. if env.IsCronJob {
  409. log.Printf("skipping because this is a cron job")
  410. os.Exit(0)
  411. }
  412. if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
  413. log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
  414. os.Exit(0)
  415. }
  416. }
  417. // Debian Packaging
  418. func doDebianSource(cmdline []string) {
  419. var (
  420. signer = flag.String("signer", "", `Signing key name, also used as package author`)
  421. upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
  422. workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
  423. now = time.Now()
  424. )
  425. flag.CommandLine.Parse(cmdline)
  426. *workdir = makeWorkdir(*workdir)
  427. env := build.Env()
  428. maybeSkipArchive(env)
  429. // Import the signing key.
  430. if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
  431. key, err := base64.StdEncoding.DecodeString(b64key)
  432. if err != nil {
  433. log.Fatal("invalid base64 PPA_SIGNING_KEY")
  434. }
  435. gpg := exec.Command("gpg", "--import")
  436. gpg.Stdin = bytes.NewReader(key)
  437. build.MustRun(gpg)
  438. }
  439. // Create the packages.
  440. for _, distro := range debDistros {
  441. meta := newDebMetadata(distro, *signer, env, now)
  442. pkgdir := stageDebianSource(*workdir, meta)
  443. debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
  444. debuild.Dir = pkgdir
  445. build.MustRun(debuild)
  446. changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
  447. changes = filepath.Join(*workdir, changes)
  448. if *signer != "" {
  449. build.MustRunCommand("debsign", changes)
  450. }
  451. if *upload != "" {
  452. build.MustRunCommand("dput", *upload, changes)
  453. }
  454. }
  455. }
  456. func makeWorkdir(wdflag string) string {
  457. var err error
  458. if wdflag != "" {
  459. err = os.MkdirAll(wdflag, 0744)
  460. } else {
  461. wdflag, err = ioutil.TempDir("", "geth-build-")
  462. }
  463. if err != nil {
  464. log.Fatal(err)
  465. }
  466. return wdflag
  467. }
  468. func isUnstableBuild(env build.Environment) bool {
  469. if env.Tag != "" {
  470. return false
  471. }
  472. return true
  473. }
  474. type debMetadata struct {
  475. Env build.Environment
  476. // go-ethereum version being built. Note that this
  477. // is not the debian package version. The package version
  478. // is constructed by VersionString.
  479. Version string
  480. Author string // "name <email>", also selects signing key
  481. Distro, Time string
  482. Executables []debExecutable
  483. }
  484. type debExecutable struct {
  485. Name, Description string
  486. }
  487. func newDebMetadata(distro, author string, env build.Environment, t time.Time) debMetadata {
  488. if author == "" {
  489. // No signing key, use default author.
  490. author = "Ethereum Builds <fjl@ethereum.org>"
  491. }
  492. return debMetadata{
  493. Env: env,
  494. Author: author,
  495. Distro: distro,
  496. Version: build.VERSION(),
  497. Time: t.Format(time.RFC1123Z),
  498. Executables: debExecutables,
  499. }
  500. }
  501. // Name returns the name of the metapackage that depends
  502. // on all executable packages.
  503. func (meta debMetadata) Name() string {
  504. if isUnstableBuild(meta.Env) {
  505. return "ethereum-unstable"
  506. }
  507. return "ethereum"
  508. }
  509. // VersionString returns the debian version of the packages.
  510. func (meta debMetadata) VersionString() string {
  511. vsn := meta.Version
  512. if meta.Env.Buildnum != "" {
  513. vsn += "+build" + meta.Env.Buildnum
  514. }
  515. if meta.Distro != "" {
  516. vsn += "+" + meta.Distro
  517. }
  518. return vsn
  519. }
  520. // ExeList returns the list of all executable packages.
  521. func (meta debMetadata) ExeList() string {
  522. names := make([]string, len(meta.Executables))
  523. for i, e := range meta.Executables {
  524. names[i] = meta.ExeName(e)
  525. }
  526. return strings.Join(names, ", ")
  527. }
  528. // ExeName returns the package name of an executable package.
  529. func (meta debMetadata) ExeName(exe debExecutable) string {
  530. if isUnstableBuild(meta.Env) {
  531. return exe.Name + "-unstable"
  532. }
  533. return exe.Name
  534. }
  535. // ExeConflicts returns the content of the Conflicts field
  536. // for executable packages.
  537. func (meta debMetadata) ExeConflicts(exe debExecutable) string {
  538. if isUnstableBuild(meta.Env) {
  539. // Set up the conflicts list so that the *-unstable packages
  540. // cannot be installed alongside the regular version.
  541. //
  542. // https://www.debian.org/doc/debian-policy/ch-relationships.html
  543. // is very explicit about Conflicts: and says that Breaks: should
  544. // be preferred and the conflicting files should be handled via
  545. // alternates. We might do this eventually but using a conflict is
  546. // easier now.
  547. return "ethereum, " + exe.Name
  548. }
  549. return ""
  550. }
  551. func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
  552. pkg := meta.Name() + "-" + meta.VersionString()
  553. pkgdir = filepath.Join(tmpdir, pkg)
  554. if err := os.Mkdir(pkgdir, 0755); err != nil {
  555. log.Fatal(err)
  556. }
  557. // Copy the source code.
  558. build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator))
  559. // Put the debian build files in place.
  560. debian := filepath.Join(pkgdir, "debian")
  561. build.Render("build/deb.rules", filepath.Join(debian, "rules"), 0755, meta)
  562. build.Render("build/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta)
  563. build.Render("build/deb.control", filepath.Join(debian, "control"), 0644, meta)
  564. build.Render("build/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta)
  565. build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta)
  566. build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta)
  567. for _, exe := range meta.Executables {
  568. install := filepath.Join(debian, meta.ExeName(exe)+".install")
  569. docs := filepath.Join(debian, meta.ExeName(exe)+".docs")
  570. build.Render("build/deb.install", install, 0644, exe)
  571. build.Render("build/deb.docs", docs, 0644, exe)
  572. }
  573. return pkgdir
  574. }
  575. // Windows installer
  576. func doWindowsInstaller(cmdline []string) {
  577. // Parse the flags and make skip installer generation on PRs
  578. var (
  579. arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging")
  580. signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`)
  581. upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
  582. workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
  583. )
  584. flag.CommandLine.Parse(cmdline)
  585. *workdir = makeWorkdir(*workdir)
  586. env := build.Env()
  587. maybeSkipArchive(env)
  588. // Aggregate binaries that are included in the installer
  589. var (
  590. devTools []string
  591. allTools []string
  592. gethTool string
  593. )
  594. for _, file := range allToolsArchiveFiles {
  595. if file == "COPYING" { // license, copied later
  596. continue
  597. }
  598. allTools = append(allTools, filepath.Base(file))
  599. if filepath.Base(file) == "geth.exe" {
  600. gethTool = file
  601. } else {
  602. devTools = append(devTools, file)
  603. }
  604. }
  605. // Render NSIS scripts: Installer NSIS contains two installer sections,
  606. // first section contains the geth binary, second section holds the dev tools.
  607. templateData := map[string]interface{}{
  608. "License": "COPYING",
  609. "Geth": gethTool,
  610. "DevTools": devTools,
  611. }
  612. build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil)
  613. build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData)
  614. build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools)
  615. build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil)
  616. build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil)
  617. build.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll", 0755)
  618. build.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING", 0755)
  619. // Build the installer. This assumes that all the needed files have been previously
  620. // built (don't mix building and packaging to keep cross compilation complexity to a
  621. // minimum).
  622. version := strings.Split(build.VERSION(), ".")
  623. if env.Commit != "" {
  624. version[2] += "-" + env.Commit[:8]
  625. }
  626. installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, env) + ".exe")
  627. build.MustRunCommand("makensis.exe",
  628. "/DOUTPUTFILE="+installer,
  629. "/DMAJORVERSION="+version[0],
  630. "/DMINORVERSION="+version[1],
  631. "/DBUILDVERSION="+version[2],
  632. "/DARCH="+*arch,
  633. filepath.Join(*workdir, "geth.nsi"),
  634. )
  635. // Sign and publish installer.
  636. if err := archiveUpload(installer, *upload, *signer); err != nil {
  637. log.Fatal(err)
  638. }
  639. }
  640. // Android archives
  641. func doAndroidArchive(cmdline []string) {
  642. var (
  643. local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
  644. signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`)
  645. deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`)
  646. upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`)
  647. )
  648. flag.CommandLine.Parse(cmdline)
  649. env := build.Env()
  650. // Sanity check that the SDK and NDK are installed and set
  651. if os.Getenv("ANDROID_HOME") == "" {
  652. log.Fatal("Please ensure ANDROID_HOME points to your Android SDK")
  653. }
  654. if os.Getenv("ANDROID_NDK") == "" {
  655. log.Fatal("Please ensure ANDROID_NDK points to your Android NDK")
  656. }
  657. // Build the Android archive and Maven resources
  658. build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
  659. build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
  660. build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
  661. if *local {
  662. // If we're building locally, copy bundle to build dir and skip Maven
  663. os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar"))
  664. return
  665. }
  666. meta := newMavenMetadata(env)
  667. build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta)
  668. // Skip Maven deploy and Azure upload for PR builds
  669. maybeSkipArchive(env)
  670. // Sign and upload the archive to Azure
  671. archive := "geth-" + archiveBasename("android", env) + ".aar"
  672. os.Rename("geth.aar", archive)
  673. if err := archiveUpload(archive, *upload, *signer); err != nil {
  674. log.Fatal(err)
  675. }
  676. // Sign and upload all the artifacts to Maven Central
  677. os.Rename(archive, meta.Package+".aar")
  678. if *signer != "" && *deploy != "" {
  679. // Import the signing key into the local GPG instance
  680. b64key := os.Getenv(*signer)
  681. key, err := base64.StdEncoding.DecodeString(b64key)
  682. if err != nil {
  683. log.Fatalf("invalid base64 %s", *signer)
  684. }
  685. gpg := exec.Command("gpg", "--import")
  686. gpg.Stdin = bytes.NewReader(key)
  687. build.MustRun(gpg)
  688. keyID, err := build.PGPKeyID(string(key))
  689. if err != nil {
  690. log.Fatal(err)
  691. }
  692. // Upload the artifacts to Sonatype and/or Maven Central
  693. repo := *deploy + "/service/local/staging/deploy/maven2"
  694. if meta.Develop {
  695. repo = *deploy + "/content/repositories/snapshots"
  696. }
  697. build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X",
  698. "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh",
  699. "-Dgpg.keyname="+keyID,
  700. "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar")
  701. }
  702. }
  703. func gomobileTool(subcmd string, args ...string) *exec.Cmd {
  704. cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd)
  705. cmd.Args = append(cmd.Args, args...)
  706. cmd.Env = []string{
  707. "GOPATH=" + build.GOPATH(),
  708. "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"),
  709. }
  710. for _, e := range os.Environ() {
  711. if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") {
  712. continue
  713. }
  714. cmd.Env = append(cmd.Env, e)
  715. }
  716. return cmd
  717. }
  718. type mavenMetadata struct {
  719. Version string
  720. Package string
  721. Develop bool
  722. Contributors []mavenContributor
  723. }
  724. type mavenContributor struct {
  725. Name string
  726. Email string
  727. }
  728. func newMavenMetadata(env build.Environment) mavenMetadata {
  729. // Collect the list of authors from the repo root
  730. contribs := []mavenContributor{}
  731. if authors, err := os.Open("AUTHORS"); err == nil {
  732. defer authors.Close()
  733. scanner := bufio.NewScanner(authors)
  734. for scanner.Scan() {
  735. // Skip any whitespace from the authors list
  736. line := strings.TrimSpace(scanner.Text())
  737. if line == "" || line[0] == '#' {
  738. continue
  739. }
  740. // Split the author and insert as a contributor
  741. re := regexp.MustCompile("([^<]+) <(.+)>")
  742. parts := re.FindStringSubmatch(line)
  743. if len(parts) == 3 {
  744. contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]})
  745. }
  746. }
  747. }
  748. // Render the version and package strings
  749. version := build.VERSION()
  750. if isUnstableBuild(env) {
  751. version += "-SNAPSHOT"
  752. }
  753. return mavenMetadata{
  754. Version: version,
  755. Package: "geth-" + version,
  756. Develop: isUnstableBuild(env),
  757. Contributors: contribs,
  758. }
  759. }
  760. // XCode frameworks
  761. func doXCodeFramework(cmdline []string) {
  762. var (
  763. local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`)
  764. signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`)
  765. deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`)
  766. upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
  767. )
  768. flag.CommandLine.Parse(cmdline)
  769. env := build.Env()
  770. // Build the iOS XCode framework
  771. build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
  772. build.MustRun(gomobileTool("init"))
  773. bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
  774. if *local {
  775. // If we're building locally, use the build folder and stop afterwards
  776. bind.Dir, _ = filepath.Abs(GOBIN)
  777. build.MustRun(bind)
  778. return
  779. }
  780. archive := "geth-" + archiveBasename("ios", env)
  781. if err := os.Mkdir(archive, os.ModePerm); err != nil {
  782. log.Fatal(err)
  783. }
  784. bind.Dir, _ = filepath.Abs(archive)
  785. build.MustRun(bind)
  786. build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive)
  787. // Skip CocoaPods deploy and Azure upload for PR builds
  788. maybeSkipArchive(env)
  789. // Sign and upload the framework to Azure
  790. if err := archiveUpload(archive+".tar.gz", *upload, *signer); err != nil {
  791. log.Fatal(err)
  792. }
  793. // Prepare and upload a PodSpec to CocoaPods
  794. if *deploy != "" {
  795. meta := newPodMetadata(env, archive)
  796. build.Render("build/pod.podspec", "Geth.podspec", 0755, meta)
  797. build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings", "--verbose")
  798. }
  799. }
  800. type podMetadata struct {
  801. Version string
  802. Commit string
  803. Archive string
  804. Contributors []podContributor
  805. }
  806. type podContributor struct {
  807. Name string
  808. Email string
  809. }
  810. func newPodMetadata(env build.Environment, archive string) podMetadata {
  811. // Collect the list of authors from the repo root
  812. contribs := []podContributor{}
  813. if authors, err := os.Open("AUTHORS"); err == nil {
  814. defer authors.Close()
  815. scanner := bufio.NewScanner(authors)
  816. for scanner.Scan() {
  817. // Skip any whitespace from the authors list
  818. line := strings.TrimSpace(scanner.Text())
  819. if line == "" || line[0] == '#' {
  820. continue
  821. }
  822. // Split the author and insert as a contributor
  823. re := regexp.MustCompile("([^<]+) <(.+)>")
  824. parts := re.FindStringSubmatch(line)
  825. if len(parts) == 3 {
  826. contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]})
  827. }
  828. }
  829. }
  830. version := build.VERSION()
  831. if isUnstableBuild(env) {
  832. version += "-unstable." + env.Buildnum
  833. }
  834. return podMetadata{
  835. Archive: archive,
  836. Version: version,
  837. Commit: env.Commit,
  838. Contributors: contribs,
  839. }
  840. }
  841. // Cross compilation
  842. func doXgo(cmdline []string) {
  843. var (
  844. alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`)
  845. )
  846. flag.CommandLine.Parse(cmdline)
  847. env := build.Env()
  848. // Make sure xgo is available for cross compilation
  849. gogetxgo := goTool("get", "github.com/karalabe/xgo")
  850. build.MustRun(gogetxgo)
  851. // If all tools building is requested, build everything the builder wants
  852. args := append(buildFlags(env), flag.Args()...)
  853. if *alltools {
  854. args = append(args, []string{"--dest", GOBIN}...)
  855. for _, res := range allToolsArchiveFiles {
  856. if strings.HasPrefix(res, GOBIN) {
  857. // Binary tool found, cross build it explicitly
  858. args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))
  859. xgo := xgoTool(args)
  860. build.MustRun(xgo)
  861. args = args[:len(args)-1]
  862. }
  863. }
  864. return
  865. }
  866. // Otherwise xxecute the explicit cross compilation
  867. path := args[len(args)-1]
  868. args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...)
  869. xgo := xgoTool(args)
  870. build.MustRun(xgo)
  871. }
  872. func xgoTool(args []string) *exec.Cmd {
  873. cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...)
  874. cmd.Env = []string{
  875. "GOPATH=" + build.GOPATH(),
  876. "GOBIN=" + GOBIN,
  877. }
  878. for _, e := range os.Environ() {
  879. if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
  880. continue
  881. }
  882. cmd.Env = append(cmd.Env, e)
  883. }
  884. return cmd
  885. }
  886. // Binary distribution cleanups
  887. func doPurge(cmdline []string) {
  888. var (
  889. store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
  890. limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
  891. )
  892. flag.CommandLine.Parse(cmdline)
  893. if env := build.Env(); !env.IsCronJob {
  894. log.Printf("skipping because not a cron job")
  895. os.Exit(0)
  896. }
  897. // Create the azure authentication and list the current archives
  898. auth := build.AzureBlobstoreConfig{
  899. Account: strings.Split(*store, "/")[0],
  900. Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"),
  901. Container: strings.SplitN(*store, "/", 2)[1],
  902. }
  903. blobs, err := build.AzureBlobstoreList(auth)
  904. if err != nil {
  905. log.Fatal(err)
  906. }
  907. // Iterate over the blobs, collect and sort all unstable builds
  908. for i := 0; i < len(blobs); i++ {
  909. if !strings.Contains(blobs[i].Name, "unstable") {
  910. blobs = append(blobs[:i], blobs[i+1:]...)
  911. i--
  912. }
  913. }
  914. for i := 0; i < len(blobs); i++ {
  915. for j := i + 1; j < len(blobs); j++ {
  916. iTime, err := time.Parse(time.RFC1123, blobs[i].Properties.LastModified)
  917. if err != nil {
  918. log.Fatal(err)
  919. }
  920. jTime, err := time.Parse(time.RFC1123, blobs[j].Properties.LastModified)
  921. if err != nil {
  922. log.Fatal(err)
  923. }
  924. if iTime.After(jTime) {
  925. blobs[i], blobs[j] = blobs[j], blobs[i]
  926. }
  927. }
  928. }
  929. // Filter out all archives more recent that the given threshold
  930. for i, blob := range blobs {
  931. timestamp, _ := time.Parse(time.RFC1123, blob.Properties.LastModified)
  932. if time.Since(timestamp) < time.Duration(*limit)*24*time.Hour {
  933. blobs = blobs[:i]
  934. break
  935. }
  936. }
  937. // Delete all marked as such and return
  938. if err := build.AzureBlobstoreDelete(auth, blobs); err != nil {
  939. log.Fatal(err)
  940. }
  941. }