diff --git a/.gitignore b/.gitignore index d4fc4209..5c62a846 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,15 @@ -/*.tgz +//*.tgz /*.tar.gz -.rspec +internal/recipe/artifacts/ .bundle .ccache /logs ports/ +cflinuxfs4/ tmp/ mini_portile/ .idea/ oracle_client_libs *deb* **/.idea +/binary-builder diff --git a/.rspec b/.rspec deleted file mode 100644 index ddd9d61c..00000000 --- a/.rspec +++ /dev/null @@ -1,4 +0,0 @@ ---color ---require rspec/instafail ---format RSpec::Instafail ---format documentation diff --git a/.rubocop.yml b/.rubocop.yml deleted file mode 100644 index 8a606d73..00000000 --- a/.rubocop.yml +++ /dev/null @@ -1,18 +0,0 @@ -# This is the configuration used to check the rubocop source code. - -Style/Encoding: - Enabled: true - -Metrics/LineLength: - Enabled: false - -require: rubocop-rspec - -RSpec/FilePath: - Enabled: false - -Performance/RedundantMatch: - Enabled: false - -AllCops: - TargetRubyVersion: 3.4 diff --git a/.ruby-version b/.ruby-version deleted file mode 100644 index 1cf82530..00000000 --- a/.ruby-version +++ /dev/null @@ -1 +0,0 @@ -3.4.6 diff --git a/AGENTS.md b/AGENTS.md index 202e5ad8..c2677852 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,23 +1,287 @@ # Agent Guidelines for Binary Builder -## Test Commands -- Run all tests: `bundle exec rspec` -- Run single test: `bundle exec rspec spec/integration/ruby_spec.rb` -- Exclude Oracle PHP tests: `bundle exec rspec --tag ~run_oracle_php_tests` +## Overview -## Lint Commands -- Run RuboCop: `bundle exec rubocop` +Go tool that compiles CF buildpack dependencies (ruby, node, python, httpd, …) for a +specific CF stack (cflinuxfs4, cflinuxfs5). Entry point: `cmd/binary-builder/main.go`. + +--- + +## Build & Test Commands + +### Unit tests (Tier 1 — no Docker, no network) +```bash +go test ./... # all packages +go test -race ./... # with race detector (CI requirement) +``` + +### Run a single test or test file +```bash +# By test name (regex) within a package: +go test ./internal/recipe/ -run TestRubyRecipeBuild +go test ./internal/runner/ -run TestFakeRunner +go test ./internal/stack/ -run TestLoad + +# Run an entire package verbosely: +go test -v ./internal/recipe/ + +# Run with race detector for a single package: +go test -race ./internal/recipe/ -run TestHTTPD +``` + +### Parity tests (Tier 2 — requires Docker + network) +```bash +make parity-test DEP=httpd [STACK=cflinuxfs4] +make parity-test-all [STACK=cflinuxfs4] +``` + +### Exerciser test (Tier 3 — requires Docker) +```bash +make exerciser-test ARTIFACT=/tmp/ruby_3.3.6_...tgz STACK=cflinuxfs4 +``` + +### Build the binary +```bash +go build ./cmd/binary-builder +``` + +--- + +## Architecture + +``` +cmd/binary-builder/main.go ← CLI entry point +internal/ + recipe/ ← one file per dep (ruby.go, node.go, httpd.go, …) + php/ ← PHP extension recipes (pecl.go, fake_pecl.go) + runner/ ← Runner interface + RealRunner + FakeRunner + stack/ ← Stack struct + YAML loader + fetch/ ← Fetcher interface + HTTPFetcher + archive/ ← StripTopLevelDir, StripFiles, InjectFile + source/ ← source.Input (data.json parser) + output/ ← OutData, BuildOutput, DepMetadataOutput + artifact/ ← Artifact naming, SHA256, S3 URL helpers + apt/ ← apt.New(runner).Install(ctx, pkgs...) + portile/ ← configure/make/install wrapper +stacks/ ← cflinuxfs4.yaml, cflinuxfs5.yaml ← ALL stack-specific data lives here +``` + +### Key design rules +- **Stack config is data, not code.** Every Ubuntu-version-specific value + (apt packages, compiler paths, bootstrap URLs) lives in `stacks/{stack}.yaml`. + Recipes read from `*stack.Stack`; no stack names are hardcoded in Go source. +- **Runner interface** — all `exec.Cmd` usage goes through `runner.Runner`. + `RealRunner` executes; `FakeRunner` records calls for tests. +- **Fetcher interface** — all HTTP calls go through `fetch.Fetcher`. + `HTTPFetcher` does the real work; `FakeFetcher` (in `recipe_test.go`) is used in tests. +- **`RunInDirWithEnv` appends env vars** — appended vars win over inherited env on Linux. + So `GOTOOLCHAIN=local` appended DOES override any existing `GOTOOLCHAIN`. +- **miniconda3-py39 is URL-passthrough**: `Build()` sets `outData.URL`/`outData.SHA256` + directly instead of writing a file. `main.go` checks `if outData.URL == ""` before + calling `handleArtifact`. + +--- ## Code Style -- **Encoding**: Add `# encoding: utf-8` at the top of all Ruby files -- **Imports**: Use `require_relative` for local files, `require` for gems -- **Naming**: Use snake_case for methods/variables, CamelCase for classes -- **Classes**: Recipe classes inherit from `BaseRecipe` or `MiniPortile` -- **Error Handling**: Use `or raise 'Error message'` for critical failures, check `$?.success?` for command execution -- **String Interpolation**: Prefer double quotes and `#{}` for interpolation -- **Methods**: Define helper methods as private when appropriate + +### Language & toolchain +- **Go only.** The Ruby binary-builder has been fully removed. +- Module: `github.com/cloudfoundry/binary-builder` — use this import path. +- Minimum Go version: see `go.mod`. + +### Naming conventions +| Kind | Convention | Example | +|------|-----------|---------| +| Exported type / func | PascalCase | `RubyRecipe`, `NewRegistry` | +| Unexported func / var | camelCase | `buildRegistry`, `mustCwd` | +| Interface | noun (no `I` prefix) | `Runner`, `Fetcher`, `Recipe` | +| Test helper | camelCase | `newFakeFetcher`, `useTempWorkDir` | +| Constants | PascalCase (exported) or camelCase | — | + +### Import grouping +Three groups, separated by blank lines: +```go +import ( + // 1. stdlib + "context" + "fmt" + "os" + + // 2. third-party + "gopkg.in/yaml.v3" + "github.com/stretchr/testify/assert" + + // 3. internal + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/stack" +) +``` + +### Error handling +- Return errors explicitly; never panic in production code paths. +- Wrap with context using `fmt.Errorf("component: action: %w", err)`. +- Pattern: `return fmt.Errorf("ruby: apt install ruby_build: %w", err)` +- Error strings are lowercase (Go convention). +- On fatal CLI errors: `fmt.Fprintf(os.Stderr, "binary-builder: %v\n", err); os.Exit(1)`. + +### Comments +- Package-level doc comment on every package: `// Package foo does X.` +- Exported types/funcs always have doc comments. +- Inline comments explain *why*, not *what*. +- Use `// nolint:errcheck` only when the error is genuinely ignorable (e.g., closing + a writer in a test after all data is flushed). + +### Structs and interfaces +- Define interfaces where the consumer lives, not where the implementation lives. +- Struct fields use PascalCase; YAML tags use snake_case: `InstallDir string \`yaml:"install_dir"\``. +- Zero-value structs should be usable where practical. + +--- ## Recipe Patterns -- Override `computed_options`, `url`, `archive_files`, `prefix_path` in recipe classes -- Use `execute()` for build steps, `run()` for apt/system commands -- Place recipes in `recipe/` directory, tests in `spec/integration/` + +Every recipe implements `recipe.Recipe`: +```go +type Recipe interface { + Name() string + Build(ctx context.Context, s *stack.Stack, src *source.Input, r runner.Runner, outData *output.OutData) error + Artifact() ArtifactMeta // OS, Arch, Stack ("" = use build stack) +} +``` + +### Shared recipe abstractions + +Before writing a new recipe from scratch, check whether one of these abstractions fits: + +| Abstraction | Location | Use when | +|-------------|----------|----------| +| `autoconf.Recipe` | `internal/autoconf/` | configure / make / make install cycle (libunwind, libgdiplus, openresty, nginx) | +| `RepackRecipe` | `internal/recipe/repack.go` | Download an archive and optionally strip its top-level dir (bower, yarn, setuptools, rubygems) | +| `BundleRecipe` | `internal/recipe/bundle.go` | `pip3 download` multiple packages into a tarball (pip, pipenv) | +| `GoToolRecipe` | `internal/recipe/dep.go` | Download + build a Go tool with `go get`/`go build` (dep, glide, godep) | +| `PassthroughRecipe` | `internal/recipe/passthrough.go` | No build step — just record the upstream URL and SHA256 | + +#### Using `autoconf.Recipe` + +`autoconf.Recipe` lives in `internal/autoconf/` (separate package to avoid import cycles). +It is **not** a `recipe.Recipe` itself — wrap it in a thin struct in `internal/recipe/`: + +```go +type MyRecipe struct{ Fetcher fetch.Fetcher } + +func (r *MyRecipe) Name() string { return "mylib" } +func (r *MyRecipe) Artifact() ArtifactMeta { return ArtifactMeta{OS: "linux", Arch: "x64"} } + +func (r *MyRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, + run runner.Runner, out *output.OutData) error { + return (&autoconf.Recipe{ + DepName: "mylib", + Fetcher: r.Fetcher, + Hooks: autoconf.Hooks{ + AptPackages: func(s *stack.Stack) []string { return s.AptPackages["mylib_build"] }, + ConfigureArgs: func(_, prefix string) []string { return []string{"--prefix=" + prefix, "--enable-shared"} }, + PackDirs: func() []string { return []string{"include", "lib"} }, + }, + }).Build(ctx, s, src, run, out) +} +``` + +Available hooks (all optional — nil = default behaviour): + +| Hook | Default | Typical override | +|------|---------|-----------------| +| `AptPackages` | `s.AptPackages["{name}_build"]` | Custom package list | +| `BeforeDownload` | no-op | GPG verification (nginx) | +| `SourceProvider` | fetch tarball, extract to `/tmp/{name}-{version}` | `git clone` (libgdiplus), read from `source/` (libunwind) | +| `AfterExtract` | no-op | `autoreconf -i`, `autogen.sh` | +| `ConfigureArgs` | `["--prefix={prefix}"]` | Full custom args | +| `ConfigureEnv` | nil | `CFLAGS`, `CXXFLAGS` | +| `MakeArgs` | nil | `["-j2"]` | +| `InstallEnv` | nil (falls back to `ConfigureEnv`) | `DESTDIR` (nginx) | +| `AfterInstall` | no-op | Remove runtime dirs, symlinks | +| `PackDirs` | `["."]` | `["include", "lib"]`, `["lib"]` | +| `AfterPack` | no-op | `archive.StripTopLevelDir` (nginx) | + +### Adding a new recipe +1. Create `internal/recipe/{name}.go` with a struct implementing `Recipe`. +2. Register it in `buildRegistry()` in `cmd/binary-builder/main.go`. +3. Add a test in `internal/recipe/recipe_test.go` or a new `{name}_test.go` file. +4. If the dep is architecture-neutral, set `Arch: "noarch"` in `ArtifactMeta`. +5. For URL-passthrough deps (no build step), use `PassthroughRecipe` or + set `outData.URL`/`outData.SHA256` directly. +6. For autoconf-based deps, use `autoconf.Recipe` with hooks (see above). +7. For download-and-strip deps, use `RepackRecipe`. + +### Stack-specific behaviour +Recipes **must not** contain `if s.Name == "cflinuxfs4"` guards. Instead: +- Add the relevant value to both `stacks/cflinuxfs4.yaml` and `stacks/cflinuxfs5.yaml`. +- Read it from `s.AptPackages["key"]`, `s.Python.UseForceYes`, etc. + +--- + +## Testing Conventions + +- Test package: always use `package recipe_test` (external test package) for `internal/recipe/`. + Other packages follow the same `_test` suffix convention. +- Assertion library: `github.com/stretchr/testify/assert` (non-fatal) and + `github.com/stretchr/testify/require` (fatal / setup). +- Use `require.NoError` for setup steps; `assert.*` for behaviour assertions. +- **`FakeRunner`** in `internal/runner/runner.go` — inject instead of `RealRunner` in tests. + Inspect `fakeRunner.Calls` to verify command sequence, args, env, and dir. +- **`FakeFetcher`** defined in `recipe_test.go` — inject instead of `HTTPFetcher`. + Inspect `f.DownloadedURLs` and set `f.ErrMap` / `f.BodyMap` to control behaviour. +- **`useTempWorkDir(t)`** — helper in `recipe_helpers_test.go`. Call it in tests that + need a clean CWD (recipes write artifacts relative to CWD). NOT safe for `t.Parallel()`. +- **`writeFakeArtifact(t, name)`** — creates a minimal valid `.tgz` in CWD so archive + helpers don't fail when processing a fake build output. +- **Table-driven tests** are preferred for multiple similar cases (see + `TestCompiledRecipeArtifactMetaSanity`). +- Test names follow `Test{Type}{Behaviour}` pattern: `TestRubyRecipeBuild`, + `TestNodeRecipeStripsVPrefix`. + +--- + +## Parity Test Infrastructure + +- Script: `test/parity/compare-builds.sh --dep --data-json [--stack ]` +- Logs: `/tmp/parity-logs/--.log` +- The Go builder is compiled **inside the container at runtime** (`go build ./cmd/binary-builder`), + so source changes are always picked up on re-run without a separate image rebuild. + +### Sample data.json files (on the build host at `/tmp/`) +| File | Dep | Version | +|------|-----|---------| +| `/tmp/go-data.json` | go | 1.22.0 | +| `/tmp/node-data.json` | node | 20.11.0 | +| `/tmp/php-data.json` | php | 8.1.32 | +| `/tmp/r-data.json` | r | 4.2.3 | +| `/tmp/jruby-data.json` | jruby | 9.4.14.0 | +| `/tmp/appdynamics-data.json` | appdynamics | 23.11.0-839 | +| `/tmp/skywalking-data.json` | skywalking-agent | 9.5.0 | + +R sub-dep data.json files live under `/tmp/r-sub-deps/source-{pkg}-latest/data.json` +(forecast, plumber, rserve, shiny). + +--- + +## Key Files Reference + +| File | Purpose | +|------|---------| +| `cmd/binary-builder/main.go` | CLI, `findIntermediateArtifact`, `handleArtifact`, `buildRegistry` | +| `internal/runner/runner.go` | `Runner` interface, `RealRunner`, `FakeRunner` + `Call` type | +| `internal/fetch/fetch.go` | `Fetcher` interface, `HTTPFetcher` | +| `internal/stack/stack.go` | `Stack` struct, `Load(stacksDir, name)` | +| `internal/recipe/recipe.go` | `Recipe` interface, `Registry` | +| `internal/archive/archive.go` | `StripTopLevelDir`, `StripFiles`, `InjectFile` | +| `internal/portile/` | configure/make/install abstraction | +| `internal/apt/` | apt-get install wrapper | +| `stacks/cflinuxfs4.yaml` | All cflinuxfs4-specific values | +| `stacks/cflinuxfs5.yaml` | All cflinuxfs5-specific values | +| `test/parity/compare-builds.sh` | Parity test harness | +| `Makefile` | `unit-test`, `unit-test-race`, `parity-test`, `exerciser-test` | + +## cflinuxfs4/ports/ note +The `cflinuxfs4/ports/` directory contains root-owned build artifacts from previous Ruby +parity test runs. These cannot be removed without `sudo` and are NOT tracked by git. +They do not affect builds or tests. diff --git a/Gemfile b/Gemfile deleted file mode 100644 index c863b472..00000000 --- a/Gemfile +++ /dev/null @@ -1,21 +0,0 @@ -# encoding: utf-8 -source 'https://rubygems.org' - -ruby '~> 3.4' - -gem 'mini_portile', git: 'https://github.com/cf-buildpacks-eng/mini_portile' -gem 'net-ftp' - -group :test do - gem 'rspec' - gem 'rspec-instafail' - gem 'pry' -end - -group :development do - gem 'rainbow', '~> 2.1.0' - gem 'rubocop', '~> 0.43.0' - gem 'rubocop-rspec', '~> 1.7.0' - gem 'racc' - gem 'base64' -end diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index 44d4237c..00000000 --- a/Gemfile.lock +++ /dev/null @@ -1,77 +0,0 @@ -GIT - remote: https://github.com/cf-buildpacks-eng/mini_portile - revision: 53130df22960c079c6976e780cb2f74bebbb0031 - specs: - mini_portile (0.7.0.rc4) - -GEM - remote: https://rubygems.org/ - specs: - ast (2.4.0) - base64 (0.3.0) - coderay (1.1.2) - date (3.4.1) - diff-lcs (1.3) - method_source (0.9.0) - net-ftp (0.3.8) - net-protocol - time - net-protocol (0.2.2) - timeout - parser (2.5.1.2) - ast (~> 2.4.0) - powerpack (0.1.2) - pry (0.11.3) - coderay (~> 1.1.0) - method_source (~> 0.9.0) - racc (1.8.1) - rainbow (2.1.0) - rspec (3.8.0) - rspec-core (~> 3.8.0) - rspec-expectations (~> 3.8.0) - rspec-mocks (~> 3.8.0) - rspec-core (3.8.0) - rspec-support (~> 3.8.0) - rspec-expectations (3.8.1) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.8.0) - rspec-instafail (1.0.0) - rspec - rspec-mocks (3.8.0) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.8.0) - rspec-support (3.8.0) - rubocop (0.43.0) - parser (>= 2.3.1.1, < 3.0) - powerpack (~> 0.1) - rainbow (>= 1.99.1, < 3.0) - ruby-progressbar (~> 1.7) - unicode-display_width (~> 1.0, >= 1.0.1) - rubocop-rspec (1.7.0) - rubocop (>= 0.42.0) - ruby-progressbar (1.10.0) - time (0.4.1) - date - timeout (0.4.3) - unicode-display_width (1.4.0) - -PLATFORMS - ruby - -DEPENDENCIES - base64 - mini_portile! - net-ftp - pry - racc - rainbow (~> 2.1.0) - rspec - rspec-instafail - rubocop (~> 0.43.0) - rubocop-rspec (~> 1.7.0) - -RUBY VERSION - ruby 3.4.6p54 - -BUNDLED WITH - 2.7.2 diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..5797770d --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +.PHONY: test unit-test unit-test-race parity-test parity-test-all exerciser-test + +# Tier 1: unit tests (no Docker, no network) +unit-test: + go test ./... + +# Tier 1 with race detector +unit-test-race: + go test -race ./... + +# Tier 2: parity test for a single dep (requires Docker + network) +# Uses the same data.json values as parity-test-all (defined in run-all.sh). +# Usage: make parity-test DEP=httpd [STACK=cflinuxfs4] +STACK ?= cflinuxfs4 +parity-test: + @test -n "$(DEP)" || (echo "DEP is required. Usage: make parity-test DEP= [STACK=]"; exit 1) + DEP=$(DEP) ./test/parity/run-all.sh "$(STACK)" + +# Tier 2: parity test for all deps in the matrix (requires Docker + network) +parity-test-all: + ./test/parity/run-all.sh "$(STACK)" + +# Tier 3: exerciser test for a single artifact (requires Docker) +# Usage: make exerciser-test ARTIFACT=/tmp/ruby_3.3.6_...tgz STACK=cflinuxfs4 +exerciser-test: + @test -n "$(ARTIFACT)" || (echo "ARTIFACT is required"; exit 1) + @test -n "$(STACK)" || (echo "STACK is required"; exit 1) + ARTIFACT="$(ARTIFACT)" STACK="$(STACK)" \ + go test -tags integration ./test/exerciser/ -v + +# Run Tier 1 + Tier 2 (requires Docker + network) +test: unit-test parity-test-all diff --git a/PHP-Geoip.md b/PHP-Geoip.md deleted file mode 100644 index fc480305..00000000 --- a/PHP-Geoip.md +++ /dev/null @@ -1,17 +0,0 @@ -# PHP & GeoIP Support - -The binary builder has support for building the `geoip` extension for PHP 5.5, 5.6 and 7.0. In order for the `geoip` extension to function properly though it requires a database of geoip data. The company MaxMind provides both commercial and less accurate open source versions of these databases. - -In the default mode, binary-builder will not bundle any database files with PHP, however it will bundle a script that can be run to download an up-to-date version of the open source (also called "lite") databases. - -If you would like binary-builder to download the open source version of the databases and bundle them with PHP, it can do that too. To instruct binary-builder to do that, create a file called `BUNDLE_GEOIP_LITE` in the top level of the project and set the contents of the file to `true`. When binary-builder sees that this file exists and that the contents are `true` it will download and bundle the open source version of the databases with the resulting PHP binary. - -Right now binary-builder can't be configured to download and bundle commercial versions of the geoip data from MaxMind. You can download commercial versions via the included script (see below), but there is no option exposed to configure the script through binary-builder. - -## What's Included - -When you build PHP binaries with GeoIP support you get the following included with the PHP binary: - -1. The `geoip` extension (from PECL) -2. The library file `geoipdb/lib/geoip_downloader.rb` and script `geoipdb/bin/download_geoip_db.rb` which can be run at a later date to download a copy of the geoip databases from MaxMind. By default they will download the "lite" or open source versions but the script can be configured to use a user id & license key to download paid versions as well. -3. If bundled, the geoip databases will be under `geoipdb/dbs`. If not bundled, that directory will be empty. If bundled, the following databases will be included: `GeoLiteCityv6.dat`, `GeoLiteASNum.dat`, `GeoLiteCountry.dat`, `GeoIPv6.dat`, and `GeoLiteCity.dat`. diff --git a/PHP-Oracle.md b/PHP-Oracle.md deleted file mode 100644 index c9615722..00000000 --- a/PHP-Oracle.md +++ /dev/null @@ -1,57 +0,0 @@ -# PHP & Oracle Support - -The binary builder has support for building the `oci8` and `pdo_oci` extensions of PHP 5.5, 5.6 and 7.0. While the builder is capable of building them, it does *not* provide the Oracle libraries and SDK which are required to build these extensions. These are not provided by binary-builder because of licensing restrictions and binary builder does not automatically download them because they are behind an Oracle pay-wall. To build the extensions, you *must* download and provide these to the builder. - -## What to Downwload - -The requirements to build these extensions are listed here. - - http://php.net/manual/en/oci8.requirements.php - -This basically boils down to installing the Oracle Instant Client Basic or Basic Lite plus the SDK. Use the ZIP installs and extract them to a location on your local machine. Then create the following symbolic library before building: `ln -s libclntsh.so.12.1 libclntsh.so` (version number must vary). - -You only need to do this once. - -## How to Build - -To build, you just [follow the normal instructions for building PHP with binary builder & Docker](https://github.com/cloudfoundry/binary-builder/blob/master/README.md). The only exception is that you need to map the path where you extracted the Oracle instant client and SDK to `/oracle` in the docker container used by binary builder. - -This is done by adding an additiona `-v` argument to the `docker run` command. - -Ex: - -``` -docker run -w /binary-builder -v `pwd`:/binary-builder -v /path/to/oracle:/oracle -it cloudfoundry/cflinuxfs3 bash -export STACK=cflinuxfs3 -./bin/binary-builder --name=php --version=7.1.29 --md5=ae625e0cfcfdacea3e7a70a075e47155 --php-extensions-file=./php71-extensions.yml -``` - -## What's Included - -When you build PHP binaries with Oracle support you get the following included with the PHP binary: - -1. The `oci8` extension (from PECL) - - PHP 5.5 & 5.6 include oci8 2.0.x - - PHP 7.0 includes oci 2.1.x -2. The `pdo_oci` extension bundled with PHP -3. The following libraries which are required by the extensions are include in `php/lib` - - libclntshcore.so - - libclntsh.so - - libipc1.so - - libmql1.so - - libnnz12.so - - libociicus.so - - libons.so - -Two notes on the included libraries: - -1. The file `libociicus.so` is a US English specific library from Instant Client lite. If you need multi-language support, you will need to install the full Instant Client and likely include additional libraries. That's not supported at this time, but patches are welcome. - -2. The PHP bundle that is built by binary builder has Oracle libraries from the Instant Client packaged with it. As such, you should not publicly distribute these libraries unless you are licensed to do so by Oracle. - -## Disabling Oracle Support - -By default Oracle Support is *not* included. The binary builder will not and cannot build these extensions unless you provide it with the Oracle libraries as listed in the *What to Download* section above. - -If you want to build with and without Oracle support, you can control if binary-builder will include the Oracle support by adding or removing the volume map for `/oracle` on your `docker run` command. If that volume is mounted then binary builder will attempt to build Oracle support. If it's not mounted, Oracle support is disabled. - diff --git a/README.md b/README.md index 67d66e51..5e079431 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,294 @@ -# Introduction +# binary-builder -This tool provides a mechanism for building binaries for the Cloud Foundry buildpacks. +A Go tool for building binaries used by Cloud Foundry buildpacks. -## Currently supported binaries +## Supported binaries -* NodeJS -* Ruby -* JRuby -* Python -* PHP -* Nginx -* Apache HTTPD Server -* Go -* Glide -* Godep -* Bundler +| Dependency | Stacks | +|---|---| +| Ruby | cflinuxfs4, cflinuxfs5 | +| JRuby | cflinuxfs4, cflinuxfs5 | +| Python | cflinuxfs4, cflinuxfs5 | +| Node.js | cflinuxfs4, cflinuxfs5 | +| Go | cflinuxfs4, cflinuxfs5 | +| PHP | cflinuxfs4, cflinuxfs5 | +| Nginx / nginx-static / OpenResty | cflinuxfs4, cflinuxfs5 | +| Apache HTTPD | cflinuxfs4, cflinuxfs5 | +| Bundler | cflinuxfs4, cflinuxfs5 | +| RubyGems | cflinuxfs4, cflinuxfs5 | +| Yarn / Bower / Composer | cflinuxfs4, cflinuxfs5 | +| Pip / Pipenv / Setuptools | cflinuxfs4, cflinuxfs5 | +| OpenJDK / Zulu / SAPMachine | cflinuxfs4, cflinuxfs5 | +| .NET SDK / Runtime / ASP.NET Core | cflinuxfs4, cflinuxfs5 | +| HWC | cflinuxfs4, cflinuxfs5 | +| R | cflinuxfs4, cflinuxfs5 | +| libgdiplus / libunwind | cflinuxfs4, cflinuxfs5 | +| miniconda3-py39 | cflinuxfs4, cflinuxfs5 | +| AppDynamics / SkyWalking / JProfiler / YourKit | cflinuxfs4, cflinuxfs5 | +| Tomcat | cflinuxfs4, cflinuxfs5 | -# Usage +## Usage -The scripts are meant to be run as root on a Cloud Foundry [stack](https://docs.cloudfoundry.org/concepts/stacks.html). +The tool supports two input modes. -## Running within Docker +### Mode 1 — Direct flags (manual / local use) -To run `binary-builder` from within the cflinuxfs3 rootfs, use [Docker](https://docker.io): +``` +binary-builder build \ + --stack cflinuxfs4 \ + --name ruby \ + --version 3.3.6 \ + --sha256 +``` -```bash -docker run -w /binary-builder -v `pwd`:/binary-builder -it cloudfoundry/cflinuxfs3 bash -export STACK=cflinuxfs3 -./bin/binary-builder --name=[binary_name] --version=[binary_version] --(md5|sha256)=[checksum_value] +`--url`, `--sha256`, and `--sha512` are optional; include whichever checksums the +recipe needs to verify the source download. + +### Mode 2 — Source file (CI / depwatcher use) + +``` +binary-builder build \ + --stack cflinuxfs4 \ + --source-file source/data.json ``` -This generates a gzipped tarball in the binary-builder directory with the filename format `binary_name-binary_version-linux-x64`. +`data.json` is the standard depwatcher output format: -For example, if you were building ruby 2.2.3, you'd run the following commands: +```json +{ + "source": { "name": "ruby", "type": "github_releases", "repo": "ruby/ruby" }, + "version": { "url": "https://...", "ref": "3.3.6", "sha256": "...", "sha512": "" } +} +``` -```bash -$ docker run -w /binary-builder -v `pwd`:/binary-builder -it cloudfoundry/cflinuxfs3:ruby-2.2.4 ./bin/binary-builder --name=ruby --version=2.2.3 --md5=150a5efc5f5d8a8011f30aa2594a7654 -$ ls -ruby-2.2.3-linux-x64.tgz +If `--source-file` is omitted and `source/data.json` exists in the current +working directory, it is used automatically. + +### Common flags + +| Flag | Default | Description | +|---|---|---| +| `--stack` | *(required)* | Stack name, e.g. `cflinuxfs4` or `cflinuxfs5` | +| `--stacks-dir` | `stacks` | Directory containing per-stack YAML config files | +| `--output-file` | `summary.json` | Path for the JSON build summary (see below) | + +### Output + +The artifact (`.tgz` or `.zip`) is written to the **current working directory** +using the canonical filename: + +``` +_____. ``` -# Building PHP +A JSON summary is written to `--output-file` (default: `summary.json`): + +```json +{ + "artifact_path": "ruby_3.3.6_linux_x64_cflinuxfs4_abcdef01.tgz", + "version": "3.3.6", + "sha256": "abcdef01...", + "url": "https://buildpacks.cloudfoundry.org/dependencies/ruby/ruby_3.3.6_...", + "source": { "url": "...", "sha256": "...", "sha512": "...", "md5": "...", "sha1": "..." }, + "sub_dependencies": { "bundler": { "version": "2.5.6", "source": { ... } } }, + "git_commit_sha": "..." +} +``` -To build PHP, you also need to pass in a YAML file containing information about the various PHP extensions to be built. For example +`sub_dependencies` and `git_commit_sha` are omitted when not applicable. +All build subprocess output (compiler, make, etc.) goes to stdout/stderr so it +is visible in logs without corrupting the structured JSON output file. + +The CI task that wraps this tool is responsible for moving the artifact, +writing dep-metadata and builds-artifacts JSON, and committing to git. + +### PHP + +PHP is built the same way as any other dependency — no extra flags needed. +Extension and native module definitions are embedded directly in the binary +(see `internal/php/assets/`): + +``` +binary-builder build \ + --stack cflinuxfs4 \ + --name php \ + --version 8.1.32 \ + --sha256 +``` + +To add support for a new PHP minor version, create +`internal/php/assets/php-extensions-patch.yml` with any +additions or exclusions relative to the major-version base file. No code +changes are required — the file is discovered automatically at build time. + +## Building ```bash -docker run -w /binary-builder -v `pwd`:/binary-builder -it cloudfoundry/cflinuxfs3 bash -export STACK=cflinuxfs3 -./bin/binary-builder --name=php7 --version=7.3.14 --sha256=6aff532a380b0f30c9e295b67dc91d023fee3b0ae14b4771468bf5dda4cbf108 --php-extensions-file=./php7-extensions.yml +go build ./cmd/binary-builder ``` -For an example of what this file looks like, see: [php7-base-extensions.yml](https://github.com/cloudfoundry/buildpacks-ci/tree/master/tasks/build-binary-new) and the various `php*-extensions-patch.yml` files in that same directory. Patch files adjust the base-extensions.yml file by adding/removing extensions. The `--php-extensions-file` argument will need the base-extensions file with one of the patch files applied. That normally happens automatically through the pipeline, so if you are building manually you need to manually create this file. +## Testing -**TIP** If you are updating or building a specific PHP extension, remove everything except that specific extension from your `--php-extensions-file` file. This will decrease the build times & make it easier for you to test your changes. +```bash +# Unit tests (no Docker or network required) +make unit-test + +# Unit tests with race detector +make unit-test-race -# Building nginx +# Parity test for a single dep from the matrix (requires Docker + network) +# VERSION is not an argument — each dep runs at the version pinned in run-all.sh. +make parity-test DEP=ruby +make parity-test DEP=php STACK=cflinuxfs4 -Nginx uses GPG keys to verify the source tarball, so you'll need something like the following code to build the NGinx binary: +# To test a specific version not in the matrix, call compare-builds.sh directly +# with a custom data.json: +test/parity/compare-builds.sh --dep php --data-json /tmp/php-8.3.0-data.json --stack cflinuxfs4 +# Parity test for all deps +make parity-test-all ``` -version=1.15.9 -gpg_signature_url="http://nginx.org/download/nginx-${version}.tar.gz.asc" -gpg_signature=`curl -sL ${gpg_signature_url}` -docker run -w /binary-builder -v `pwd`:/binary-builder \ - -it cloudfoundry/cflinuxfs3 ./bin/binary-builder \ - --name=nginx-static --gpg-rsa-key-id=A1C052F8 \ - --version=$version --gpg-signature="${gpg_signature}" +## Architecture + +- `cmd/binary-builder/` — CLI entry point +- `internal/recipe/` — per-dependency build recipes +- `internal/php/` — PHP extension build logic and embedded extension data (`assets/`) +- `internal/archive/` — tarball / zip manipulation helpers +- `internal/runner/` — subprocess execution helpers +- `stacks/` — per-stack YAML configuration (versions, URLs, paths) +- `test/parity/` — Parity test scripts (compare Ruby vs Go builder outputs) + +## Parity Tests + +The parity tests verify that the Go builder produces identical output to the +original Ruby builder for every supported dependency. This is the primary +confidence check that the Go rewrite is correct. + +### Scripts + +| Script | Purpose | +|---|---| +| `test/parity/run-all.sh` | Runs every dep in the test matrix sequentially; prints a pass/fail summary and tails failure logs | +| `test/parity/compare-builds.sh` | Runs both builders for a single dep and diffs their output | + +### How it works + +For each dependency, `compare-builds.sh` does the following: + +**1. Source pre-download** + +Some deps (`libunwind`, `dotnet-*`, `jprofiler-profiler`, `your-kit-profiler`) +are built from a source tarball that must already be present in a `source/` +directory at build time — neither builder downloads them inline. The script +downloads the tarball on the host first, then mounts it into both containers +as a read-only volume at `/tmp/host-source/`. + +All other deps download their own source inside the container during the build. + +**2. Run the Ruby builder** + +Runs `buildpacks-ci/tasks/build-binary-new-cflinuxfs4/build.rb` inside a +`cloudfoundry/` Docker container with this layout: + +``` +/task/ + source/data.json ← the depwatcher input + source/ ← pre-downloaded source (if applicable) + source-*-latest/ ← R sub-dep data.json dirs (r dep only) + binary-builder/ ← symlink to this repo + buildpacks-ci/ ← symlink to ../buildpacks-ci + artifacts/ ← artifact output (*.tgz / *.zip) + dep-metadata/ ← dep-metadata JSON output + builds-artifacts/ + binary-builds-new// ← builds JSON output +``` + +`SKIP_COMMIT=true` prevents git commits. Ruby 3.4.6 is compiled from source +inside the container if not already present. + +**3. Run the Go builder** + +Compiles `binary-builder` from source inside the same `cloudfoundry/` +container (using `mise` to install the required Go version), then runs: + +``` +binary-builder build \ + --stack \ + --source-file /tmp/data.json \ + --stacks-dir /binary-builder/stacks \ + --output-file /out/summary.json ``` -# Contributing +The JSON summary written to `--output-file` is then used by the script to move +the artifact, write the dep-metadata JSON, and write the builds-artifacts JSON +into `/out/` — mirroring exactly what the CI task (`tasks/build-binary/build.sh`) +does in production. -Find our guidelines [here](./CONTRIBUTING.md). +The source tarball (if any) and R sub-dep dirs are copied into the working +directory before the build runs. -# Reporting Issues +**4. Compare outputs** -Open an issue on this project +If the Ruby builder failed, the comparison is skipped entirely — the test exits +0 with `RUBY BROKEN`. Otherwise all three output types are compared: -# Active Development +| Output | How it is compared | Hard failure? | +|---|---|---| +| **Artifact filename** | Both filenames are normalised by replacing the 8-char content SHA (`_.`) with `_.` then compared | Yes | +| **Artifact contents** | Files inside the `.tgz` or `.zip` are listed and sorted, then diffed | Yes | +| **Builds JSON** | Fields `version`, `source.url`, `source.sha256`, `source.sha512`, `source.md5`, `url`, `sha256`, and `sub_dependencies[*].version` are compared individually | Yes | +| **Dep-metadata JSON structural fields** | All fields except `sha256` and `url` (the artifact hash) and `sub_dependencies[*].source.sha256` are compared with `jq -S` (sorted keys) | Yes | +| **Dep-metadata JSON artifact hash** | Top-level `sha256` and `url` fields are diffed | Warn only — non-reproducible builds (e.g. `bundler`) legitimately differ | +| **Sub-dep source sha256** | `sub_dependencies[*].source.sha256` | Warn only — Ruby builder has a known bug where it records the sha256 of an HTTP redirect response body rather than the actual tarball | -The project backlog is on [Pivotal Tracker](https://www.pivotaltracker.com/projects/1042066) +### Exit outcomes -# Running the tests +| Result | Meaning | +|---|---| +| `PASS` | Both builders produced identical output on all hard-failure checks | +| `RUBY BROKEN` | Ruby builder failed; Go builder output not compared; exits 0 | +| `FAIL` | One or more hard-failure mismatches; exits 1 | + +### Input format + +Both builders receive the same depwatcher `data.json`: + +```json +{ + "source": { "name": "ruby", "type": "github_releases", "repo": "ruby/ruby" }, + "version": { "url": "https://...", "ref": "3.3.6", "sha256": "...", "sha512": "" } +} +``` + +For SHA512-only deps (e.g. `dotnet-*`, `skywalking-agent`), `sha256` is `""` +and `sha512` carries the real checksum. Both fields are always present in the +builder output — the `sha256` field is never omitted even when empty. + +### Running + +```bash +# All deps (requires Docker + network) +test/parity/run-all.sh [] + +# Single dep +test/parity/compare-builds.sh --dep ruby --data-json /tmp/ruby-data.json --stack cflinuxfs4 + +# R dep (needs sub-dep data.json dirs) +test/parity/compare-builds.sh --dep r --data-json /tmp/r-data.json \ + --sub-deps-dir /tmp/r-sub-deps +``` + +All output is written to both the terminal and +`/tmp/parity-logs/--.log`. To watch a running build: + +```bash +tail -f /tmp/parity-logs/ruby-3.3.6-cflinuxfs4.log +``` -The integration test suite includes specs that test the functionality for building [PHP with Oracle client libraries](./PHP-Oracle.md). These tests are tagged `:run_oracle_php_tests` and require access to an S3 bucket containing the Oracle client libraries. This is configured using the environment variables `AWS_ACCESS_KEY` and `AWS_SECRET_ACCESS_KEY`. +`run-all.sh` prints a summary at the end and tails the last 20 lines of each +failure log automatically. -Optionally provide `AWS_ASSUME_ROLE_ARN` to assume a role. +## Contributing -If you do not need to test this functionality, exclude the tag `:run_oracle_php_tests` when you run `rspec`. +See [CONTRIBUTING.md](./CONTRIBUTING.md). diff --git a/bin/binary-builder b/bin/binary-builder deleted file mode 100755 index b64498be..00000000 --- a/bin/binary-builder +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -# workaround to prevent tons of deprecation warnings from spamming the screen -# the deprecations are a result of Debian/Ubuntu packaging issues we cannot control -export DEBIAN_DISABLE_RUBYGEMS_INTEGRATION=foo - -gem update --system --no-document -q --silent > /dev/null -gem install bundler:2.4.22 --no-document -f -q --silent > /dev/null -bundle config mirror.https://rubygems.org ${RUBYGEM_MIRROR} -bundle install -bundle exec ./bin/binary-builder.rb "$@" diff --git a/bin/binary-builder.rb b/bin/binary-builder.rb deleted file mode 100755 index 2f21ffb6..00000000 --- a/bin/binary-builder.rb +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env ruby -# encoding: utf-8 - -require 'bundler' -require 'optparse' -require_relative '../lib/yaml_presenter' -require_relative '../lib/archive_recipe' -Dir['recipe/*.rb'].each { |f| require File.expand_path(f) } - -recipes = { - 'ruby' => RubyRecipe, - 'bundler' => BundlerRecipe, - 'node' => NodeRecipe, - 'jruby' => JRubyMeal, - 'httpd' => HTTPdMeal, - 'python' => PythonRecipe, - 'php' => PhpMeal, - 'nginx-static' => NginxRecipe, - 'godep' => GodepMeal, - 'glide' => GlideRecipe, - 'go' => GoRecipe, - 'dep' => DepRecipe, - 'hwc' => HwcRecipe -} - -options = {} -optparser = OptionParser.new do |opts| - opts.banner = 'USAGE: binary-builder [options] (A checksum method is required)' - - opts.on('-nNAME', '--name=NAME', "Name of the binary. Options: [#{recipes.keys.join(", ")}]") do |n| - options[:name] = n - end - opts.on('-vVERSION', '--version=VERSION', 'Version of the binary e.g. 1.7.11') do |n| - options[:version] = n - end - opts.on('--sha256=SHA256', 'SHA256 of the binary ') do |n| - options[:sha256] = n - end - opts.on('--md5=MD5', 'MD5 of the binary ') do |n| - options[:md5] = n - end - opts.on('--gpg-rsa-key-id=RSA_KEY_ID', 'RSA Key Id e.g. 10FDE075') do |n| - options[:gpg] ||= {} - options[:gpg][:key] = n - end - opts.on('--gpg-signature=ASC_KEY', 'content of the .asc file') do |n| - options[:gpg] ||= {} - options[:gpg][:signature] = n - end - opts.on('--git-commit-sha=SHA', 'git commit sha of the specified version') do |n| - options[:git] ||= {} - options[:git][:commit_sha] = n - end - opts.on('--php-extensions-file=FILE', 'yaml file containing PHP extensions + versions') do |n| - options[:php_extensions_file] = n - end -end -optparser.parse! - -unless options[:name] && options[:version] && ( - options[:sha256] || - options[:md5] || - (options.has_key?(:git) && options[:git][:commit_sha]) || - (options[:gpg][:signature] && options[:gpg][:key]) -) - raise optparser.help -end - -raise "Unsupported recipe [#{options[:name]}], supported options are [#{recipes.keys.join(", ")}]" unless recipes.has_key?(options[:name]) - -recipe_options = DetermineChecksum.new(options).to_h - -if options[:php_extensions_file] - recipe_options[:php_extensions_file] = options[:php_extensions_file] -end -recipe = recipes[options[:name]].new( - options[:name], - options[:version], - recipe_options -) -Bundler.with_clean_env do - puts "Source URL: #{recipe.url}" - - recipe.cook - ArchiveRecipe.new(recipe).compress! - - puts 'Source YAML:' - puts YAMLPresenter.new(recipe).to_yaml -end diff --git a/bin/download_geoip_db.rb b/bin/download_geoip_db.rb deleted file mode 100755 index f1475d4d..00000000 --- a/bin/download_geoip_db.rb +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env ruby -# encoding: utf-8 - -require "net/http" -require "uri" -require "digest" -require "tempfile" -require "optparse" -require_relative "../lib/geoip_downloader" - -options = {} -optparser = OptionParser.new do |opts| - opts.banner = 'USAGE: download_geoip_db [options]' - - opts.on('-uUSER', '--user=USER', 'User Id from MaxMind. Default "999999".') do |n| - options[:user] = n - end - - opts.on('-lLICENSE', '--license=LICENSE', 'License from MaxMind. Default "000000000000".') do |n| - options[:license] = n - end - - opts.on('-oOUTPUTDIR', '--output_dir=OUTPUTDIR', 'Directory where databases might exist and will be written / updated. Default "."') do |n| - options[:output_dir] = n - end - - opts.on('-pPRODUCTS', '--products=PRODUCTS', 'Space separated list of product ids. Default "GeoLite-Legacy-IPv6-City GeoLite-Legacy-IPv6-Country 506 517 533".') do |n| - options[:products] = n - end -end -optparser.parse! - -options[:user] ||= MaxMindGeoIpUpdater.FREE_USER -options[:license] ||= MaxMindGeoIpUpdater.FREE_LICENSE -options[:output_dir] ||= '.' -options[:products] ||= 'GeoLite-Legacy-IPv6-City GeoLite-Legacy-IPv6-Country 506 517 533' - -updater = MaxMindGeoIpUpdater.new(options[:user], options[:license], options[:output_dir]) - -options[:products].split(" ").each do |product| - updater.download_product(product) -end diff --git a/cflinuxfs4/.gitignore b/cflinuxfs4/.gitignore deleted file mode 100644 index 08bc84bb..00000000 --- a/cflinuxfs4/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -/*.tgz -/*.tar.gz -.rspec -.bundle -.ccache -/logs -ports/ -tmp/ -.idea/ -oracle_client_libs -*deb* diff --git a/cflinuxfs4/.rubocop.yml b/cflinuxfs4/.rubocop.yml deleted file mode 100644 index d19e8b8a..00000000 --- a/cflinuxfs4/.rubocop.yml +++ /dev/null @@ -1,15 +0,0 @@ -# This is the configuration used to check the rubocop source code. - -Style/Encoding: - Enabled: true - -Metrics/LineLength: - Enabled: false - -require: rubocop-rspec - -RSpec/FilePath: - Enabled: false - -AllCops: - TargetRubyVersion: 3.0.2 diff --git a/cflinuxfs4/.ruby-version b/cflinuxfs4/.ruby-version deleted file mode 100644 index 1cf82530..00000000 --- a/cflinuxfs4/.ruby-version +++ /dev/null @@ -1 +0,0 @@ -3.4.6 diff --git a/cflinuxfs4/CONTRIBUTING.md b/cflinuxfs4/CONTRIBUTING.md deleted file mode 100644 index 0d541800..00000000 --- a/cflinuxfs4/CONTRIBUTING.md +++ /dev/null @@ -1,13 +0,0 @@ -# Contributing - -## Run the tests - -```bash -bundle -bundle exec rspec -``` - -## Pull Requests - -1. Fork the project -1. Submit a pull request diff --git a/cflinuxfs4/Gemfile b/cflinuxfs4/Gemfile deleted file mode 100644 index 2cee8b48..00000000 --- a/cflinuxfs4/Gemfile +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -source 'https://rubygems.org' - -ruby '~> 3.4' - -gem 'mini_portile2' - -group :test do - gem 'pry' - gem 'rspec' - gem 'rspec-instafail' -end - -group :local do - gem 'rubocop' - gem 'rubocop-rspec' -end - -group :development do - gem 'rainbow' -end diff --git a/cflinuxfs4/Gemfile.lock b/cflinuxfs4/Gemfile.lock deleted file mode 100644 index 9e1746b4..00000000 --- a/cflinuxfs4/Gemfile.lock +++ /dev/null @@ -1,67 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - ast (2.4.2) - coderay (1.1.3) - diff-lcs (1.5.0) - json (2.6.2) - method_source (1.0.0) - mini_portile2 (2.8.0) - parallel (1.22.1) - parser (3.1.2.0) - ast (~> 2.4.1) - pry (0.14.1) - coderay (~> 1.1) - method_source (~> 1.0) - rainbow (3.1.1) - regexp_parser (2.5.0) - rexml (3.2.5) - rspec (3.11.0) - rspec-core (~> 3.11.0) - rspec-expectations (~> 3.11.0) - rspec-mocks (~> 3.11.0) - rspec-core (3.11.0) - rspec-support (~> 3.11.0) - rspec-expectations (3.11.0) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.11.0) - rspec-instafail (1.0.0) - rspec - rspec-mocks (3.11.1) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.11.0) - rspec-support (3.11.0) - rubocop (1.31.2) - json (~> 2.3) - parallel (~> 1.10) - parser (>= 3.1.0.0) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.8, < 3.0) - rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.18.0, < 2.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.19.1) - parser (>= 3.1.1.0) - rubocop-rspec (2.12.1) - rubocop (~> 1.31) - ruby-progressbar (1.11.0) - unicode-display_width (2.2.0) - -PLATFORMS - x86_64-linux - -DEPENDENCIES - mini_portile2 - pry - rainbow - rspec - rspec-instafail - rubocop - rubocop-rspec - -RUBY VERSION - ruby 3.4.6p54 - -BUNDLED WITH - 2.3.18 diff --git a/cflinuxfs4/LICENSE b/cflinuxfs4/LICENSE deleted file mode 100644 index 5d37e1cf..00000000 --- a/cflinuxfs4/LICENSE +++ /dev/null @@ -1,1168 +0,0 @@ -Node's license follows: - -==== - -Copyright Joyent, Inc. and other Node contributors. All rights reserved. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to -deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. - -==== - -This license applies to all parts of Node that are not externally -maintained libraries. The externally maintained libraries used by Node are: - -- V8, located at deps/v8. V8's license follows: - """ - This license applies to all parts of V8 that are not externally - maintained libraries. The externally maintained libraries used by V8 - are: - - - PCRE test suite, located in - test/mjsunit/third_party/regexp-pcre.js. This is based on the - test suite from PCRE-7.3, which is copyrighted by the University - of Cambridge and Google, Inc. The copyright notice and license - are embedded in regexp-pcre.js. - - - Layout tests, located in test/mjsunit/third_party. These are - based on layout tests from webkit.org which are copyrighted by - Apple Computer, Inc. and released under a 3-clause BSD license. - - - Strongtalk assembler, the basis of the files assembler-arm-inl.h, - assembler-arm.cc, assembler-arm.h, assembler-ia32-inl.h, - assembler-ia32.cc, assembler-ia32.h, assembler-x64-inl.h, - assembler-x64.cc, assembler-x64.h, assembler-mips-inl.h, - assembler-mips.cc, assembler-mips.h, assembler.cc and assembler.h. - This code is copyrighted by Sun Microsystems Inc. and released - under a 3-clause BSD license. - - - Valgrind client API header, located at third_party/valgrind/valgrind.h - This is release under the BSD license. - - These libraries have their own licenses; we recommend you read them, - as their terms may differ from the terms below. - - Copyright 2006-2012, the V8 project authors. All rights reserved. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """ - -- C-Ares, an asynchronous DNS client, located at deps/cares. C-Ares license - follows: - """ - /* Copyright 1998 by the Massachusetts Institute of Technology. - * - * Permission to use, copy, modify, and distribute this - * software and its documentation for any purpose and without - * fee is hereby granted, provided that the above copyright - * notice appear in all copies and that both that copyright - * notice and this permission notice appear in supporting - * documentation, and that the name of M.I.T. not be used in - * advertising or publicity pertaining to distribution of the - * software without specific, written prior permission. - * M.I.T. makes no representations about the suitability of - * this software for any purpose. It is provided "as is" - * without express or implied warranty. - """ - -- OpenSSL located at deps/openssl. OpenSSL is cryptographic software written - by Eric Young (eay@cryptsoft.com) to provide SSL/TLS encryption. OpenSSL's - license follows: - """ - /* ==================================================================== - * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * 3. All advertising materials mentioning features or use of this - * software must display the following acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" - * - * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to - * endorse or promote products derived from this software without - * prior written permission. For written permission, please contact - * openssl-core@openssl.org. - * - * 5. Products derived from this software may not be called "OpenSSL" - * nor may "OpenSSL" appear in their names without prior written - * permission of the OpenSSL Project. - * - * 6. Redistributions of any form whatsoever must retain the following - * acknowledgment: - * "This product includes software developed by the OpenSSL Project - * for use in the OpenSSL Toolkit (http://www.openssl.org/)" - * - * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY - * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR - * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - * OF THE POSSIBILITY OF SUCH DAMAGE. - * ==================================================================== - * - * This product includes cryptographic software written by Eric Young - * (eay@cryptsoft.com). This product includes software written by Tim - * Hudson (tjh@cryptsoft.com). - * - */ - """ - -- HTTP Parser, located at deps/http_parser. HTTP Parser's license follows: - """ - http_parser.c is based on src/http/ngx_http_parse.c from NGINX copyright - Igor Sysoev. - - Additional changes are licensed under the same terms as NGINX and - copyright Joyent, Inc. and other Node contributors. All rights reserved. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to - deal in the Software without restriction, including without limitation the - rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - IN THE SOFTWARE. - """ - -- Closure Linter is located at tools/closure_linter. Closure's license - follows: - """ - # Copyright (c) 2007, Google Inc. - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, are permitted provided that the following conditions are - # met: - # - # * Redistributions of source code must retain the above copyright - # notice, this list of conditions and the following disclaimer. - # * Redistributions in binary form must reproduce the above - # copyright notice, this list of conditions and the following disclaimer - # in the documentation and/or other materials provided with the - # distribution. - # * Neither the name of Google Inc. nor the names of its - # contributors may be used to endorse or promote products derived from - # this software without specific prior written permission. - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """ - -- tools/cpplint.py is a C++ linter. Its license follows: - """ - # Copyright (c) 2009 Google Inc. All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, are permitted provided that the following conditions are - # met: - # - # * Redistributions of source code must retain the above copyright - # notice, this list of conditions and the following disclaimer. - # * Redistributions in binary form must reproduce the above - # copyright notice, this list of conditions and the following disclaimer - # in the documentation and/or other materials provided with the - # distribution. - # * Neither the name of Google Inc. nor the names of its - # contributors may be used to endorse or promote products derived from - # this software without specific prior written permission. - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """ - -- lib/punycode.js is copyright 2011 Mathias Bynens - and released under the MIT license. - """ - * Punycode.js - * Copyright 2011 Mathias Bynens - * Available under MIT license - """ - -- tools/gyp. GYP is a meta-build system. GYP's license follows: - """ - Copyright (c) 2009 Google Inc. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """ - -- Zlib at deps/zlib. zlib's license follows: - """ - /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.8, April 28th, 2013 - - Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - */ - """ - -- npm is a package manager program located at deps/npm. - npm's license follows: - """ - Copyright (c) Isaac Z. Schlueter - All rights reserved. - - npm is released under the Artistic 2.0 License. - The text of the License follows: - - - -------- - - - The Artistic License 2.0 - - Copyright (c) 2000-2006, The Perl Foundation. - - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - This license establishes the terms under which a given free software - Package may be copied, modified, distributed, and/or redistributed. - The intent is that the Copyright Holder maintains some artistic - control over the development of that Package while still keeping the - Package available as open source and free software. - - You are always permitted to make arrangements wholly outside of this - license directly with the Copyright Holder of a given Package. If the - terms of this license do not permit the full use that you propose to - make of the Package, you should contact the Copyright Holder and seek - a different licensing arrangement. - - Definitions - - "Copyright Holder" means the individual(s) or organization(s) - named in the copyright notice for the entire Package. - - "Contributor" means any party that has contributed code or other - material to the Package, in accordance with the Copyright Holder's - procedures. - - "You" and "your" means any person who would like to copy, - distribute, or modify the Package. - - "Package" means the collection of files distributed by the - Copyright Holder, and derivatives of that collection and/or of - those files. A given Package may consist of either the Standard - Version, or a Modified Version. - - "Distribute" means providing a copy of the Package or making it - accessible to anyone else, or in the case of a company or - organization, to others outside of your company or organization. - - "Distributor Fee" means any fee that you charge for Distributing - this Package or providing support for this Package to another - party. It does not mean licensing fees. - - "Standard Version" refers to the Package if it has not been - modified, or has been modified only in ways explicitly requested - by the Copyright Holder. - - "Modified Version" means the Package, if it has been changed, and - such changes were not explicitly requested by the Copyright - Holder. - - "Original License" means this Artistic License as Distributed with - the Standard Version of the Package, in its current version or as - it may be modified by The Perl Foundation in the future. - - "Source" form means the source code, documentation source, and - configuration files for the Package. - - "Compiled" form means the compiled bytecode, object code, binary, - or any other form resulting from mechanical transformation or - translation of the Source form. - - - Permission for Use and Modification Without Distribution - - (1) You are permitted to use the Standard Version and create and use - Modified Versions for any purpose without restriction, provided that - you do not Distribute the Modified Version. - - - Permissions for Redistribution of the Standard Version - - (2) You may Distribute verbatim copies of the Source form of the - Standard Version of this Package in any medium without restriction, - either gratis or for a Distributor Fee, provided that you duplicate - all of the original copyright notices and associated disclaimers. At - your discretion, such verbatim copies may or may not include a - Compiled form of the Package. - - (3) You may apply any bug fixes, portability changes, and other - modifications made available from the Copyright Holder. The resulting - Package will still be considered the Standard Version, and as such - will be subject to the Original License. - - - Distribution of Modified Versions of the Package as Source - - (4) You may Distribute your Modified Version as Source (either gratis - or for a Distributor Fee, and with or without a Compiled form of the - Modified Version) provided that you clearly document how it differs - from the Standard Version, including, but not limited to, documenting - any non-standard features, executables, or modules, and provided that - you do at least ONE of the following: - - (a) make the Modified Version available to the Copyright Holder - of the Standard Version, under the Original License, so that the - Copyright Holder may include your modifications in the Standard - Version. - - (b) ensure that installation of your Modified Version does not - prevent the user installing or running the Standard Version. In - addition, the Modified Version must bear a name that is different - from the name of the Standard Version. - - (c) allow anyone who receives a copy of the Modified Version to - make the Source form of the Modified Version available to others - under - - (i) the Original License or - - (ii) a license that permits the licensee to freely copy, - modify and redistribute the Modified Version using the same - licensing terms that apply to the copy that the licensee - received, and requires that the Source form of the Modified - Version, and of any works derived from it, be made freely - available in that license fees are prohibited but Distributor - Fees are allowed. - - - Distribution of Compiled Forms of the Standard Version - or Modified Versions without the Source - - (5) You may Distribute Compiled forms of the Standard Version without - the Source, provided that you include complete instructions on how to - get the Source of the Standard Version. Such instructions must be - valid at the time of your distribution. If these instructions, at any - time while you are carrying out such distribution, become invalid, you - must provide new instructions on demand or cease further distribution. - If you provide valid instructions or cease distribution within thirty - days after you become aware that the instructions are invalid, then - you do not forfeit any of your rights under this license. - - (6) You may Distribute a Modified Version in Compiled form without - the Source, provided that you comply with Section 4 with respect to - the Source of the Modified Version. - - - Aggregating or Linking the Package - - (7) You may aggregate the Package (either the Standard Version or - Modified Version) with other packages and Distribute the resulting - aggregation provided that you do not charge a licensing fee for the - Package. Distributor Fees are permitted, and licensing fees for other - components in the aggregation are permitted. The terms of this license - apply to the use and Distribution of the Standard or Modified Versions - as included in the aggregation. - - (8) You are permitted to link Modified and Standard Versions with - other works, to embed the Package in a larger work of your own, or to - build stand-alone binary or bytecode versions of applications that - include the Package, and Distribute the result without restriction, - provided the result does not expose a direct interface to the Package. - - - Items That are Not Considered Part of a Modified Version - - (9) Works (including, but not limited to, modules and scripts) that - merely extend or make use of the Package, do not, by themselves, cause - the Package to be a Modified Version. In addition, such works are not - considered parts of the Package itself, and are not subject to the - terms of this license. - - - General Provisions - - (10) Any use, modification, and distribution of the Standard or - Modified Versions is governed by this Artistic License. By using, - modifying or distributing the Package, you accept this license. Do not - use, modify, or distribute the Package, if you do not accept this - license. - - (11) If your Modified Version has been derived from a Modified - Version made by someone other than you, you are nevertheless required - to ensure that your Modified Version complies with the requirements of - this license. - - (12) This license does not grant you the right to use any trademark, - service mark, tradename, or logo of the Copyright Holder. - - (13) This license includes the non-exclusive, worldwide, - free-of-charge patent license to make, have made, use, offer to sell, - sell, import and otherwise transfer the Package with respect to any - patent claims licensable by the Copyright Holder that are necessarily - infringed by the Package. If you institute patent litigation - (including a cross-claim or counterclaim) against any party alleging - that the Package constitutes direct or contributory patent - infringement, then this Artistic License to you shall terminate on the - date that such litigation is filed. - - (14) Disclaimer of Warranty: - THE PACKAGE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS - IS' AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES. THE IMPLIED - WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR - NON-INFRINGEMENT ARE DISCLAIMED TO THE EXTENT PERMITTED BY YOUR LOCAL - LAW. UNLESS REQUIRED BY LAW, NO COPYRIGHT HOLDER OR CONTRIBUTOR WILL - BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL - DAMAGES ARISING IN ANY WAY OUT OF THE USE OF THE PACKAGE, EVEN IF - ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - - -------- - - - "Node.js" and "node" trademark Joyent, Inc. npm is not officially - part of the Node.js project, and is neither owned by nor - officially affiliated with Joyent, Inc. - - Packages published in the npm registry (other than the Software and - its included dependencies) are not part of npm itself, are the sole - property of their respective maintainers, and are not covered by - this license. - - "npm Logo" created by Mathias Pettersson and Brian Hammond, - used with permission. - - "Gubblebum Blocky" font - Copyright (c) by Tjarda Koster, http://jelloween.deviantart.com - included for use in the npm website and documentation, - used with permission. - - This program uses several Node modules contained in the node_modules/ - subdirectory, according to the terms of their respective licenses. - """ - -- tools/doc/node_modules/marked. Marked is a Markdown parser. Marked's - license follows: - """ - Copyright (c) 2011-2012, Christopher Jeffrey (https://github.com/chjj/) - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. - """ - -- test/gc/node_modules/weak. Node-weak is a node.js addon that provides garbage - collector notifications. Node-weak's license follows: - """ - Copyright (c) 2011, Ben Noordhuis - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - """ - -- wrk is located at tools/wrk. wrk's license follows: - """ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - """ - -- ICU's license follows: - From http://source.icu-project.org/repos/icu/icu/trunk/license.html - """ - ICU License - ICU 1.8.1 and later - - COPYRIGHT AND PERMISSION NOTICE - - Copyright (c) 1995-2014 International Business Machines Corporation and others - - All rights reserved. - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, copy, - modify, merge, publish, distribute, and/or sell copies of the - Software, and to permit persons to whom the Software is furnished - to do so, provided that the above copyright notice(s) and this - permission notice appear in all copies of the Software and that - both the above copyright notice(s) and this permission notice - appear in supporting documentation. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE - COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR - ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR - ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR - PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER - TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - PERFORMANCE OF THIS SOFTWARE. - - Except as contained in this notice, the name of a copyright holder - shall not be used in advertising or otherwise to promote the sale, - use or other dealings in this Software without prior written - authorization of the copyright holder. - - All trademarks and registered trademarks mentioned herein are the - property of their respective owners. - - Third-Party Software Licenses - - This section contains third-party software notices and/or - additional terms for licensed third-party software components - included within ICU libraries. - - 1. Unicode Data Files and Software - COPYRIGHT AND PERMISSION NOTICE - - Copyright © 1991-2014 Unicode, Inc. All rights reserved. - Distributed under the Terms of Use in - http://www.unicode.org/copyright.html. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of the Unicode data files and any associated documentation - (the "Data Files") or Unicode software and any associated documentation - (the "Software") to deal in the Data Files or Software - without restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, and/or sell copies of - the Data Files or Software, and to permit persons to whom the Data Files - or Software are furnished to do so, provided that - (a) this copyright and permission notice appear with all copies - of the Data Files or Software, - (b) this copyright and permission notice appear in associated - documentation, and - (c) there is clear notice in each modified Data File or in the Software - as well as in the documentation associated with the Data File(s) or - Software that the data or software has been modified. - - THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF - ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT OF THIRD PARTY RIGHTS. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS - NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL - DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, - DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER - TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - PERFORMANCE OF THE DATA FILES OR SOFTWARE. - - Except as contained in this notice, the name of a copyright holder - shall not be used in advertising or otherwise to promote the sale, - use or other dealings in these Data Files or Software without prior - written authorization of the copyright holder. - - 2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) - # The Google Chrome software developed by Google is licensed - # under the BSD license. Other software included in this distribution - # is provided under other licenses, as set forth below. - # - # The BSD License - # http://opensource.org/licenses/bsd-license.php - # Copyright (C) 2006-2008, Google Inc. - # - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or - # without modification, are permitted provided that the following - # conditions are met: - # - # Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - # Redistributions in binary form must reproduce the above - # copyright notice, this list of conditions and the following - # disclaimer in the documentation and/or other materials provided with - # the distribution. - # Neither the name of Google Inc. nor the names of its - # contributors may be used to endorse or promote products derived from - # this software without specific prior written permission. - # - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - # - # - # The word list in cjdict.txt are generated by combining three - word lists listed - # below with further processing for compound word breaking. The - frequency is generated - # with an iterative training against Google web corpora. - # - # * Libtabe (Chinese) - # - https://sourceforge.net/project/?group_id=1519 - # - Its license terms and conditions are shown below. - # - # * IPADIC (Japanese) - # - http://chasen.aist-nara.ac.jp/chasen/distribution.html - # - Its license terms and conditions are shown below. - # - # ---------COPYING.libtabe ---- BEGIN-------------------- - # - # /* - # * Copyrighy (c) 1999 TaBE Project. - # * Copyright (c) 1999 Pai-Hsiang Hsiao. - # * All rights reserved. - # * - # * Redistribution and use in source and binary forms, with or without - # * modification, are permitted provided that the following conditions - # * are met: - # * - # * . Redistributions of source code must retain the above copyright - # * notice, this list of conditions and the following disclaimer. - # * . Redistributions in binary form must reproduce the above copyright - # * notice, this list of conditions and the following disclaimer in - # * the documentation and/or other materials provided with the - # * distribution. - # * . Neither the name of the TaBE Project nor the names of its - # * contributors may be used to endorse or promote products derived - # * from this software without specific prior written permission. - # * - # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - # * OF THE POSSIBILITY OF SUCH DAMAGE. - # */ - # - # /* - # * Copyright (c) 1999 Computer Systems and Communication Lab, - # * Institute of Information Science, Academia Sinica. - # * All rights reserved. - # * - # * Redistribution and use in source and binary forms, with or without - # * modification, are permitted provided that the following conditions - # * are met: - # * - # * . Redistributions of source code must retain the above copyright - # * notice, this list of conditions and the following disclaimer. - # * . Redistributions in binary form must reproduce the above copyright - # * notice, this list of conditions and the following disclaimer in - # * the documentation and/or other materials provided with the - # * distribution. - # * . Neither the name of the Computer Systems and Communication Lab - # * nor the names of its contributors may be used to endorse or - # * promote products derived from this software without specific - # * prior written permission. - # * - # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - # * OF THE POSSIBILITY OF SUCH DAMAGE. - # */ - # - # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, University of Illinois - # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 - # - # ---------------COPYING.libtabe-----END------------------------------------ - # - # - # ---------------COPYING.ipadic-----BEGIN------------------------------------ - # - # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science - # and Technology. All Rights Reserved. - # - # Use, reproduction, and distribution of this software is permitted. - # Any copy of this software, whether in its original form or modified, - # must include both the above copyright notice and the following - # paragraphs. - # - # Nara Institute of Science and Technology (NAIST), - # the copyright holders, disclaims all warranties with regard to this - # software, including all implied warranties of merchantability and - # fitness, in no event shall NAIST be liable for - # any special, indirect or consequential damages or any damages - # whatsoever resulting from loss of use, data or profits, whether in an - # action of contract, negligence or other tortuous action, arising out - # of or in connection with the use or performance of this software. - # - # A large portion of the dictionary entries - # originate from ICOT Free Software. The following conditions for ICOT - # Free Software applies to the current dictionary as well. - # - # Each User may also freely distribute the Program, whether in its - # original form or modified, to any third party or parties, PROVIDED - # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear - # on, or be attached to, the Program, which is distributed substantially - # in the same form as set out herein and that such intended - # distribution, if actually made, will neither violate or otherwise - # contravene any of the laws and regulations of the countries having - # jurisdiction over the User or the intended distribution itself. - # - # NO WARRANTY - # - # The program was produced on an experimental basis in the course of the - # research and development conducted during the project and is provided - # to users as so produced on an experimental basis. Accordingly, the - # program is provided without any warranty whatsoever, whether express, - # implied, statutory or otherwise. The term "warranty" used herein - # includes, but is not limited to, any warranty of the quality, - # performance, merchantability and fitness for a particular purpose of - # the program and the nonexistence of any infringement or violation of - # any right of any third party. - # - # Each user of the program will agree and understand, and be deemed to - # have agreed and understood, that there is no warranty whatsoever for - # the program and, accordingly, the entire risk arising from or - # otherwise connected with the program is assumed by the user. - # - # Therefore, neither ICOT, the copyright holder, or any other - # organization that participated in or was otherwise related to the - # development of the program and their respective officials, directors, - # officers and other employees shall be held liable for any and all - # damages, including, without limitation, general, special, incidental - # and consequential damages, arising out of or otherwise in connection - # with the use or inability to use the program or any product, material - # or result produced or otherwise obtained by using the program, - # regardless of whether they have been advised of, or otherwise had - # knowledge of, the possibility of such damages at any time during the - # project or thereafter. Each user will be deemed to have agreed to the - # foregoing by his or her commencement of use of the program. The term - # "use" as used herein includes, but is not limited to, the use, - # modification, copying and distribution of the program and the - # production of secondary products from the program. - # - # In the case where the program, whether in its original form or - # modified, was distributed or delivered to or received by a user from - # any person, organization or entity other than ICOT, unless it makes or - # grants independently of ICOT any specific warranty to the user in - # writing, such person, organization or entity, will also be exempted - # from and not be held liable to the user for any such damages as noted - # above as far as the program is concerned. - # - # ---------------COPYING.ipadic-----END------------------------------------ - -3. Lao Word Break Dictionary Data (laodict.txt) - # Copyright (c) 2013 International Business Machines Corporation - # and others. All Rights Reserved. - # - # Project: http://code.google.com/p/lao-dictionary/ - # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt - # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt - # (copied below) - # - # This file is derived from the above dictionary, with slight modifications. - # -------------------------------------------------------------------------------- - # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without modification, - # are permitted provided that the following conditions are met: - # - # Redistributions of source code must retain the above copyright notice, this - # list of conditions and the following disclaimer. Redistributions in binary - # form must reproduce the above copyright notice, this list of conditions and - # the following disclaimer in the documentation and/or other materials - # provided with the distribution. - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - # -------------------------------------------------------------------------------- - - 4. Burmese Word Break Dictionary Data (burmesedict.txt) - # Copyright (c) 2014 International Business Machines Corporation - # and others. All Rights Reserved. - # - # This list is part of a project hosted at: - # github.com/kanyawtech/myanmar-karen-word-lists - # - # -------------------------------------------------------------------------------- - # Copyright (c) 2013, LeRoy Benjamin Sharon - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without modification, - # are permitted provided that the following conditions are met: - # - # Redistributions of source code must retain the above copyright notice, this - # list of conditions and the following disclaimer. - # - # Redistributions in binary form must reproduce the above copyright notice, this - # list of conditions and the following disclaimer in the documentation and/or - # other materials provided with the distribution. - # - # Neither the name Myanmar Karen Word Lists, nor the names of its - # contributors may be used to endorse or promote products derived from - # this software without specific prior written permission. - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - # -------------------------------------------------------------------------------- - - 5. Time Zone Database - ICU uses the public domain data and code derived from Time Zone - Database for its time zone support. The ownership of the TZ - database is explained in BCP 175: Procedure for Maintaining the - Time Zone Database section 7. - - 7. Database Ownership - - The TZ database itself is not an IETF Contribution or an IETF - document. Rather it is a pre-existing and regularly updated work - that is in the public domain, and is intended to remain in the public - domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do not apply - to the TZ Database or contributions that individuals make to it. - Should any claims be made and substantiated against the TZ Database, - the organization that is providing the IANA Considerations defined in - this RFC, under the memorandum of understanding with the IETF, - currently ICANN, may act in accordance with all competent court - orders. No ownership claims will be made by ICANN or the IETF Trust - on the database or the code. Any person making a contribution to the - database or code waives all rights to future claims in that - contribution or in the TZ Database. - """ diff --git a/cflinuxfs4/NOTICE b/cflinuxfs4/NOTICE deleted file mode 100644 index 97fe3423..00000000 --- a/cflinuxfs4/NOTICE +++ /dev/null @@ -1,15 +0,0 @@ -binary-builder - -Copyright (c) 2013-Present CloudFoundry.org Foundation, Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/cflinuxfs4/PHP-Geoip.md b/cflinuxfs4/PHP-Geoip.md deleted file mode 100644 index fc480305..00000000 --- a/cflinuxfs4/PHP-Geoip.md +++ /dev/null @@ -1,17 +0,0 @@ -# PHP & GeoIP Support - -The binary builder has support for building the `geoip` extension for PHP 5.5, 5.6 and 7.0. In order for the `geoip` extension to function properly though it requires a database of geoip data. The company MaxMind provides both commercial and less accurate open source versions of these databases. - -In the default mode, binary-builder will not bundle any database files with PHP, however it will bundle a script that can be run to download an up-to-date version of the open source (also called "lite") databases. - -If you would like binary-builder to download the open source version of the databases and bundle them with PHP, it can do that too. To instruct binary-builder to do that, create a file called `BUNDLE_GEOIP_LITE` in the top level of the project and set the contents of the file to `true`. When binary-builder sees that this file exists and that the contents are `true` it will download and bundle the open source version of the databases with the resulting PHP binary. - -Right now binary-builder can't be configured to download and bundle commercial versions of the geoip data from MaxMind. You can download commercial versions via the included script (see below), but there is no option exposed to configure the script through binary-builder. - -## What's Included - -When you build PHP binaries with GeoIP support you get the following included with the PHP binary: - -1. The `geoip` extension (from PECL) -2. The library file `geoipdb/lib/geoip_downloader.rb` and script `geoipdb/bin/download_geoip_db.rb` which can be run at a later date to download a copy of the geoip databases from MaxMind. By default they will download the "lite" or open source versions but the script can be configured to use a user id & license key to download paid versions as well. -3. If bundled, the geoip databases will be under `geoipdb/dbs`. If not bundled, that directory will be empty. If bundled, the following databases will be included: `GeoLiteCityv6.dat`, `GeoLiteASNum.dat`, `GeoLiteCountry.dat`, `GeoIPv6.dat`, and `GeoLiteCity.dat`. diff --git a/cflinuxfs4/PHP-Oracle.md b/cflinuxfs4/PHP-Oracle.md deleted file mode 100644 index c9615722..00000000 --- a/cflinuxfs4/PHP-Oracle.md +++ /dev/null @@ -1,57 +0,0 @@ -# PHP & Oracle Support - -The binary builder has support for building the `oci8` and `pdo_oci` extensions of PHP 5.5, 5.6 and 7.0. While the builder is capable of building them, it does *not* provide the Oracle libraries and SDK which are required to build these extensions. These are not provided by binary-builder because of licensing restrictions and binary builder does not automatically download them because they are behind an Oracle pay-wall. To build the extensions, you *must* download and provide these to the builder. - -## What to Downwload - -The requirements to build these extensions are listed here. - - http://php.net/manual/en/oci8.requirements.php - -This basically boils down to installing the Oracle Instant Client Basic or Basic Lite plus the SDK. Use the ZIP installs and extract them to a location on your local machine. Then create the following symbolic library before building: `ln -s libclntsh.so.12.1 libclntsh.so` (version number must vary). - -You only need to do this once. - -## How to Build - -To build, you just [follow the normal instructions for building PHP with binary builder & Docker](https://github.com/cloudfoundry/binary-builder/blob/master/README.md). The only exception is that you need to map the path where you extracted the Oracle instant client and SDK to `/oracle` in the docker container used by binary builder. - -This is done by adding an additiona `-v` argument to the `docker run` command. - -Ex: - -``` -docker run -w /binary-builder -v `pwd`:/binary-builder -v /path/to/oracle:/oracle -it cloudfoundry/cflinuxfs3 bash -export STACK=cflinuxfs3 -./bin/binary-builder --name=php --version=7.1.29 --md5=ae625e0cfcfdacea3e7a70a075e47155 --php-extensions-file=./php71-extensions.yml -``` - -## What's Included - -When you build PHP binaries with Oracle support you get the following included with the PHP binary: - -1. The `oci8` extension (from PECL) - - PHP 5.5 & 5.6 include oci8 2.0.x - - PHP 7.0 includes oci 2.1.x -2. The `pdo_oci` extension bundled with PHP -3. The following libraries which are required by the extensions are include in `php/lib` - - libclntshcore.so - - libclntsh.so - - libipc1.so - - libmql1.so - - libnnz12.so - - libociicus.so - - libons.so - -Two notes on the included libraries: - -1. The file `libociicus.so` is a US English specific library from Instant Client lite. If you need multi-language support, you will need to install the full Instant Client and likely include additional libraries. That's not supported at this time, but patches are welcome. - -2. The PHP bundle that is built by binary builder has Oracle libraries from the Instant Client packaged with it. As such, you should not publicly distribute these libraries unless you are licensed to do so by Oracle. - -## Disabling Oracle Support - -By default Oracle Support is *not* included. The binary builder will not and cannot build these extensions unless you provide it with the Oracle libraries as listed in the *What to Download* section above. - -If you want to build with and without Oracle support, you can control if binary-builder will include the Oracle support by adding or removing the volume map for `/oracle` on your `docker run` command. If that volume is mounted then binary builder will attempt to build Oracle support. If it's not mounted, Oracle support is disabled. - diff --git a/cflinuxfs4/README.md b/cflinuxfs4/README.md deleted file mode 100644 index c458a556..00000000 --- a/cflinuxfs4/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Introduction - -This tool provides a mechanism for building binaries for the Cloud Foundry buildpacks. - -## Currently supported binaries - -* NodeJS -* Ruby -* JRuby -* Python -* PHP -* Nginx -* Apache HTTPD Server -* Go -* Glide -* Godep -* Bundler - -# Usage - -The scripts are meant to be run as root on a Cloud Foundry [stack](https://docs.cloudfoundry.org/concepts/stacks.html). - -## Running within Docker - -To run `binary-builder` from within the cflinuxfs3 rootfs, use [Docker](https://docker.io): - -```bash -docker run -w /binary-builder -v `pwd`:/binary-builder -it cloudfoundry/cflinuxfs3 bash -export STACK=cflinuxfs3 -./bin/binary-builder --name=[binary_name] --version=[binary_version] --(md5|sha256)=[checksum_value] -``` - -This generates a gzipped tarball in the binary-builder directory with the filename format `binary_name-binary_version-linux-x64`. - -For example, if you were building ruby 2.2.3, you'd run the following commands: - -```bash -$ docker run -w /binary-builder -v `pwd`:/binary-builder -it cloudfoundry/cflinuxfs3:ruby-2.2.4 ./bin/binary-builder --name=ruby --version=2.2.3 --md5=150a5efc5f5d8a8011f30aa2594a7654 -$ ls -ruby-2.2.3-linux-x64.tgz -``` - -# Building PHP - -To build PHP, you also need to pass in a YAML file containing information about the various PHP extensions to be built. For example - -```bash -docker run -w /binary-builder -v `pwd`:/binary-builder -it cloudfoundry/cflinuxfs3 bash -export STACK=cflinuxfs3 -./bin/binary-builder --name=php7 --version=7.3.14 --sha256=6aff532a380b0f30c9e295b67dc91d023fee3b0ae14b4771468bf5dda4cbf108 --php-extensions-file=./php7-extensions.yml -``` - -For an example of what this file looks like, see: [php7-base-extensions.yml](https://github.com/cloudfoundry/buildpacks-ci/tree/master/tasks/build-binary-new) and the various `php*-extensions-patch.yml` files in that same directory. Patch files adjust the base-extensions.yml file by adding/removing extensions. The `--php-extensions-file` argument will need the base-extensions file with one of the patch files applied. That normally happens automatically through the pipeline, so if you are building manually you need to manually create this file. - -**TIP** If you are updating or building a specific PHP extension, remove everything except that specific extension from your `--php-extensions-file` file. This will decrease the build times & make it easier for you to test your changes. - -# Building nginx - -Nginx uses GPG keys to verify the source tarball, so you'll need something like the following code to build the NGinx binary: - -``` -version=1.15.9 -gpg_signature_url="http://nginx.org/download/nginx-${version}.tar.gz.asc" -gpg_signature=`curl -sL ${gpg_signature_url}` - -docker run -w /binary-builder -v `pwd`:/binary-builder \ - -it cloudfoundry/cflinuxfs3 ./bin/binary-builder \ - --name=nginx-static --gpg-rsa-key-id=A1C052F8 \ - --version=$version --gpg-signature="${gpg_signature}" -``` - -# Contributing - -Find our guidelines [here](./CONTRIBUTING.md). - -# Reporting Issues - -Open an issue on this project - -# Active Development - -The project backlog is on [Pivotal Tracker](https://www.pivotaltracker.com/projects/1042066) - -# Running the tests - -The integration test suite includes specs that test the functionality for building [PHP with Oracle client libraries](./PHP-Oracle.md). These tests are tagged `:run_oracle_php_tests` and require access to an S3 bucket containing the Oracle client libraries. This is configured using the environment variables `AWS_ACCESS_KEY` and `AWS_SECRET_ACCESS_KEY` - -Optionally provide `AWS_ASSUME_ROLE_ARN` to assume a role. - -If you do not need to test this functionality, exclude the tag `:run_oracle_php_tests` when you run `rspec`. diff --git a/cflinuxfs4/bin/binary-builder b/cflinuxfs4/bin/binary-builder deleted file mode 100755 index 09b438c9..00000000 --- a/cflinuxfs4/bin/binary-builder +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -# workaround to prevent tons of deprecation warnings from spamming the screen -# the deprecations are a result of Debian/Ubuntu packaging issues we cannot control -export DEBIAN_DISABLE_RUBYGEMS_INTEGRATION=foo - -gem update --system --no-document -q --silent > /dev/null -gem install bundler --no-document -q --silent > /dev/null -bundle config mirror.https://rubygems.org ${RUBYGEM_MIRROR} -bundle config set --local without 'local' -bundle install -bundle exec ./bin/binary-builder.rb "$@" diff --git a/cflinuxfs4/bin/binary-builder.rb b/cflinuxfs4/bin/binary-builder.rb deleted file mode 100755 index 47716c1a..00000000 --- a/cflinuxfs4/bin/binary-builder.rb +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'bundler' -require 'optparse' -require_relative '../lib/yaml_presenter' -require_relative '../lib/archive_recipe' -Dir['recipe/*.rb'].each { |f| require File.expand_path(f) } - -recipes = { - 'ruby' => RubyRecipe, - 'bundler' => BundlerRecipe, - 'node' => NodeRecipe, - 'jruby' => JRubyMeal, - 'httpd' => HTTPdMeal, - 'python' => PythonRecipe, - 'php' => PhpMeal, - 'nginx-static' => NginxRecipe, - 'godep' => GoDepMeal, - 'glide' => GlideRecipe, - 'go' => GoRecipe, - 'dep' => DepRecipe, - 'hwc' => HwcRecipe -} - -options = {} -option_parser = OptionParser.new do |opts| - opts.banner = 'USAGE: binary-builder [options] (A checksum method is required)' - - opts.on('-nNAME', '--name=NAME', "Name of the binary. Options: [#{recipes.keys.join(', ')}]") do |n| - options[:name] = n - end - opts.on('-vVERSION', '--version=VERSION', 'Version of the binary e.g. 1.7.11') do |n| - options[:version] = n - end - opts.on('--sha256=SHA256', 'SHA256 of the binary ') do |n| - options[:sha256] = n - end - opts.on('--md5=MD5', 'MD5 of the binary ') do |n| - options[:md5] = n - end - opts.on('--gpg-rsa-key-id=RSA_KEY_ID', 'RSA Key Id e.g. 10FDE075') do |n| - options[:gpg] ||= {} - options[:gpg][:key] = n - end - opts.on('--gpg-signature=ASC_KEY', 'content of the .asc file') do |n| - options[:gpg] ||= {} - options[:gpg][:signature] = n - end - opts.on('--git-commit-sha=SHA', 'git commit sha of the specified version') do |n| - options[:git] ||= {} - options[:git][:commit_sha] = n - end - opts.on('--php-extensions-file=FILE', 'yaml file containing PHP extensions + versions') do |n| - options[:php_extensions_file] = n - end -end -option_parser.parse! - -unless options[:name] && options[:version] && ( - options[:sha256] || - options[:md5] || - (options.key?(:git) && options[:git][:commit_sha]) || - (options[:gpg][:signature] && options[:gpg][:key]) -) - raise option_parser.help -end - -raise "Unsupported recipe [#{options[:name]}], supported options are [#{recipes.keys.join(', ')}]" unless recipes.key?(options[:name]) - -recipe_options = DetermineChecksum.new(options).to_h - -recipe_options[:php_extensions_file] = options[:php_extensions_file] if options[:php_extensions_file] -recipe = recipes[options[:name]].new( - options[:name], - options[:version], - recipe_options -) -Bundler.with_unbundled_env do - puts "Source URL: #{recipe.url}" - - recipe.cook - ArchiveRecipe.new(recipe).compress! - - puts 'Source YAML:' - puts YAMLPresenter.new(recipe).to_yaml -end diff --git a/cflinuxfs4/bin/download_geoip_db.rb b/cflinuxfs4/bin/download_geoip_db.rb deleted file mode 100755 index 48f46fbe..00000000 --- a/cflinuxfs4/bin/download_geoip_db.rb +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'net/http' -require 'uri' -require 'digest' -require 'tempfile' -require 'optparse' -require_relative '../lib/geoip_downloader' - -options = {} -option_parser = OptionParser.new do |opts| - opts.banner = 'USAGE: download_geoip_db [options]' - - opts.on('-uUSER', '--user=USER', 'User Id from MaxMind. Default "999999".') do |n| - options[:user] = n - end - - opts.on('-lLICENSE', '--license=LICENSE', 'License from MaxMind. Default "000000000000".') do |n| - options[:license] = n - end - - opts.on('-oOUTPUTDIR', '--output_dir=OUTPUTDIR', 'Directory where databases might exist and will be written / updated. Default "."') do |n| - options[:output_dir] = n - end - - opts.on('-pPRODUCTS', '--products=PRODUCTS', 'Space separated list of product ids. Default "GeoLite-Legacy-IPv6-City GeoLite-Legacy-IPv6-Country 506 517 533".') do |n| - options[:products] = n - end -end -option_parser.parse! - -options[:user] ||= MaxMindGeoIpUpdater.FREE_USER -options[:license] ||= MaxMindGeoIpUpdater.FREE_LICENSE -options[:output_dir] ||= '.' -options[:products] ||= 'GeoLite-Legacy-IPv6-City GeoLite-Legacy-IPv6-Country 506 517 533' - -updater = MaxMindGeoIpUpdater.new(options[:user], options[:license], options[:output_dir]) - -options[:products].split(' ').each do |product| - updater.download_product(product) -end diff --git a/cflinuxfs4/go-version.yml b/cflinuxfs4/go-version.yml deleted file mode 100644 index 8a3be269..00000000 --- a/cflinuxfs4/go-version.yml +++ /dev/null @@ -1,3 +0,0 @@ -go: - - version: 1.20.1 - sha256: 000a5b1fca4f75895f78befeb2eecf10bfff3c428597f3f1e69133b63b911b02 diff --git a/cflinuxfs4/lib/archive_recipe.rb b/cflinuxfs4/lib/archive_recipe.rb deleted file mode 100644 index 91725723..00000000 --- a/cflinuxfs4/lib/archive_recipe.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -require 'tmpdir' -require_relative 'yaml_presenter' - -class ArchiveRecipe - def initialize(recipe) - @recipe = recipe - end - - def compress! - return if @recipe.archive_files.empty? - - @recipe.setup_tar if @recipe.respond_to? :setup_tar - - Dir.mktmpdir do |dir| - archive_path = File.join(dir, @recipe.archive_path_name) - FileUtils.mkdir_p(archive_path) - - @recipe.archive_files.each do |glob| - `cp -r #{glob} #{archive_path}` - end - - File.write("#{dir}/sources.yml", YAMLPresenter.new(@recipe).to_yaml) - - print "Running 'archive' for #{@recipe.name} #{@recipe.version}... " - if @recipe.archive_filename.split('.').last == 'zip' - output_dir = Dir.pwd - - Dir.chdir(dir) do - `zip #{File.join(output_dir, @recipe.archive_filename)} -r .` - end - elsif @recipe.name == 'php' - `ls -A #{dir} | xargs tar --dereference -czf #{@recipe.archive_filename} -C #{dir}` - else - `ls -A #{dir} | xargs tar czf #{@recipe.archive_filename} -C #{dir}` - end - puts 'OK' - end - end -end diff --git a/cflinuxfs4/lib/geoip_downloader.rb b/cflinuxfs4/lib/geoip_downloader.rb deleted file mode 100644 index 2784dd1a..00000000 --- a/cflinuxfs4/lib/geoip_downloader.rb +++ /dev/null @@ -1,148 +0,0 @@ -# frozen_string_literal: true - -require 'net/http' -require 'uri' -require 'digest' -require 'tempfile' - -class MaxMindGeoIpUpdater - @@FREE_LICENSE = '000000000000' - @@FREE_USER = '999999' - - def initialize(user_id, license, output_dir) - @proto = 'http' - @host = 'updates.maxmind.com' - @user_id = user_id - @license = license - @output_dir = output_dir - @client_ip = nil - @challenge_digest = nil - end - - def self.FREE_LICENSE - @@FREE_LICENSE - end - - def self.FREE_USER - @@FREE_USER - end - - def get_filename(product_id) - uri = URI.parse("#{@proto}://#{@host}/app/update_getfilename") - uri.query = URI.encode_www_form({ product_id: product_id }) - resp = Net::HTTP.get_response(uri) - resp.body - end - - def client_ip - @client_ip ||= begin - uri = URI.parse("#{@proto}://#{@host}/app/update_getipaddr") - resp = Net::HTTP.get_response(uri) - resp.body - end - end - - def download_database(db_digest, challenge_digest, product_id, file_path) - uri = URI.parse("#{@proto}://#{@host}/app/update_secure") - uri.query = URI.encode_www_form({ - db_md5: db_digest, - challenge_md5: challenge_digest, - user_id: @user_id, - edition_id: product_id - }) - - Net::HTTP.start(uri.host, uri.port) do |http| - req = Net::HTTP::Get.new(uri.request_uri) - - http.request(req) do |resp| - file = Tempfile.new('geoip_db_download') - begin - if resp['content-type'] == 'text/plain; charset=utf-8' - puts "\tAlready up-to-date." - else - resp.read_body do |chunk| - file.write(chunk) - end - file.rewind - extract_file(file, file_path) - puts "\tDatabase updated." - end - ensure - file.close - file.unlink - end - end - end - end - - def download_free_database(product_id, file_path) - product_uris = { - 'GeoLite-Legacy-IPv6-City' => 'http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz', - 'GeoLite-Legacy-IPv6-Country' => 'http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz' - } - - if !product_uris.include?(product_id) - puts "\tProduct '#{product_id}' is not available under free license. Available products are: #{product_uris.keys.join(', ')}." - else - uri = URI.parse(product_uris[product_id]) - Net::HTTP.start(uri.host, uri.port) do |http| - req = Net::HTTP::Get.new(uri.request_uri) - - http.request(req) do |resp| - file = Tempfile.new('geoip_db_download') - begin - resp.read_body do |chunk| - file.write(chunk) - end - file.rewind - extract_file(file, file_path) - puts "\tDatabase updated." - ensure - file.close - file.unlink - end - end - end - end - end - - def extract_file(file, file_path) - gz = Zlib::GzipReader.new(file) - begin - File.open(file_path, 'w') do |out| - IO.copy_stream(gz, out) - end - ensure - gz.close - end - end - - def download_product(product_id) - puts 'Downloading...' - file_name = get_filename(product_id) - file_path = File.join(@output_dir, file_name) - db_digest = db_digest(file_path) - puts "\tproduct_id: #{product_id}" - puts "\tfile_name: #{file_name}" - puts "\tip: #{client_ip}" - puts "\tdb: #{db_digest}" - - if @license == @@FREE_LICENSE - # As of April 1, 2018, free legacy databases are no longer available through the GeoIP update - # API. Therefore, we'll fetch them from static URLs they've provided. - # This will NOT work using free GeoIP2 databases. - download_free_database(product_id, file_path) - else - puts "\tchallenge: #{challenge_digest}" - download_database(db_digest, challenge_digest, product_id, file_path) - end - end - - def db_digest(path) - File.exist?(path) ? Digest::MD5.file(path) : '00000000000000000000000000000000' - end - - def challenge_digest - Digest::MD5.hexdigest("#{@license}#{client_ip}") - end -end diff --git a/cflinuxfs4/lib/install_go.rb b/cflinuxfs4/lib/install_go.rb deleted file mode 100644 index 7ec34a1d..00000000 --- a/cflinuxfs4/lib/install_go.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true -require_relative 'utils' -require 'net/http' -require 'json' - -def install_go_compiler - url = URI('https://go.dev/dl/?mode=json') - res = Net::HTTP.get(url) - latest_go = JSON.parse(res).first - - go_version = latest_go['version'].delete_prefix('go') - go_sha256 = "" - - latest_go['files'].each do |file| - if file['filename'] == "go#{go_version}.linux-amd64.tar.gz" - go_sha256 = file['sha256'] - break - end - end - - Dir.chdir('/usr/local') do - go_download = "https://go.dev/dl/go#{go_version}.linux-amd64.tar.gz" - go_tar = 'go.tar.gz' - - HTTPHelper.download(go_download, go_tar, "sha256", go_sha256) - - system("tar xf #{go_tar}") - end -end diff --git a/cflinuxfs4/lib/openssl_replace.rb b/cflinuxfs4/lib/openssl_replace.rb deleted file mode 100644 index 51f18f0a..00000000 --- a/cflinuxfs4/lib/openssl_replace.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -require 'English' -require_relative '../recipe/base' - -class OpenSSLReplace - def self.run(*args) - system({ 'DEBIAN_FRONTEND' => 'noninteractive' }, *args) - raise "Could not run #{args}" unless $CHILD_STATUS.success? - end - - def self.replace_openssl - file_base = 'OpenSSL_1_1_0g' - file_name = "#{file_base}.tar.gz" - openssl_tar = "https://github.com/openssl/openssl/archive/#{file_name}" - - Dir.mktmpdir do |_dir| - run('wget', openssl_tar) - run('tar', 'xf', file_name) - Dir.chdir("openssl-#{file_base}") do - run('./config', - '--prefix=/usr', - '--libdir=/lib/x86_64-linux-gnu', - '--openssldir=/include/x86_64-linux-gnu/openssl') - run('make') - run('make', 'install') - end - end - end -end diff --git a/cflinuxfs4/lib/utils.rb b/cflinuxfs4/lib/utils.rb deleted file mode 100644 index cf38ce7f..00000000 --- a/cflinuxfs4/lib/utils.rb +++ /dev/null @@ -1,56 +0,0 @@ -require 'net/http' - -module HTTPHelper - class << self - def download_with_follow_redirects(uri) - uri = URI(uri) - Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |httpRequest| - response = httpRequest.request_get(uri) - if response.is_a?(Net::HTTPRedirection) - download_with_follow_redirects(response['location']) - else - response - end - end - end - - def download(uri, filename, digest_algorithm, sha) - response = download_with_follow_redirects(uri) - if response.code == '200' - Sha.verify_digest(response.body, digest_algorithm, sha) - File.write(filename, response.body) - else - str = "Failed to download #{uri} with code #{response.code} error: \n#{response.body}" - raise str - end - end - - def read_file(url) - uri = URI.parse(url) - response = Net::HTTP.get_response(uri) - response.body if response.code == '200' - end - end -end - -module Sha - class << self - def verify_digest(content, algorithm, expected_digest) - file_digest = get_digest(content, algorithm) - raise "sha256 verification failed: expected #{expected_digest}, got #{file_digest}" if expected_digest != file_digest - end - - def get_digest(content, algorithm) - case algorithm - when 'sha256' - Digest::SHA2.new(256).hexdigest(content) - when 'md5' - Digest::MD5.hexdigest(content) - when 'sha1' - Digest::SHA1.hexdigest(content) - else - raise 'Unknown digest algorithm' - end - end - end -end diff --git a/cflinuxfs4/lib/yaml_presenter.rb b/cflinuxfs4/lib/yaml_presenter.rb deleted file mode 100644 index ffb4fba5..00000000 --- a/cflinuxfs4/lib/yaml_presenter.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -require 'yaml' -require 'digest' - -class YAMLPresenter - def initialize(recipe) - @recipe = recipe - end - - def to_yaml - @recipe.send(:files_hashs).collect do |file| - if file.key?(:git) - { - 'url' => file[:url], - 'git_commit_sha' => file[:git][:commit_sha] - } - else - { - 'url' => file[:url], - 'sha256' => Digest::SHA256.file(file[:local_path]).hexdigest.force_encoding('UTF-8') - } - end - end.to_yaml - end -end diff --git a/cflinuxfs4/recipe/base.rb b/cflinuxfs4/recipe/base.rb deleted file mode 100644 index b1c73c17..00000000 --- a/cflinuxfs4/recipe/base.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -require 'mini_portile2' -require 'tmpdir' -require 'fileutils' -require_relative 'determine_checksum' -require_relative '../lib/yaml_presenter' - -class BaseRecipe < MiniPortile - def initialize(name, version, options = {}) - super name, version - - options.each do |key, value| - instance_variable_set("@#{key}", value) - end - - @files = [{ - url: url - }.merge(DetermineChecksum.new(options).to_h)] - end - - def configure_options - [] - end - - def compile - execute('compile', [make_cmd, '-j4']) - end - - def archive_filename - "#{name}-#{version}-linux-x64.tgz" - end - - def archive_files - [] - end - - def archive_path_name - '' - end - - private - - # NOTE: https://www.virtualbox.org/ticket/10085 - def tmp_path - "/tmp/#{@host}/ports/#{@name}/#{@version}" - end -end diff --git a/cflinuxfs4/recipe/bundler.rb b/cflinuxfs4/recipe/bundler.rb deleted file mode 100644 index e20d0b35..00000000 --- a/cflinuxfs4/recipe/bundler.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -require 'mini_portile2' -require_relative 'base' - -class BundlerRecipe < BaseRecipe - def url - "https://rubygems.org/downloads/bundler-#{version}.gem" - end - - def cook - download unless downloaded? - extract - compile - end - - def compile - current_dir = ENV['PWD'] - puts current_dir - Dir.mktmpdir("bundler-#{version}") do |tmpdir| - Dir.chdir(tmpdir) do |_dir| - FileUtils.rm_rf("#{tmpdir}/*") - - in_gem_env(tmpdir) do - system("unset RUBYOPT; gem install bundler --version #{version} --no-document --env-shebang") - replace_shebangs(tmpdir) - system("rm -f bundler-#{version}.gem") - system("rm -rf cache/bundler-#{version}.gem") - system("tar czvf #{current_dir}/#{archive_filename} .") - puts "#{current_dir}/#{archive_filename}" - end - end - end - end - - def archive_filename - "#{name}-#{version}.tgz" - end - - private - - def in_gem_env(gem_home) - old_gem_home = ENV['GEM_HOME'] - old_gem_path = ENV['GEM_PATH'] - ENV['GEM_HOME'] = ENV['GEM_PATH'] = gem_home.to_s - - yield - - ENV['GEM_HOME'] = old_gem_home - ENV['GEM_PATH'] = old_gem_path - end - - def replace_shebangs(dir) - Dir.glob("#{dir}/bin/*").each do |bin_script| - original_contents = File.read(bin_script) - new_contents = original_contents.gsub(/^#!.*ruby.*/, '#!/usr/bin/env ruby') - File.open(bin_script, 'w') { |file| file.puts(new_contents) } - end - end -end diff --git a/cflinuxfs4/recipe/dep.rb b/cflinuxfs4/recipe/dep.rb deleted file mode 100644 index 31b938a3..00000000 --- a/cflinuxfs4/recipe/dep.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require_relative 'base' - -class DepRecipe < BaseRecipe - attr_reader :name, :version - - def cook - download unless downloaded? - extract - - install_go_compiler - - FileUtils.rm_rf("#{tmp_path}/dep") - FileUtils.mv(Dir.glob("#{tmp_path}/dep-*").first, "#{tmp_path}/dep") - Dir.chdir("#{tmp_path}/dep") do - system( - { 'GOPATH' => "#{tmp_path}/dep/deps/_workspace:/tmp" }, - '/usr/local/go/bin/go get -asmflags -trimpath ./...' - ) or raise 'Could not install dep' - end - FileUtils.mv("#{tmp_path}/dep/LICENSE", '/tmp/LICENSE') - end - - def archive_files - %w[/tmp/bin/dep /tmp/LICENSE] - end - - def archive_path_name - 'bin' - end - - def url - "https://github.com/golang/dep/archive/#{version}.tar.gz" - end - - def go_recipe - @go_recipe ||= GoRecipe.new(@name, @version) - end - - def tmp_path - '/tmp/src/github.com/golang' - end -end diff --git a/cflinuxfs4/recipe/determine_checksum.rb b/cflinuxfs4/recipe/determine_checksum.rb deleted file mode 100644 index a7236437..00000000 --- a/cflinuxfs4/recipe/determine_checksum.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -class DetermineChecksum - def initialize(options) - @options = options - end - - def to_h - checksum_type = (%i[md5 sha256 gpg git] & @options.keys).first - { - checksum_type => @options[checksum_type] - } - end -end diff --git a/cflinuxfs4/recipe/glide.rb b/cflinuxfs4/recipe/glide.rb deleted file mode 100644 index 7e8c5048..00000000 --- a/cflinuxfs4/recipe/glide.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -require_relative 'base' - -class GlideRecipe < BaseRecipe - attr_reader :name, :version - - def cook - download unless downloaded? - extract - - install_go_compiler - - FileUtils.rm_rf("#{tmp_path}/glide") - FileUtils.mv(Dir.glob("#{tmp_path}/glide-*").first, "#{tmp_path}/glide") - Dir.chdir("#{tmp_path}/glide") do - system( - { 'GOPATH' => '/tmp', - 'PATH' => "#{ENV['PATH']}:/usr/local/go/bin" }, - '/usr/local/go/bin/go build' - ) or raise 'Could not install glide' - end - - FileUtils.mv("#{tmp_path}/glide/glide", '/tmp/glide') - FileUtils.mv("#{tmp_path}/glide/LICENSE", '/tmp/LICENSE') - end - - def archive_files - %w[/tmp/glide /tmp/LICENSE] - end - - def archive_path_name - 'bin' - end - - def url - "https://github.com/Masterminds/glide/archive/#{version}.tar.gz" - end - - def go_recipe - @go_recipe ||= GoRecipe.new(@name, @version) - end - - def tmp_path - '/tmp/src/github.com/Masterminds' - end -end diff --git a/cflinuxfs4/recipe/go.rb b/cflinuxfs4/recipe/go.rb deleted file mode 100644 index 492a8edb..00000000 --- a/cflinuxfs4/recipe/go.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require_relative 'base' -require_relative '../lib/utils' - -class GoRecipe < BaseRecipe - attr_reader :name, :version - - def cook - download unless downloaded? - extract - - # Installs go1.24.2 to $HOME/go1.24 - go124_sha256 = '68097bd680839cbc9d464a0edce4f7c333975e27a90246890e9f1078c7e702ad' - - Dir.chdir("#{ENV['HOME']}") do - go_download_uri = "https://go.dev/dl/go1.24.2.linux-amd64.tar.gz" - go_tar = "go.tar.gz" - HTTPHelper.download(go_download_uri, go_tar, "sha256", go124_sha256) - - system("tar xf #{go_tar}") - system("mv ./go ./go1.24") - end - - # The GOROOT_BOOTSTRAP defaults to $HOME/go1.4 so we need to update it for this command - Dir.chdir("#{tmp_path}/go/src") do - system( - 'GOROOT_BOOTSTRAP=$HOME/go1.24 ./make.bash' - ) or raise "Could not install go" - end - end - - def archive_files - ["#{tmp_path}/go/*"] - end - - def archive_path_name - 'go' - end - - def archive_filename - "#{name}#{version}.linux-amd64.tar.gz" - end - - def url - "https://go.dev/dl/go#{version}.src.tar.gz" - end - -end diff --git a/cflinuxfs4/recipe/godep.rb b/cflinuxfs4/recipe/godep.rb deleted file mode 100644 index cce6ceb6..00000000 --- a/cflinuxfs4/recipe/godep.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require_relative 'base' - -class GoDepMeal < BaseRecipe - attr_reader :name, :version - - def cook - download unless downloaded? - extract - - install_go_compiler - - FileUtils.rm_rf("#{tmp_path}/godep") - FileUtils.mv(Dir.glob("#{tmp_path}/godep-*").first, "#{tmp_path}/godep") - Dir.chdir("#{tmp_path}/godep") do - system( - { 'GOPATH' => "#{tmp_path}/godep/Godeps/_workspace:/tmp" }, - '/usr/local/go/bin/go get ./...' - ) or raise 'Could not install godep' - end - FileUtils.mv("#{tmp_path}/godep/License", '/tmp/License') - end - - def archive_files - %w[/tmp/bin/godep /tmp/License] - end - - def archive_path_name - 'bin' - end - - def url - "https://github.com/tools/godep/archive/#{version}.tar.gz" - end - - def go_recipe - @go_recipe ||= GoRecipe.new(@name, @version) - end - - def tmp_path - '/tmp/src/github.com/tools' - end -end diff --git a/cflinuxfs4/recipe/httpd_meal.rb b/cflinuxfs4/recipe/httpd_meal.rb deleted file mode 100644 index be2c73a2..00000000 --- a/cflinuxfs4/recipe/httpd_meal.rb +++ /dev/null @@ -1,221 +0,0 @@ -# frozen_string_literal: true - -require 'English' -require_relative 'base' - -class AprRecipe < BaseRecipe - def url - "https://archive.apache.org/dist/apr/apr-#{version}.tar.gz" - end -end - -class AprIconvRecipe < BaseRecipe - def configure_options - [ - "--with-apr=#{@apr_path}/bin/apr-1-config" - ] - end - - def url - "https://archive.apache.org/dist/apr/apr-iconv-#{version}.tar.gz" - end -end - -class AprUtilRecipe < BaseRecipe - def configure_options - [ - "--with-apr=#{@apr_path}", - "--with-iconv=#{@apr_iconv_path}", - '--with-crypto', - '--with-openssl', - '--with-mysql', - '--with-pgsql', - '--with-gdbm', - '--with-ldap' - ] - end - - def url - "https://archive.apache.org/dist/apr/apr-util-#{version}.tar.gz" - end -end - -class HTTPdRecipe < BaseRecipe - def computed_options - [ - '--prefix=/app/httpd', - "--with-apr=#{@apr_path}", - "--with-apr-util=#{@apr_util_path}", - '--with-ssl=/usr/lib/x86_64-linux-gnu', - '--enable-mpms-shared=worker event', - '--enable-mods-shared=reallyall', - '--disable-isapi', - '--disable-dav', - '--disable-dialup' - ] - end - - def install - return if installed? - - execute('install', [make_cmd, 'install', "prefix=#{path}"]) - end - - def url - "https://archive.apache.org/dist/httpd/httpd-#{version}.tar.bz2" - end - - def archive_files - ["#{path}/*"] - end - - def archive_path_name - 'httpd' - end - - def setup_tar - system <<-EOF - cd #{path} - - rm -rf cgi-bin/ error/ icons/ include/ man/ manual/ htdocs/ - rm -rf conf/extra/* conf/httpd.conf conf/httpd.conf.bak conf/magic conf/original - - mkdir -p lib - cp "#{@apr_path}/lib/libapr-1.so.0" ./lib - cp "#{@apr_util_path}/lib/libaprutil-1.so.0" ./lib - mkdir -p "./lib/apr-util-1" - cp "#{@apr_util_path}/lib/apr-util-1/"*.so ./lib/apr-util-1/ - mkdir -p "./lib/iconv" - cp "#{@apr_iconv_path}/lib/libapriconv-1.so.0" ./lib - cp "#{@apr_iconv_path}/lib/iconv/"*.so ./lib/iconv/ - cp /usr/lib/x86_64-linux-gnu/libcjose.so* ./lib/ - cp /usr/lib/x86_64-linux-gnu/libhiredis.so* ./lib/ - cp /usr/lib/x86_64-linux-gnu/libjansson.so* ./lib/ - EOF - end -end - -class ModAuthOpenidcRecipe < BaseRecipe - def url - "https://github.com/zmartzone/mod_auth_openidc/releases/download/v#{version}/mod_auth_openidc-#{version}.tar.gz" - end - - def configure_options - ENV['APR_LIBS'] = `#{@apr_path}/bin/apr-1-config --link-ld --libs` - ENV['APR_CFLAGS'] = `#{@apr_path}/bin/apr-1-config --cflags --includes` - [ - "--with-apxs2=#{@httpd_path}/bin/apxs" - ] - end -end - -class HTTPdMeal - attr_reader :name, :version - - def initialize(name, version, options = {}) - @name = name - @version = version - @options = options - end - - def cook - run('mkdir /app') - run('apt update') or raise 'Failed to apt update' - run('apt-get install -y libldap2-dev') or raise 'Failed to install libldap2-dev' - - apr_recipe.cook - apr_iconv_recipe.cook - apr_util_recipe.cook - httpd_recipe.cook - - # this symlink is needed so that modules can call `apxs` - # putting it here because we only need to do it once - system <<-EOF - cd /app - if ! [ -L "/app/httpd" ]; then - ln -s "#{httpd_recipe.path}" httpd - fi - EOF - - run('apt-get install -y libjansson-dev libcjose-dev libhiredis-dev') or raise 'Failed to install additional dependencies' - mod_auth_openidc_recipe.cook - end - - def url - httpd_recipe.url - end - - def archive_files - httpd_recipe.archive_files - end - - def archive_path_name - httpd_recipe.archive_path_name - end - - def archive_filename - httpd_recipe.archive_filename - end - - def setup_tar - httpd_recipe.setup_tar - end - - private - - def run(command) - output = `#{command}` - if $CHILD_STATUS.success? - true - else - $stdout.puts 'ERROR, output was:' - $stdout.puts output - false - end - end - - def latest_github_version(repo) - puts "Getting latest tag from #{repo}..." - repo = "https://github.com/#{repo}" - return `git -c 'versionsort.suffix=-' ls-remote --exit-code --refs --sort='version:refname' --tags #{repo} '*.*.*' | tail -1 | cut -d/ --fields=3`.strip - end - - def files_hashs - httpd_recipe.send(:files_hashs) + - apr_recipe.send(:files_hashs) + - apr_iconv_recipe.send(:files_hashs) + - apr_util_recipe.send(:files_hashs) + - mod_auth_openidc_recipe.send(:files_hashs) - end - - def mod_auth_openidc_recipe - @mod_auth_openidc ||= ModAuthOpenidcRecipe.new('mod-auth-openidc', '2.3.8', - httpd_path: httpd_recipe.path, - apr_path: apr_recipe.path, - md5: 'd6abc2f68dabf5d2557400af2499f500') - end - - def httpd_recipe - @http_recipe ||= HTTPdRecipe.new(@name, @version, { - apr_path: apr_recipe.path, - apr_util_path: apr_util_recipe.path, - apr_iconv_path: apr_iconv_recipe.path - }.merge(DetermineChecksum.new(@options).to_h)) - end - - def apr_util_recipe - apr_util_version = latest_github_version("apache/apr-util") - @apr_util_recipe ||= AprUtilRecipe.new('apr-util', apr_util_version, apr_path: apr_recipe.path, - apr_iconv_path: apr_iconv_recipe.path) - end - - def apr_iconv_recipe - apr_iconv_version = latest_github_version("apache/apr-iconv") - @apr_iconv_recipe ||= AprIconvRecipe.new('apr-iconv', apr_iconv_version, apr_path: apr_recipe.path) - end - - def apr_recipe - apr_version = latest_github_version("apache/apr") - @apr_recipe ||= AprRecipe.new('apr', apr_version) - end -end diff --git a/cflinuxfs4/recipe/hwc.rb b/cflinuxfs4/recipe/hwc.rb deleted file mode 100644 index c4df3486..00000000 --- a/cflinuxfs4/recipe/hwc.rb +++ /dev/null @@ -1,54 +0,0 @@ -# encoding: utf-8 -require_relative 'base' -require_relative '../lib/install_go' -require 'yaml' -require 'digest' - -class HwcRecipe < BaseRecipe - attr_reader :name, :version - - def cook - download unless downloaded? - extract - - install_go_compiler - - system <<-eof - sudo apt-get update - sudo apt-get -y upgrade - sudo apt-get -y install mingw-w64 - eof - - FileUtils.rm_rf("#{tmp_path}/hwc") - FileUtils.mv(Dir.glob("#{tmp_path}/hwc-*").first, "#{tmp_path}/hwc") - Dir.chdir("#{tmp_path}/hwc") do - system( - { 'PATH' => "#{ENV['PATH']}:/usr/local/go/bin" }, - "./bin/release-binaries.bash amd64 windows #{version} #{tmp_path}/hwc" - ) or raise 'Could not build hwc amd64' - system( - { 'PATH' => "#{ENV['PATH']}:/usr/local/go/bin" }, - "./bin/release-binaries.bash 386 windows #{version} #{tmp_path}/hwc" - ) or raise 'Could not build hwc 386' - end - - FileUtils.mv("#{tmp_path}/hwc/hwc-windows-amd64", '/tmp/hwc.exe') - FileUtils.mv("#{tmp_path}/hwc/hwc-windows-386", '/tmp/hwc_x86.exe') - end - - def archive_files - ['/tmp/hwc.exe', '/tmp/hwc_x86.exe'] - end - - def url - "https://github.com/cloudfoundry/hwc/archive/#{version}.tar.gz" - end - - def tmp_path - '/tmp/src/code.cloudfoundry.org' - end - - def archive_filename - "#{name}-#{version}-windows-x86-64.zip" - end -end diff --git a/cflinuxfs4/recipe/jruby.rb b/cflinuxfs4/recipe/jruby.rb deleted file mode 100644 index e17d6f23..00000000 --- a/cflinuxfs4/recipe/jruby.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -require 'mini_portile2' -require_relative 'base' - -class JRubyRecipe < BaseRecipe - def archive_files - %W[#{work_path}/bin #{work_path}/lib] - end - - def url - "https://repo1.maven.org/maven2/org/jruby/jruby-dist/#{jruby_version}/jruby-dist-#{jruby_version}-src.zip" - end - - def cook - download unless downloaded? - extract_zip - compile - end - - def compile - execute('compile', ['mvn', '-P', '!truffle', "-Djruby.default.ruby.version=#{ruby_version}"]) - end - - def extract_zip - files_hashs.each do |file| - verify_file(file) - - filename = File.basename(file[:local_path]) - message "Unzipping #{filename} into #{tmp_path}... " - FileUtils.mkdir_p tmp_path - execute('unzip', ["unzip", "-o", file[:local_path], "-d", tmp_path], {:cd => Dir.pwd, :initial_message => false}) - end - end - - def ruby_version - @ruby_version ||= version.match(/.*-ruby-(\d+\.\d+)/)[1] - end - - def jruby_version - @jruby_version ||= version.match(/(.*)-ruby-\d+\.\d+/)[1] - end -end diff --git a/cflinuxfs4/recipe/jruby_meal.rb b/cflinuxfs4/recipe/jruby_meal.rb deleted file mode 100644 index 121ad758..00000000 --- a/cflinuxfs4/recipe/jruby_meal.rb +++ /dev/null @@ -1,75 +0,0 @@ -# frozen_string_literal: true - -require_relative 'jruby' -require_relative 'maven' -require 'fileutils' -require 'digest' - -class JRubyMeal - attr_reader :name, :version - - def initialize(name, version, options = {}) - @name = name - @version = version - @options = options - end - - def cook - # We compile against the OpenJDK8 that the java buildpack team builds - # This is the openjdk-jdk that contains the openjdk-jre used in the ruby buildpack - - java_jdk_dir = '/opt/java' - java_jdk_tar_file = File.join(java_jdk_dir, 'openjdk-8-jdk.tar.gz') - java_jdk_bin_dir = File.join(java_jdk_dir, 'bin') - java_jdk_sha256 = 'dcb9fea2fc3a9b003031874ed17aa5d5a7ebbe397b276ecc8c814633003928fe' - java_buildpack_java_sdk = 'https://java-buildpack.cloudfoundry.org/openjdk-jdk/bionic/x86_64/openjdk-jdk-1.8.0_242-bionic.tar.gz' - - FileUtils.mkdir_p(java_jdk_dir) - raise 'Downloading openjdk-8-jdk failed.' unless system("wget #{java_buildpack_java_sdk} -O #{java_jdk_tar_file}") - - downloaded_sha = Digest::SHA256.file(java_jdk_tar_file).hexdigest - - raise "sha256 verification failed: expected #{java_jdk_sha256}, got #{downloaded_sha}" if java_jdk_sha256 != downloaded_sha - - raise 'Untarring openjdk-8-jdk failed.' unless system("tar xvf #{java_jdk_tar_file} -C #{java_jdk_dir}") - - ENV['JAVA_HOME'] = java_jdk_dir - ENV['PATH'] = "#{ENV['PATH']}:#{java_jdk_bin_dir}" - - maven.cook - maven.activate - - jruby.cook - end - - def url - jruby.url - end - - def archive_files - jruby.archive_files - end - - def archive_path_name - jruby.archive_path_name - end - - def archive_filename - jruby.archive_filename - end - - private - - def files_hashs - maven.send(:files_hashs) + - jruby.send(:files_hashs) - end - - def jruby - @jruby ||= JRubyRecipe.new(@name, @version, @options) - end - - def maven - @maven ||= MavenRecipe.new('maven', '3.6.3', md5: '42e4430d19894524e5d026217f2b3ecd') - end -end diff --git a/cflinuxfs4/recipe/maven.rb b/cflinuxfs4/recipe/maven.rb deleted file mode 100644 index 342fbe2b..00000000 --- a/cflinuxfs4/recipe/maven.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -require_relative 'base' - -class MavenRecipe < BaseRecipe - def url - "https://archive.apache.org/dist/maven/maven-3/#{version}/source/apache-maven-#{version}-src.tar.gz" - end - - def cook - download unless downloaded? - extract - - # install maven 3.6.3 to $HOME/apache-maven-3.6.3 - sha512 = 'c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0' - - Dir.chdir((ENV['HOME']).to_s) do - maven_download = "https://archive.apache.org/dist/maven/maven-3/#{version}/binaries/apache-maven-#{version}-bin.tar.gz" - maven_tar = "apache-maven-#{version}-bin.tar.gz" - system("curl -L #{maven_download} -o #{maven_tar}") - - downloaded_sha = Digest::SHA512.file(maven_tar).hexdigest - - raise "sha512 verification failed: expected #{sha512}, got #{downloaded_sha}" if sha512 != downloaded_sha - - system("tar xf #{maven_tar}") - end - - old_path = ENV['PATH'] - ENV['PATH'] = "#{ENV['HOME']}/apache-maven-3.6.3/bin:#{old_path}" - - install - ENV['PATH'] = old_path - FileUtils.rm_rf(File.join(ENV['HOME'], 'apache-maven-3.6.3')) - end - - def install - FileUtils.rm_rf(path) - execute('install', [ - 'mvn', - "-DdistributionTargetDir=#{path}", - 'clean', - 'package' - ]) - end -end diff --git a/cflinuxfs4/recipe/nginx.rb b/cflinuxfs4/recipe/nginx.rb deleted file mode 100644 index 354be00a..00000000 --- a/cflinuxfs4/recipe/nginx.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -require_relative '../lib/openssl_replace' -require_relative 'base' - -class NginxRecipe < BaseRecipe - def initialize(name, version, options = {}) - super name, version, options - # override openssl in container - OpenSSLReplace.replace_openssl - end - - def computed_options - [ - '--prefix=/', - '--error-log-path=stderr', - '--with-http_ssl_module', - '--with-http_realip_module', - '--with-http_gunzip_module', - '--with-http_gzip_static_module', - '--with-http_auth_request_module', - '--with-http_random_index_module', - '--with-http_secure_link_module', - '--with-http_stub_status_module', - '--without-http_uwsgi_module', - '--without-http_scgi_module', - '--with-pcre', - '--with-pcre-jit', - '--with-cc-opt=-fPIE -pie', - '--with-ld-opt=-fPIE -pie -z now' - ] - end - - def install - return if installed? - - execute('install', [make_cmd, 'install', "DESTDIR=#{path}"]) - end - - def archive_files - ["#{path}/*"] - end - - def archive_path_name - 'nginx' - end - - def setup_tar - `rm -Rf #{path}/html/ #{path}/conf/*` - end - - def url - "http://nginx.org/download/nginx-#{version}.tar.gz" - end -end diff --git a/cflinuxfs4/recipe/node.rb b/cflinuxfs4/recipe/node.rb deleted file mode 100644 index 254b7b0d..00000000 --- a/cflinuxfs4/recipe/node.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -require 'mini_portile2' -require 'fileutils' -require_relative 'base' - -class NodeRecipe < BaseRecipe - def computed_options - %w[--prefix=/ --openssl-use-def-ca-store] - end - - def install - execute('install', [make_cmd, 'install', "DESTDIR=#{dest_dir}", 'PORTABLE=1']) - end - - def archive_files - [dest_dir] - end - - def setup_tar - FileUtils.cp( - "#{work_path}/LICENSE", - dest_dir - ) - end - - def url - "https://nodejs.org/dist/v#{version}/node-v#{version}.tar.gz" - end - - def dest_dir - "/tmp/node-v#{version}-linux-x64" - end - - def configure - # Node building requires python https://github.com/nodejs/node/blob/main/BUILDING.md#unix-and-macos - # But cflinuxfs4 image does not come with python - system <<-EOF - #!/bin/sh - if [ -z $(command -v python3) ]; then - apt update - apt install -y python3 python3-pip - fi - EOF - execute('configure', %w(./configure) + computed_options) - end -end diff --git a/cflinuxfs4/recipe/php_common_recipes.rb b/cflinuxfs4/recipe/php_common_recipes.rb deleted file mode 100644 index 8f84ad4b..00000000 --- a/cflinuxfs4/recipe/php_common_recipes.rb +++ /dev/null @@ -1,569 +0,0 @@ -# frozen_string_literal: true - -require_relative 'base' -require_relative '../lib/geoip_downloader' -require 'uri' - -class BasePHPModuleRecipe < BaseRecipe - def initialize(name, version, options = {}) - super name, version, options - - @files = [{ - url: url, - local_path: local_path - }.merge(DetermineChecksum.new(options).to_h)] - end - - def local_path - File.join(archives_path, File.basename(url)) - end - - # override this method to allow local_path to be specified - # this prevents recipes with the same versions downloading colliding files (such as `v1.0.0.tar.gz`) - def files_hashs - @files.map do |file| - hash = case file - when String - { url: file } - when Hash - file.dup - else - raise ArgumentError, 'files must be an Array of Stings or Hashs' - end - - hash[:local_path] = local_path - hash - end - end -end - -class PeclRecipe < BasePHPModuleRecipe - def url - "http://pecl.php.net/get/#{name}-#{version}.tgz" - end - - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config" - ] - end - - def configure - return if configured? - - md5_file = File.join(tmp_path, 'configure.md5') - digest = Digest::MD5.hexdigest(computed_options.to_s) - File.open(md5_file, 'w') { |f| f.write digest } - - execute('phpize', 'phpize') - execute('configure', %w[sh configure] + computed_options) - end -end - -class AmqpPeclRecipe < PeclRecipe - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config" - ] - end -end - -class PkgConfigLibRecipe < BasePHPModuleRecipe - def cook - exists = system("PKG_CONFIG_PATH=$PKG_CONFIG_PATH:#{pkg_path} pkg-config #{pkgcfg_name} --exists") - super() unless exists - end - - def pkg_path - "#{File.expand_path(port_path)}/lib/pkgconfig/" - end -end - -class MaxMindRecipe < PeclRecipe - def work_path - File.join(tmp_path, "maxminddb-#{version}", 'ext') - end -end - -class HiredisRecipe < PkgConfigLibRecipe - def url - "https://github.com/redis/hiredis/archive/v#{version}.tar.gz" - end - - def local_path - "hiredis-#{version}.tar.gz" - end - - def configure; end - - def install - return if installed? - - execute('install', ['bash', '-c', "LIBRARY_PATH=lib PREFIX='#{path}' #{make_cmd} install"]) - end - - def pkgcfg_name - 'hiredis' - end -end - -class ImagickRecipe < BasePHPModuleRecipe - def url - "http://pecl.php.net/get/#{name}-#{version}.tgz" - end - - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config" - ] - end - - def configure - return if configured? - - md5_file = File.join(tmp_path, 'configure.md5') - digest = Digest::MD5.hexdigest(computed_options.to_s) - File.open(md5_file, 'w') { |f| f.write digest } - - # Setup libmagickwand-dev libmagickcore-dev - execute('install', %w[apt-get update]) - execute('install', %w[apt-get install -y libmagickwand-dev]) - - # Configure - execute('phpize', 'phpize') - execute('configure', %w[sh configure] + computed_options) - end -end - -class LibSodiumRecipe < PkgConfigLibRecipe - def url - "https://github.com/jedisct1/libsodium/archive/#{version}-RELEASE.tar.gz" - end - - def pkgcfg_name - 'libsodium' - end -end - -class IonCubeRecipe < BaseRecipe - def url - "http://downloads3.ioncube.com/loader_downloads/ioncube_loaders_lin_x86-64_#{version}.tar.gz" - end - - def configure; end - - def compile; end - - def install; end - - def self.build_ioncube?(_php_version) - true - end - - def path - tmp_path - end -end - -class LibRdKafkaRecipe < PkgConfigLibRecipe - def url - "https://github.com/edenhill/librdkafka/archive/v#{version}.tar.gz" - end - - def pkgcfg_name - 'rdkafka' - end - - def local_path - "librdkafka-#{version}.tar.gz" - end - - def work_path - File.join(tmp_path, "librdkafka-#{version}") - end - - def configure_prefix - '--prefix=/usr' - end - - def configure - return if configured? - - md5_file = File.join(tmp_path, 'configure.md5') - digest = Digest::MD5.hexdigest(computed_options.to_s) - File.open(md5_file, 'w') { |f| f.write digest } - - execute('configure', %w[bash ./configure] + computed_options) - end -end - -class CassandraCppDriverRecipe < PkgConfigLibRecipe - def url - "https://github.com/datastax/cpp-driver/archive/#{version}.tar.gz" - end - - def pkgcfg_name - 'cassandra' - end - - def local_path - "cassandra-cpp-driver-#{version}.tar.gz" - end - - def configure; end - - def compile - execute('compile', ['bash', '-c', 'mkdir -p build && cd build && cmake .. && make']) - end - - def install - execute('install', ['bash', '-c', 'cd build && make install']) - end -end - -class LuaPeclRecipe < PeclRecipe - def configure_options - %W[--with-php-config=#{@php_path}/bin/php-config --with-lua=#{@lua_path}] - end -end - -class LuaRecipe < BaseRecipe - def url - "http://www.lua.org/ftp/lua-#{version}.tar.gz" - end - - def configure; end - - def compile - execute('compile', ['bash', '-c', "#{make_cmd} linux MYCFLAGS=-fPIC"]) - end - - def install - return if installed? - - execute('install', ['bash', '-c', "#{make_cmd} install INSTALL_TOP=#{path}"]) - end -end - -class MemcachedPeclRecipe < PeclRecipe - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config", - '--with-libmemcached-dir', - '--enable-memcached-sasl', - '--enable-memcached-msgpack', - '--enable-memcached-igbinary', - '--enable-memcached-json' - ] - end -end - -class FakePeclRecipe < PeclRecipe - def url - "file://#{@php_source}/ext/#{name}-#{version}.tar.gz" - end - - def download - # this copys an extension folder out of the PHP source director (i.e. `ext/`) - # it pretends to download it by making a zip of the extension files - # that way the rest of the PeclRecipe works normally - files_hashs.each do |file| - path = URI(file[:url]).path.rpartition('-')[0] # only need path before the `-`, see url above - system <<-EOF - tar czf "#{file[:local_path]}" -C "#{File.dirname(path)}" "#{File.basename(path)}" - EOF - end - end -end - -class Gd72and73FakePeclRecipe < FakePeclRecipe - def configure_options - baseOpts = %w[--with-jpeg-dir --with-png-dir --with-xpm-dir --with-freetype-dir --with-webp-dir --with-zlib-dir] - - if version.start_with?('7.2') - baseOpts.push('--enable-gd-jis-conv') - else - baseOpts.push('--with-gd-jis-conv') - end - end -end - -class Gd74FakePeclRecipe < FakePeclRecipe - # how to build gd.so in PHP 7.4 changed dramatically - # In 7.4+, you can just use libgd from Ubuntu - def configure_options - [ - '--with-external-gd' - ] - end -end - -class OdbcRecipe < FakePeclRecipe - def configure_options - [ - '--with-unixODBC=shared,/usr' - ] - end - - def patch - system <<-EOF - cd #{work_path} - echo 'AC_DEFUN([PHP_ALWAYS_SHARED],[])dnl' > temp.m4 - echo >> temp.m4 - cat config.m4 >> temp.m4 - mv temp.m4 config.m4 - EOF - end - - def setup_tar - system <<-EOF - cp -a /usr/lib/x86_64-linux-gnu/libodbc.so* #{@php_path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libodbcinst.so* #{@php_path}/lib/ - EOF - end -end - -class SodiumRecipe < FakePeclRecipe - def configure_options - ENV['LDFLAGS'] = "-L#{@libsodium_path}/lib" - ENV['PKG_CONFIG_PATH'] = "#{@libsodium_path}/lib/pkgconfig" - sodium_flag = "--with-sodium=#{@libsodium_path}" - [ - "--with-php-config=#{@php_path}/bin/php-config", - sodium_flag - ] - end - - def setup_tar - system <<-EOF - cp -a #{@libsodium_path}/lib/libsodium.so* #{@php_path}/lib/ - EOF - end -end - -class PdoOdbcRecipe < FakePeclRecipe - def configure_options - [ - '--with-pdo-odbc=unixODBC,/usr' - ] - end - - def setup_tar - system <<-EOF - cp -a /usr/lib/x86_64-linux-gnu/libodbc.so* #{@php_path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libodbcinst.so* #{@php_path}/lib/ - EOF - end -end - -class OraclePdoRecipe < FakePeclRecipe - def configure_options - [ - "--with-pdo-oci=shared,instantclient,/oracle,#{OraclePdoRecipe.oracle_version}" - ] - end - - def self.oracle_version - Dir['/oracle/*'].select { |i| i.match(/libclntsh\.so\./) }.map { |i| i.sub(/.*libclntsh\.so\./, '') }.first - end - - def setup_tar - system <<-EOF - cp -an /oracle/libclntshcore.so.12.1 #{@php_path}/lib - cp -an /oracle/libclntsh.so #{@php_path}/lib - cp -an /oracle/libclntsh.so.12.1 #{@php_path}/lib - cp -an /oracle/libipc1.so #{@php_path}/lib - cp -an /oracle/libmql1.so #{@php_path}/lib - cp -an /oracle/libnnz12.so #{@php_path}/lib - cp -an /oracle/libociicus.so #{@php_path}/lib - cp -an /oracle/libons.so #{@php_path}/lib - EOF - end -end - -class OraclePeclRecipe < PeclRecipe - def configure_options - [ - '--with-oci8=shared,instantclient,/oracle' - ] - end - - def self.oracle_sdk? - File.directory?('/oracle') - end - - def setup_tar - system <<-EOF - cp -an /oracle/libclntshcore.so.12.1 #{@php_path}/lib - cp -an /oracle/libclntsh.so #{@php_path}/lib - cp -an /oracle/libclntsh.so.12.1 #{@php_path}/lib - cp -an /oracle/libipc1.so #{@php_path}/lib - cp -an /oracle/libmql1.so #{@php_path}/lib - cp -an /oracle/libnnz12.so #{@php_path}/lib - cp -an /oracle/libociicus.so #{@php_path}/lib - cp -an /oracle/libons.so #{@php_path}/lib - EOF - end -end - -class PHPIRedisRecipe < PeclRecipe - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config", - '--enable-phpiredis', - "--with-hiredis-dir=#{@hiredis_path}" - ] - end - - def url - "https://github.com/nrk/phpiredis/archive/v#{version}.tar.gz" - end - - def local_path - "phpiredis-#{version}.tar.gz" - end -end - -class RedisPeclRecipe < PeclRecipe - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config", - '--enable-redis-igbinary', - '--enable-redis-lzf', - '--with-liblzf=no' - ] - end -end - -# TODO: Remove after PHP 7 is out of support -class PHPProtobufPeclRecipe < PeclRecipe - def url - "https://github.com/allegro/php-protobuf/archive/v#{version}.tar.gz" - end - - def local_path - "php-protobuf-#{version}.tar.gz" - end -end - -class TidewaysXhprofRecipe < PeclRecipe - def url - "https://github.com/tideways/php-xhprof-extension/archive/v#{version}.tar.gz" - end - - def local_path - "tideways-xhprof-#{version}.tar.gz" - end -end - -class EnchantFakePeclRecipe < FakePeclRecipe - def patch - super - system <<-EOF - cd #{work_path} - sed -i 's|#include "../spl/spl_exceptions.h"|#include |' enchant.c - EOF - end -end - -class RabbitMQRecipe < PkgConfigLibRecipe - def url - "https://github.com/alanxz/rabbitmq-c/archive/v#{version}.tar.gz" - end - - def pkgcfg_name - 'librabbitmq' - end - - def local_path - "rabbitmq-#{version}.tar.gz" - end - - def work_path - File.join(tmp_path, "rabbitmq-c-#{@version}") - end - - def configure; end - - def compile - execute('compile', ['bash', '-c', 'cmake .']) - execute('compile', ['bash', '-c', 'cmake --build .']) - execute('compile', ['bash', '-c', 'cmake -DCMAKE_INSTALL_PREFIX=/usr/local .']) - execute('compile', ['bash', '-c', 'cmake --build . --target install']) - end -end - -class SnmpRecipe - attr_reader :name, :version - - def initialize(name, version, options) - @name = name - @version = version - @options = options - end - - def files_hashs - [] - end - - def cook - system <<-EOF - cd #{@php_path} - mkdir -p mibs - cp -a /usr/lib/x86_64-linux-gnu/libnetsnmp.so* lib/ - # copy mibs that are packaged freely - cp -r /usr/share/snmp/mibs/* mibs - # copy mibs downloader & smistrip, will download un-free mibs - cp /usr/bin/download-mibs bin - cp /usr/bin/smistrip bin - sed -i "s|^CONFDIR=/etc/snmp-mibs-downloader|CONFDIR=\$HOME/php/mibs/conf|" bin/download-mibs - sed -i "s|^SMISTRIP=/usr/bin/smistrip|SMISTRIP=\$HOME/php/bin/smistrip|" bin/download-mibs - # copy mibs download config - cp -R /etc/snmp-mibs-downloader mibs/conf - sed -i "s|^DIR=/usr/share/doc|DIR=\$HOME/php/mibs/originals|" mibs/conf/iana.conf - sed -i "s|^DEST=iana|DEST=|" mibs/conf/iana.conf - sed -i "s|^DIR=/usr/share/doc|DIR=\$HOME/php/mibs/originals|" mibs/conf/ianarfc.conf - sed -i "s|^DEST=iana|DEST=|" mibs/conf/ianarfc.conf - sed -i "s|^DIR=/usr/share/doc|DIR=\$HOME/php/mibs/originals|" mibs/conf/rfc.conf - sed -i "s|^DEST=ietf|DEST=|" mibs/conf/rfc.conf - sed -i "s|^BASEDIR=/var/lib/mibs|BASEDIR=\$HOME/php/mibs|" mibs/conf/snmp-mibs-downloader.conf - # copy data files - # TODO: these are gone or have moved, commenting out for now - # mkdir mibs/originals - # cp -R /usr/share/doc/mibiana mibs/originals - # cp -R /usr/share/doc/mibrfcs mibs/originals - EOF - end -end - -class SuhosinPeclRecipe < PeclRecipe - def url - "https://github.com/sektioneins/suhosin/archive/#{version}.tar.gz" - end -end - -class TwigPeclRecipe < PeclRecipe - def url - "https://github.com/twigphp/Twig/archive/v#{version}.tar.gz" - end - - def work_path - "#{super}/ext/twig" - end -end - -class XcachePeclRecipe < PeclRecipe - def url - "http://xcache.lighttpd.net/pub/Releases/#{version}/xcache-#{version}.tar.gz" - end -end - -class XhprofPeclRecipe < PeclRecipe - def url - "https://github.com/phacility/xhprof/archive/#{version}.tar.gz" - end - - def work_path - "#{super}/extension" - end -end diff --git a/cflinuxfs4/recipe/php_meal.rb b/cflinuxfs4/recipe/php_meal.rb deleted file mode 100644 index 9ab830fb..00000000 --- a/cflinuxfs4/recipe/php_meal.rb +++ /dev/null @@ -1,245 +0,0 @@ -# frozen_string_literal: true - -require_relative 'php_common_recipes' -require_relative 'php_recipe' - -class PhpMeal - attr_reader :name, :version - - def initialize(name, version, options) - @name = name - @version = version - version_parts = version.split('.') - @major_version = version_parts[0] - @minor_version = version_parts[1] - @options = options - @native_modules = [] - @extensions = [] - - create_native_module_recipes - create_extension_recipes - - (@native_modules + @extensions).each do |recipe| - recipe.instance_variable_set('@php_path', php_recipe.path) - recipe.instance_variable_set('@php_source', "#{php_recipe.send(:tmp_path)}/php-#{@version}") - - if recipe.is_a? FakePeclRecipe - recipe.instance_variable_set('@version', @version) - recipe.instance_variable_set('@files', [{ url: recipe.url, md5: nil }]) - end - end - end - - def cook - run('apt-get -y update') or raise 'Failed to apt-get update' - run('apt-get -y upgrade') or raise 'Failed to apt-get upgrade' - run("apt-get -y install #{apt_packages}") or raise 'Failed to apt-get install packages' - symlink_commands - - if OraclePeclRecipe.oracle_sdk? - Dir.chdir('/oracle') do - system 'ln -s libclntsh.so.* libclntsh.so' - end - end - - php_recipe.cook - php_recipe.activate - - # native libraries - @native_modules.each(&:cook) - - # php extensions - @extensions.each do |recipe| - recipe.cook if should_cook?(recipe) - end - end - - def url - php_recipe.url - end - - def archive_files - php_recipe.archive_files - end - - def archive_path_name - php_recipe.archive_path_name - end - - def archive_filename - php_recipe.archive_filename - end - - def setup_tar - php_recipe.setup_tar - if OraclePeclRecipe.oracle_sdk? - @extensions.detect { |r| r.name == 'oci8' }.setup_tar - @extensions.detect { |r| r.name == 'pdo_oci' }.setup_tar - end - @extensions.detect { |r| r.name == 'odbc' }&.setup_tar - @extensions.detect { |r| r.name == 'pdo_odbc' }&.setup_tar - @extensions.detect { |r| r.name == 'sodium' }&.setup_tar - end - - private - - def create_native_module_recipes - return unless @options[:php_extensions_file] - - php_extensions_hash = YAML.safe_load_file(@options[:php_extensions_file]) - - php_extensions_hash['native_modules'].each do |hash| - klass = Kernel.const_get(hash['klass']) - - @native_modules << klass.new( - hash['name'], - hash['version'], - md5: hash['md5'] - ) - end - end - - def create_extension_recipes - return unless @options[:php_extensions_file] - - php_extensions_hash = YAML.safe_load_file(@options[:php_extensions_file]) - - php_extensions_hash['extensions'].each do |hash| - klass = Kernel.const_get(hash['klass']) - - @extensions << klass.new( - hash['name'], - hash['version'], - md5: hash['md5'] - ) - end - - @extensions.each do |recipe| - case recipe.name - when 'amqp' - recipe.instance_variable_set('@rabbitmq_path', @native_modules.detect { |r| r.name == 'rabbitmq' }.work_path) - when 'lua' - recipe.instance_variable_set('@lua_path', @native_modules.detect { |r| r.name == 'lua' }.path) - when 'phpiredis' - recipe.instance_variable_set('@hiredis_path', @native_modules.detect { |r| r.name == 'hiredis' }.path) - when 'sodium' - recipe.instance_variable_set('@libsodium_path', @native_modules.detect { |r| r.name == 'libsodium' }.path) - end - end - end - - def apt_packages - %w[automake - firebird-dev - libaspell-dev - libc-client2007e-dev - libcurl4-openssl-dev - libdb-dev - libedit-dev - libenchant-2-dev - libexpat1-dev - libgdbm-dev - libgeoip-dev - libgmp-dev - libgpgme11-dev - libjpeg-dev - libkrb5-dev - libldap2-dev - libmagickwand-dev - libmagickcore-dev - libmaxminddb-dev - libmcrypt-dev - libmemcached-dev - libonig-dev - libpng-dev - libpspell-dev - librecode-dev - libsasl2-dev - libsnmp-dev - libsqlite3-dev - libssh2-1-dev - libssl-dev - libtidy-dev - libtool - libwebp-dev - libxml2-dev - libzip-dev - libzookeeper-mt-dev - snmp-mibs-downloader - sqlite3 - unixodbc-dev].join(' ') - end - - # @todo: remove this method when all the tests run for cflinuxfs4 without errors - def install_libuv - %q(( - if [ "$(pkg-config libuv --print-provides | awk '{print $3}')" != "1.12.0" ]; then - cd /tmp - wget http://dist.libuv.org/dist/v1.12.0/libuv-v1.12.0.tar.gz - tar zxf libuv-v1.12.0.tar.gz - cd libuv-v1.12.0 - sh autogen.sh - ./configure - make install - fi - ) - ) - end - - def symlink_commands - run('sudo ln -s /usr/include/x86_64-linux-gnu/curl /usr/local/include/curl') - run('sudo ln -fs /usr/include/x86_64-linux-gnu/gmp.h /usr/include/gmp.h') - run('sudo ln -fs /usr/lib/x86_64-linux-gnu/libldap.so /usr/lib/libldap.so') - run('sudo ln -fs /usr/lib/x86_64-linux-gnu/libldap_r.so /usr/lib/libldap_r.so') - end - - def should_cook?(recipe) - case recipe.name - when 'ioncube' - IonCubeRecipe.build_ioncube?(version) - when 'oci8', 'pdo_oci' - OraclePeclRecipe.oracle_sdk? - else - true - end - end - - def files_hashs - native_module_hashes = @native_modules.map do |recipe| - recipe.send(:files_hashs) - end.flatten - - extension_hashes = @extensions.map do |recipe| - recipe.send(:files_hashs) if should_cook?(recipe) - end.flatten.compact - - extension_hashes + native_module_hashes - end - - def php_recipe - php_recipe_options = {} - - hiredis_recipe = @native_modules.detect { |r| r.name == 'hiredis' } - libmemcached_recipe = @native_modules.detect { |r| r.name == 'libmemcached' } - ioncube_recipe = @extensions.detect { |r| r.name == 'ioncube' } - - php_recipe_options[:hiredis_path] = hiredis_recipe.path unless hiredis_recipe.nil? - php_recipe_options[:libmemcached_path] = libmemcached_recipe.path unless libmemcached_recipe.nil? - php_recipe_options[:ioncube_path] = ioncube_recipe.path unless ioncube_recipe.nil? - - php_recipe_options.merge(DetermineChecksum.new(@options).to_h) - - @php_recipe ||= PhpRecipe.new(@name, @version, php_recipe_options) - end - - def run(command) - output = `#{command}` - if $CHILD_STATUS.success? - true - else - $stdout.puts 'ERROR, output was:' - $stdout.puts output - false - end - end -end diff --git a/cflinuxfs4/recipe/php_recipe.rb b/cflinuxfs4/recipe/php_recipe.rb deleted file mode 100644 index aa0a92bb..00000000 --- a/cflinuxfs4/recipe/php_recipe.rb +++ /dev/null @@ -1,120 +0,0 @@ -# frozen_string_literal: true - -require_relative 'php_common_recipes' - -class PhpRecipe < BaseRecipe - def configure_options - %w[--disable-static - --enable-shared - --enable-ftp=shared - --enable-sockets=shared - --enable-soap=shared - --enable-fileinfo=shared - --enable-bcmath - --enable-calendar - --enable-intl - --with-kerberos - --with-bz2=shared - --with-curl=shared - --enable-dba=shared - --with-password-argon2=/usr/lib/x86_64-linux-gnu - --with-cdb - --with-gdbm - --with-mysqli=shared - --enable-pdo=shared - --with-pdo-sqlite=shared,/usr - --with-pdo-mysql=shared,mysqlnd - --with-pdo-pgsql=shared - --with-pgsql=shared - --with-pspell=shared - --with-gettext=shared - --with-gmp=shared - --with-imap=shared - --with-imap-ssl=shared - --with-ldap=shared - --with-ldap-sasl - --with-zlib=shared - --with-xsl=shared - --with-snmp=shared - --enable-mbstring=shared - --enable-mbregex - --enable-exif=shared - --with-openssl=shared - --enable-fpm - --enable-pcntl=shared - --enable-sysvsem=shared - --enable-sysvshm=shared - --enable-sysvmsg=shared - --enable-shmop=shared].join(' ') - end - - def url - "https://github.com/php/web-php-distributions/raw/master/php-#{version}.tar.gz" - end - - def archive_files - ["#{port_path}/*"] - end - - def archive_path_name - 'php' - end - - def configure - return if configured? - - md5_file = File.join(tmp_path, 'configure.md5') - digest = Digest::MD5.hexdigest(computed_options.to_s) - File.open(md5_file, 'w') { |f| f.write digest } - - # LIBS=-lz enables using zlib when configuring - execute('configure', ['bash', '-c', "LIBS=-lz ./configure #{computed_options.join ' '}"]) - end - - def major_version - @major_version ||= version.match(/^(\d+\.\d+)/)[1] - end - - def zts_path - Dir["#{path}/lib/php/extensions/no-debug-non-zts-*"].first - end - - def setup_tar - lib_dir = '/usr/lib/x86_64-linux-gnu' - argon_dir = '/usr/lib/x86_64-linux-gnu' - - system <<-EOF - cp -a /usr/local/lib/x86_64-linux-gnu/librabbitmq.so* #{path}/lib/ - cp -a #{@hiredis_path}/lib/libhiredis.so* #{path}/lib/ - cp -a /usr/lib/libc-client.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libmcrypt.so* #{path}/lib - cp -a #{lib_dir}/libaspell.so* #{path}/lib - cp -a #{lib_dir}/libpspell.so* #{path}/lib - cp -a /usr/lib/x86_64-linux-gnu/libmemcached.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libuv.so* #{path}/lib - cp -a #{argon_dir}/libargon2.so* #{path}/lib - cp -a /usr/lib/librdkafka.so* #{path}/lib - cp -a /usr/lib/x86_64-linux-gnu/libzip.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libGeoIP.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libgpgme.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libassuan.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libgpg-error.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libtidy*.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libenchant*.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libfbclient.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/librecode.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libtommath.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libmaxminddb.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libssh2.so* #{path}/lib/ - EOF - - system "cp #{@ioncube_path}/ioncube/ioncube_loader_lin_#{major_version}.so #{zts_path}/ioncube.so" if IonCubeRecipe.build_ioncube?(version) - - system <<-EOF - # Remove unused files - rm "#{path}/etc/php-fpm.conf.default" - rm "#{path}/bin/php-cgi" - find "#{path}/lib/php/extensions" -name "*.a" -type f -delete - EOF - end -end diff --git a/cflinuxfs4/recipe/python.rb b/cflinuxfs4/recipe/python.rb deleted file mode 100644 index 9e210f00..00000000 --- a/cflinuxfs4/recipe/python.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -require 'English' -require 'fileutils' -require 'mini_portile2' -require_relative '../lib/openssl_replace' -require_relative 'base' - -class PythonRecipe < BaseRecipe - def initialize(name, version, options = {}) - super name, version, options - # override openssl in container - OpenSSLReplace.replace_openssl - end - - def computed_options - [ - '--enable-shared', - '--with-ensurepip=yes', - '--with-dbmliborder=bdb:gdbm', - '--with-tcltk-includes="-I/usr/include/tcl8.6"', - '--with-tcltk-libs="-L/usr/lib/x86_64-linux-gnu -ltcl8.6 -L/usr/lib/x86_64-linux-gnu -ltk8.6"', - "--prefix=#{prefix_path}", - '--enable-unicode=ucs4' - ] - end - - def cook - install_apt('libdb-dev libgdbm-dev tk8.6-dev') - - run('apt-get -y --force-yes -d install --reinstall libtcl8.6 libtk8.6 libxss1') or raise 'Failed to download libtcl8.6 libtk8.6 libxss1' - FileUtils.mkdir_p prefix_path - Dir.glob('/var/cache/apt/archives/lib{tcl8.6,tk8.6,xss1}_*.deb').each do |path| - $stdout.puts("dpkg -x #{path} #{prefix_path}") - run("dpkg -x #{path} #{prefix_path}") or raise "Could not extract #{path}" - end - - super - end - - def archive_files - ["#{prefix_path}/*"] - end - - def setup_tar - File.symlink('./python3', "#{prefix_path}/bin/python") unless File.exist?("#{prefix_path}/bin/python") - end - - def url - "https://www.python.org/ftp/python/#{version}/Python-#{version}.tgz" - end - - def prefix_path - '/app/.heroku/vendor' - end - - private - - def install_apt(packages) - $stdout.print "Running 'install dependencies' for #{@name} #{@version}... " - if run("sudo apt-get update && sudo apt-get -y install #{packages}") - $stdout.puts 'OK' - else - raise 'Failed to complete install dependencies task' - end - end - - def run(command) - output = `#{command}` - if $CHILD_STATUS.success? - true - else - $stdout.puts 'ERROR, output was:' - $stdout.puts output - false - end - end -end diff --git a/cflinuxfs4/recipe/ruby.rb b/cflinuxfs4/recipe/ruby.rb deleted file mode 100644 index 7a823b5c..00000000 --- a/cflinuxfs4/recipe/ruby.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -require 'English' -require 'mini_portile2' -require_relative 'base' - -class RubyRecipe < BaseRecipe - def computed_options - [ - '--enable-load-relative', - '--disable-install-doc', - 'debugflags=-g', - "--prefix=#{prefix_path}", - '--without-gmp' - ] - end - - def cook - run('apt-get update') or raise 'Failed to apt-get update' - run('apt-get -y install libffi-dev') or raise 'Failed to install libffi-dev' - super - end - - def prefix_path - "/app/vendor/ruby-#{version}" - end - - def minor_version - version.match(/(\d+\.\d+)\./)[1] - end - - def archive_files - ["#{prefix_path}/*"] - end - - def url - "https://cache.ruby-lang.org/pub/ruby/#{minor_version}/ruby-#{version}.tar.gz" - end - - private - - def run(command) - output = `#{command}` - if $CHILD_STATUS.success? - true - else - $stdout.puts 'ERROR, output was:' - $stdout.puts output - false - end - end -end diff --git a/cflinuxfs4/spec/assets/binary-exerciser.sh b/cflinuxfs4/spec/assets/binary-exerciser.sh deleted file mode 100755 index df4c3c99..00000000 --- a/cflinuxfs4/spec/assets/binary-exerciser.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set +e - -tar_name=$1; shift - -mkdir -p /tmp/binary-exerciser -current_dir=`pwd` -cd /tmp/binary-exerciser - -tar xzf $current_dir/${tar_name} -eval $(printf '%q ' "$@") diff --git a/cflinuxfs4/spec/assets/bundler-exerciser.sh b/cflinuxfs4/spec/assets/bundler-exerciser.sh deleted file mode 100755 index 96949e07..00000000 --- a/cflinuxfs4/spec/assets/bundler-exerciser.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set +e - -tar_name=$1; shift - -current_dir=`pwd` -tmpdir=$(mktemp -d /tmp/binary-builder.XXXXXXXX) -cd $tmpdir - -tar xzf $current_dir/${tar_name} --touch - -export GEM_HOME=$tmpdir -export GEM_PATH=$tmpdir - -eval $(printf '%q ' "$@") diff --git a/cflinuxfs4/spec/assets/jruby-exerciser.sh b/cflinuxfs4/spec/assets/jruby-exerciser.sh deleted file mode 100755 index 852079bf..00000000 --- a/cflinuxfs4/spec/assets/jruby-exerciser.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set +e - -mkdir -p /tmp/binary-exerciser -current_dir=`pwd` -cd /tmp/binary-exerciser - -tar xzf $current_dir/jruby-9.2.8.0-ruby-2.5-linux-x64.tgz -JAVA_HOME=/opt/java -PATH=$PATH:$JAVA_HOME/bin -./bin/jruby -e 'puts "#{RUBY_PLATFORM} #{RUBY_VERSION}"' diff --git a/cflinuxfs4/spec/assets/php-exerciser.sh b/cflinuxfs4/spec/assets/php-exerciser.sh deleted file mode 100755 index 18c0d2ea..00000000 --- a/cflinuxfs4/spec/assets/php-exerciser.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -tar_name=$1; shift -current_dir=$(pwd) -mkdir -p /tmp/binary-exerciser -cd /tmp/binary-exerciser - -tar xzf "$current_dir/$tar_name" -export LD_LIBRARY_PATH="$PWD/php/lib" -eval "$(printf '%q ' "$@")" diff --git a/cflinuxfs4/spec/assets/php-extensions.yml b/cflinuxfs4/spec/assets/php-extensions.yml deleted file mode 100644 index f075c9a8..00000000 --- a/cflinuxfs4/spec/assets/php-extensions.yml +++ /dev/null @@ -1,198 +0,0 @@ ---- -native_modules: -- name: rabbitmq - version: 0.10.0 - md5: 6f09f0cb07cea221657a768bd9c7dff7 - klass: RabbitMQRecipe -- name: lua - version: 5.4.0 - md5: dbf155764e5d433fc55ae80ea7060b60 - klass: LuaRecipe -- name: hiredis - version: 1.0.0 - md5: 209ae570cdee65a5143ea6db8ac07fe3 - klass: HiredisRecipe -- name: snmp - version: nil - md5: nil - klass: SnmpRecipe -- name: librdkafka - version: 1.5.2 - md5: f5272e30ab6556967ed82a58d2ad35e1 - klass: LibRdKafkaRecipe -- name: libsodium - version: 1.0.18 - md5: 3ca9ebc13b6b4735acae0a6a4c4f9a95 - klass: LibSodiumRecipe -extensions: -- name: apcu - version: 5.1.19 - md5: a868ee0b4179fb240cf6eb7e49723794 - klass: PeclRecipe -- name: igbinary - version: 3.1.6 - md5: 9d1e0530025c7f129f46c81f7581af98 - klass: PeclRecipe -- name: gnupg - version: 1.4.0 - md5: 2354cb56168d8ea0f643e548e139d013 - klass: PeclRecipe -- name: imagick - version: 3.4.4 - md5: 6d3a7048ab73b0fab931f28c484dbf76 - klass: PeclRecipe -- name: LZF - version: 1.6.8 - md5: 0677fba342c89795de6c694f3e72ba1d - klass: PeclRecipe -- name: mailparse - version: 3.1.1 - md5: 17d77e3c03c25acaf51926a9020a4596 - klass: PeclRecipe -- name: mongodb - version: 1.9.0 - md5: 780f206f6a9399b5a4cabfd304f6ecff - klass: PeclRecipe -- name: msgpack - version: 2.1.2 - md5: 9bcaad416fc2b3c6ffd6966e0ae30313 - klass: PeclRecipe -- name: oauth - version: 2.0.7 - md5: 8ea6eb5ac6de8a4ed399980848c04c0c - klass: PeclRecipe -- name: odbc - version: nil - md5: nil - klass: OdbcRecipe -- name: pdo_odbc - version: nil - md5: nil - klass: PdoOdbcRecipe -- name: pdo_sqlsrv - version: 5.8.1 - md5: e687989a47cefd6cf9005e1f41637289 - klass: PeclRecipe -- name: rdkafka - version: 4.1.1 - md5: f4b39d7f6b4c489bfb4c5532f71046c2 - klass: PeclRecipe -- name: redis - version: 5.3.2 - md5: 8531a792d43a60dd03f87eec7b65b381 - klass: RedisPeclRecipe -- name: ssh2 - version: 1.2 - md5: ae62ba2d4a7bbd5eff34daa8ed9f6ed6 - klass: PeclRecipe -- name: sqlsrv - version: 5.8.1 - md5: 1a237f847d4466a85f7bfdb6b2fd5ecd - klass: PeclRecipe -- name: stomp - version: 2.0.2 - md5: 507c30184fde736e924cee20c56df061 - klass: PeclRecipe -- name: xdebug - version: 3.0.1 - md5: 52891d89de6829fa8dba1132b8c66f75 - klass: PeclRecipe -- name: yaf - version: 3.2.5 - md5: 8cd86db117f65131d212cac1c60065a3 - klass: PeclRecipe -- name: yaml - version: 2.2.0 - md5: cd8d34b87d9e147691d66590c403ba46 - klass: PeclRecipe -- name: memcached - version: 3.1.5 - md5: eb535a7551aad6bff0d836a4dec9c4fa - klass: MemcachedPeclRecipe -- name: sodium - version: nil - md5: nil - klass: SodiumRecipe - -#bundled w/PHP -- name: tidy - version: nil - md5: nil - klass: FakePeclRecipe -- name: enchant - version: nil - md5: nil - klass: FakePeclRecipe -- name: pdo_firebird - version: nil - md5: nil - klass: FakePeclRecipe -- name: readline - version: nil - md5: nil - klass: FakePeclRecipe -- name: xmlrpc - version: nil - md5: nil - klass: FakePeclRecipe -- name: zip - version: nil - md5: nil - klass: FakePeclRecipe - -#non-standard -- name: amqp - version: 1.10.2 - md5: 1163a52a495cab5210a6a2c8deb60064 - klass: AmqpPeclRecipe -- name: ioncube - version: 10.4.4 - md5: 1e6b0d8a8db6c5536c99bd7e67eb6a4f - klass: IonCubeRecipe -- name: lua - version: 2.0.7 - md5: a37402f8b10753a80db56b61e2e70c29 - klass: LuaPeclRecipe -- name: maxminddb - version: 1.8.0 - md5: ed95e0d8914ff5dd340fa6cf47b2b921 - klass: MaxMindRecipe -- name: psr - version: 1.0.1 - md5: 9e44b2f2e271bf57c5dbf6b6b07a8acf - klass: PeclRecipe -- name: phalcon - version: 4.1.0 - md5: c13b72a724820027ec9889f0dca0670d - klass: PeclRecipe -- name: phpiredis - version: 1.0.1 - md5: 09a9bdb347c70832d3e034655b604064 - klass: PHPIRedisRecipe -- name: phpprotobufpecl - version: 0.12.4 - md5: 77a77a429af4a5ff97778ccafeffa43a - klass: PHPProtobufPeclRecipe -- name: tideways_xhprof - version: 5.0.2 - md5: 374cf4ff7ba03401a279777abe94815d #curl -sL https://github.com/tideways/php-xhprof-extension/archive/v5.0.2.tar.gz | md5sum | cut -d ' ' -f 1 - klass: TidewaysXhprofRecipe -- name: solr - version: 2.5.1 - md5: 29fc866198d61bccdbc4c4f53fb7ef06 - klass: PeclRecipe - -- name: gd - version: nil - md5: nil - klass: Gd74FakePeclRecipe - -#Oracle client stuff. Not included unless libs are present -- name: oci8 - version: 2.2.0 - md5: 678d2a647881cd8e5b458c669dcce215 - klass: OraclePeclRecipe -- name: pdo_oci - version: nil - md5: nil - klass: OraclePdoRecipe diff --git a/cflinuxfs4/spec/integration/bundler_spec.rb b/cflinuxfs4/spec/integration/bundler_spec.rb deleted file mode 100644 index 09a71dc2..00000000 --- a/cflinuxfs4/spec/integration/bundler_spec.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when bundler is specified' do - before(:all) do - run_binary_builder('bundler', '1.11.2', '--sha256=c7aa8ffe0af6e0c75d0dad8dd7749cb8493b834f0ed90830d4843deb61906768') - @binary_tarball_location = File.join(Dir.pwd, 'bundler-1.11.2.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, replaces the shebangs, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - bundler_version_cmd = './spec/assets/bundler-exerciser.sh bundler-1.11.2.tgz ./bin/bundle -v' - output, status = run(bundler_version_cmd) - - expect(status).to be_success - expect(output).to include('Bundler version 1.11.2') - - shebang = `tar -O -xf #{@binary_tarball_location} ./bin/bundle | head -n1`.chomp - expect(shebang).to eq('#!/usr/bin/env ruby') - shebang = `tar -O -xf #{@binary_tarball_location} ./bin/bundler | head -n1`.chomp - expect(shebang).to eq('#!/usr/bin/env ruby') - end - end -end diff --git a/cflinuxfs4/spec/integration/dep_spec.rb b/cflinuxfs4/spec/integration/dep_spec.rb deleted file mode 100644 index e75c1061..00000000 --- a/cflinuxfs4/spec/integration/dep_spec.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when dep is specified' do - before(:all) do - run_binary_builder('dep', 'v0.3.0', '--sha256=7d816ffb14f57c4b01352676998a8cda9e4fb24eaec92bd79526e1045c5a0c83') - @binary_tarball_location = File.join(Dir.pwd, 'dep-v0.3.0-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - dep_version_cmd = './spec/assets/binary-exerciser.sh dep-v0.3.0-linux-x64.tgz ./bin/dep ensure -examples' - output, status = run(dep_version_cmd) - - expect(status).to be_success - expect(output).to include('dep ensure') - end - - it 'includes the license in the tar file.' do - expect(tar_contains_file('bin/LICENSE')).to eq true - end - end -end diff --git a/cflinuxfs4/spec/integration/glide_spec.rb b/cflinuxfs4/spec/integration/glide_spec.rb deleted file mode 100644 index ba78d479..00000000 --- a/cflinuxfs4/spec/integration/glide_spec.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when glide is specified' do - before(:all) do - run_binary_builder('glide', 'v0.11.0', '--sha256=7a7023aff20ba695706a262b8c07840ee28b939ea6358efbb69ab77da04f0052') - @binary_tarball_location = File.join(Dir.pwd, 'glide-v0.11.0-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - glide_version_cmd = './spec/assets/binary-exerciser.sh glide-v0.11.0-linux-x64.tgz ./bin/glide -v' - output, status = run(glide_version_cmd) - - expect(status).to be_success - expect(output).to include('glide version 0.11.0') - end - - it 'includes the license in the tar file.' do - expect(tar_contains_file('bin/LICENSE')).to eq true - end - end -end diff --git a/cflinuxfs4/spec/integration/go_spec.rb b/cflinuxfs4/spec/integration/go_spec.rb deleted file mode 100644 index 6d89b6f9..00000000 --- a/cflinuxfs4/spec/integration/go_spec.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when go is specified' do - before(:all) do - run_binary_builder('go', '1.6.3', '--sha256=6326aeed5f86cf18f16d6dc831405614f855e2d416a91fd3fdc334f772345b00') - @binary_tarball_location = File.join(Dir.pwd, 'go1.6.3.linux-amd64.tar.gz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - go_version_cmd = './spec/assets/binary-exerciser.sh go1.6.3.linux-amd64.tar.gz GOROOT=/tmp/binary-exerciser/go ./go/bin/go version' - output, status = run(go_version_cmd) - - expect(status).to be_success - expect(output).to include('go1.6.3') - end - - it 'includes the license in the tar file.' do - expect(tar_contains_file('go/LICENSE')).to eq true - end - end -end diff --git a/cflinuxfs4/spec/integration/godep_spec.rb b/cflinuxfs4/spec/integration/godep_spec.rb deleted file mode 100644 index 7848676c..00000000 --- a/cflinuxfs4/spec/integration/godep_spec.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when godep is specified' do - before(:all) do - run_binary_builder('godep', 'v14', '--sha256=0f212bcf903d5b01db0e93a4218b79f228c6f080d5a409dd4e2ec5edfbc2aad5') - @binary_tarball_location = File.join(Dir.pwd, 'godep-v14-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - godep_version_cmd = './spec/assets/binary-exerciser.sh godep-v14-linux-x64.tgz ./bin/godep version' - output, status = run(godep_version_cmd) - - expect(status).to be_success - expect(output).to include('v14') - end - - it 'includes the license in the tar file.' do - expect(tar_contains_file('bin/License')).to eq true - end - end -end diff --git a/cflinuxfs4/spec/integration/httpd_spec.rb b/cflinuxfs4/spec/integration/httpd_spec.rb deleted file mode 100644 index 65cfa228..00000000 --- a/cflinuxfs4/spec/integration/httpd_spec.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when httpd is specified' do - before(:all) do - run_binary_builder('httpd', '2.4.41', '--sha256=133d48298fe5315ae9366a0ec66282fa4040efa5d566174481077ade7d18ea40') - @binary_tarball_location = Dir.glob(File.join(Dir.pwd, 'httpd-2.4.41-linux-x64*.tgz')).first - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - httpd_version_cmd = %(env LD_LIBRARY_PATH=/tmp/binary-exerciser/lib ./spec/assets/binary-exerciser.sh #{File.basename(@binary_tarball_location)} ./httpd/bin/httpd -v) - - output, status = run(httpd_version_cmd) - - expect(status).to be_success - expect(output).to include('2.4.41') - end - - it 'copies in *.so files for some of the compiled extensions' do - expect(tar_contains_file('httpd/lib/libapr-1.so.0')).to eq true - expect(tar_contains_file('httpd/lib/libaprutil-1.so.0')).to eq true - expect(tar_contains_file('httpd/lib/libapriconv-1.so.0')).to eq true - expect(tar_contains_file('httpd/lib/apr-util-1/apr_ldap.so')).to eq true - expect(tar_contains_file('httpd/lib/iconv/utf-8.so')).to eq true - end - end -end diff --git a/cflinuxfs4/spec/integration/hwc_spec.rb b/cflinuxfs4/spec/integration/hwc_spec.rb deleted file mode 100644 index df7a77bf..00000000 --- a/cflinuxfs4/spec/integration/hwc_spec.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' -require 'tmpdir' - -describe 'building a binary', :integration do - context 'when hwc is specified' do - before(:all) do - run_binary_builder('hwc', '20.0.0', '--sha256=643fd1225881bd6206eec205ba818cf60be00bd3a1029c86b0e5bf74a3a978ab') - @binary_zip_location = File.join(Dir.pwd, 'hwc-20.0.0-windows-x86-64.zip') - @unzip_dir = Dir.mktmpdir - end - - after(:all) do - FileUtils.rm(@binary_zip_location) - FileUtils.rm_rf(@unzip_dir) - end - - it 'builds the specified binary, zips it, and places it in your current working directory' do - expect(File).to exist(@binary_zip_location) - - zip_file_cmd = 'file hwc-20.0.0-windows-x86-64.zip' - output, status = run(zip_file_cmd) - - expect(status).to be_success - expect(output).to include('Zip archive data') - end - - it 'builds a windows binary' do - Dir.chdir(@unzip_dir) do - FileUtils.cp(@binary_zip_location, Dir.pwd) - system 'unzip hwc-20.0.0-windows-x86-64.zip' - file_output = `file hwc.exe` - expect(file_output).to include('hwc.exe: PE32+ executable') - expect(file_output).to include('for MS Windows') - - file_output = `file hwc_x86.exe` - expect(file_output).to include('hwc_x86.exe: PE32 executable') - expect(file_output).to include('for MS Windows') - end - end - end -end diff --git a/cflinuxfs4/spec/integration/jruby_spec.rb b/cflinuxfs4/spec/integration/jruby_spec.rb deleted file mode 100644 index bcca71c0..00000000 --- a/cflinuxfs4/spec/integration/jruby_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when jruby is specified' do - before(:all) do - output = run_binary_builder('jruby', '9.2.8.0-ruby-2.5', '--sha256=287ae0e946c2d969613465c738cc3b09098f9f25805893ab707dce19a7b98c43') - @binary_tarball_location = File.join(Dir.pwd, 'jruby-9.2.8.0-ruby-2.5-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - jruby_version_cmd = './spec/assets/jruby-exerciser.sh' - output, status = run(jruby_version_cmd) - - expect(status).to be_success - expect(output).to include('java 2.5.3') - end - end -end diff --git a/cflinuxfs4/spec/integration/nodejs_spec.rb b/cflinuxfs4/spec/integration/nodejs_spec.rb deleted file mode 100644 index b88c75be..00000000 --- a/cflinuxfs4/spec/integration/nodejs_spec.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when node allows openssl-use-def-ca-store' do - before(:all) do - run_binary_builder('node', '8.8.1', '--sha256=1725bbbe623d6a13ee14522730dfc90eac1c9ebe9a0a8f4c3322a402dd7e75a2') - @binary_tarball_location = File.join(Dir.pwd, 'node-8.8.1-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - node_version_cmd = "./spec/assets/binary-exerciser.sh node-8.8.1-linux-x64.tgz node-v8.8.1-linux-x64/bin/node -e 'console.log(process.version)'" - - output, status = run(node_version_cmd) - - expect(status).to be_success - expect(output).to include('v8.8.1') - end - end - - context 'when node DOES NOT allow openssl-use-def-ca-store' do - before(:all) do - run_binary_builder('node', '4.8.5', '--sha256=23980b1d31c6b0e05eff2102ffa0059a6f7a93e27e5288eb5551b9b003ec0c07') - @binary_tarball_location = File.join(Dir.pwd, 'node-4.8.5-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - node_version_cmd = "./spec/assets/binary-exerciser.sh node-4.8.5-linux-x64.tgz node-v4.8.5-linux-x64/bin/node -e 'console.log(process.version)'" - - output, status = run(node_version_cmd) - - expect(status).to be_success - expect(output).to include('v4.8.5') - end - end -end diff --git a/cflinuxfs4/spec/integration/php7_with_oracle_spec.rb b/cflinuxfs4/spec/integration/php7_with_oracle_spec.rb deleted file mode 100644 index 1feab4ab..00000000 --- a/cflinuxfs4/spec/integration/php7_with_oracle_spec.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' -require 'open-uri' - -describe 'building a binary', :run_oracle_php_tests do - context 'when php7.4 is specified with oracle libraries' do - before(:all) do - extensions_file = File.join('spec', 'assets', 'php-extensions.yml') - - run_binary_builder('php', '7.4.0', "--sha256=004a1a8176176ee1b5c112e73d705977507803f425f9e48cb4a84f42b22abf22 --php-extensions-file=#{extensions_file}") - @binary_tarball_location = Dir.glob(File.join(Dir.pwd, 'php-7.4.0-linux-x64.tgz')).first - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'can load the oci8.so and pdo_oci.so PHP extensions' do - expect(File).to exist(@binary_tarball_location) - php_oracle_module_arguments = '-dextension=oci8.so -dextension=pdo_oci.so -dextension=pdo.so' - php_info_modules_command = '-r "phpinfo(INFO_MODULES);"' - - php_info_with_oracle_modules = %(./spec/assets/php-exerciser.sh #{File.basename(@binary_tarball_location)} ./php/bin/php #{php_oracle_module_arguments} #{php_info_modules_command}) - - output, status = run(php_info_with_oracle_modules) - - expect(status).to be_success - expect(output).to include('OCI8 Support => enabled') - expect(output).to include('PDO Driver for OCI 8 and later => enabled') - end - - it 'copies in the oracle *.so files ' do - expect(tar_contains_file('php/lib/libclntshcore.so.12.1')).to eq true - expect(tar_contains_file('php/lib/libclntsh.so')).to eq true - expect(tar_contains_file('php/lib/libclntsh.so.12.1')).to eq true - expect(tar_contains_file('php/lib/libipc1.so')).to eq true - expect(tar_contains_file('php/lib/libmql1.so')).to eq true - expect(tar_contains_file('php/lib/libnnz12.so')).to eq true - expect(tar_contains_file('php/lib/libociicus.so')).to eq true - expect(tar_contains_file('php/lib/libons.so')).to eq true - end - end -end diff --git a/cflinuxfs4/spec/integration/ruby_spec.rb b/cflinuxfs4/spec/integration/ruby_spec.rb deleted file mode 100644 index 38fae302..00000000 --- a/cflinuxfs4/spec/integration/ruby_spec.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when ruby is specified' do - before(:all) do - run_binary_builder('ruby', '2.6.5', '--sha256=66976b716ecc1fd34f9b7c3c2b07bbd37631815377a2e3e85a5b194cfdcbed7d') - @binary_tarball_location = File.join(Dir.pwd, 'ruby-2.6.5-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - ruby_version_cmd = "./spec/assets/binary-exerciser.sh ruby-2.6.5-linux-x64.tgz ./bin/ruby -e 'puts RUBY_VERSION'" - output, status = run(ruby_version_cmd) - - expect(status).to be_success - expect(output).to include('2.6.5') - - libgmp_cmd = './spec/assets/binary-exerciser.sh ruby-2.6.5-linux-x64.tgz grep LIBS= lib/pkgconfig/ruby-2.6.pc' - output, status = run(libgmp_cmd) - - expect(status).to be_success - expect(output).to include('LIBS=') - expect(output).not_to include('lgmp') - end - end -end diff --git a/cflinuxfs4/spec/integration/url_output_spec.rb b/cflinuxfs4/spec/integration/url_output_spec.rb deleted file mode 100644 index 7b20dcb9..00000000 --- a/cflinuxfs4/spec/integration/url_output_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'yaml' - -describe 'building a binary', :integration do - context 'when a recipe is specified' do - before(:all) do - @output, = run_binary_builder('glide', 'v0.11.0', '--sha256=7a7023aff20ba695706a262b8c07840ee28b939ea6358efbb69ab77da04f0052') - @binary_tarball_location = File.join(Dir.pwd, 'glide-v0.11.0-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'prints the url of the source used to build the binary to stdout' do - puts @output - expect(@output).to include('Source URL: https://github.com/Masterminds/glide/archive/v0.11.0.tar.gz') - end - end - - context 'when a meal is specified' do - before(:all) do - @output, = run_binary_builder('httpd', '2.4.41', '--sha256=133d48298fe5315ae9366a0ec66282fa4040efa5d566174481077ade7d18ea40') - @binary_tarball_location = Dir.glob(File.join(Dir.pwd, 'httpd-2.4.41-linux-x64*.tgz')).first - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'prints the url of the source used to build the binary to stdout' do - puts @output - expect(@output).to include('Source URL: https://archive.apache.org/dist/httpd/httpd-2.4.41.tar.bz2') - end - end -end diff --git a/cflinuxfs4/spec/integration/yaml_flag_spec.rb b/cflinuxfs4/spec/integration/yaml_flag_spec.rb deleted file mode 100644 index 311e5f07..00000000 --- a/cflinuxfs4/spec/integration/yaml_flag_spec.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require 'yaml' - -describe 'building a binary', :integration do - context 'when a recipe is specified' do - before(:all) do - @output, = run_binary_builder('go', '1.6.3', '--sha256=6326aeed5f86cf18f16d6dc831405614f855e2d416a91fd3fdc334f772345b00') - @tarball_name = 'go1.6.3.linux-amd64.tar.gz' - @binary_tarball_location = File.join(Dir.pwd, @tarball_name) - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'prints a yaml representation of the source used to build the binary to stdout' do - yaml_source = @output.match(/Source YAML:(.*)/m)[1] - expect(YAML.safe_load(yaml_source)).to eq([ - { - 'sha256' => '6326aeed5f86cf18f16d6dc831405614f855e2d416a91fd3fdc334f772345b00', - 'url' => 'https://storage.googleapis.com/golang/go1.6.3.src.tar.gz' - } - ]) - end - - it 'includes the yaml representation of the source inside the resulting tarball' do - yaml_source = `tar xzf #{@tarball_name} -O sources.yml` - expect(YAML.safe_load(yaml_source)).to eq([ - { - 'sha256' => '6326aeed5f86cf18f16d6dc831405614f855e2d416a91fd3fdc334f772345b00', - 'url' => 'https://storage.googleapis.com/golang/go1.6.3.src.tar.gz' - } - ]) - end - end - - context 'when a meal is specified' do - before(:all) do - @output, = run_binary_builder('httpd', '2.4.41', '--sha256=133d48298fe5315ae9366a0ec66282fa4040efa5d566174481077ade7d18ea40') - @binary_tarball_location = Dir.glob(File.join(Dir.pwd, 'httpd-2.4.41-linux-x64*.tgz')).first - end - - it 'prints a yaml representation of the source used to build the binary to stdout' do - yaml_source = @output.match(/Source YAML:(.*)/m)[1] - expect(YAML.safe_load(yaml_source)).to match_array([ - { - 'sha256' => '133d48298fe5315ae9366a0ec66282fa4040efa5d566174481077ade7d18ea40', - 'url' => 'https://archive.apache.org/dist/httpd/httpd-2.4.41.tar.bz2' - }, - { - 'sha256' => '48e9dbf45ae3fdc7b491259ffb6ccf7d63049ffacbc1c0977cced095e4c2d5a2', - 'url' => 'https://apache.osuosl.org/apr/apr-1.7.0.tar.gz' - }, - { - 'sha256' => 'ce94c7722ede927ce1e5a368675ace17d96d60ff9b8918df216ee5c1298c6a5e', - 'url' => 'https://apache.osuosl.org/apr/apr-iconv-1.2.2.tar.gz' - }, - { - 'sha256' => 'b65e40713da57d004123b6319828be7f1273fbc6490e145874ee1177e112c459', - 'url' => 'https://apache.osuosl.org/apr/apr-util-1.6.1.tar.gz' - }, - { - 'sha256' => '0f078444fed34085bc83e27eb3439556718f50dcea275307ffb66d498bdabb8f', - 'url' => 'https://github.com/zmartzone/mod_auth_openidc/releases/download/v2.3.8/mod_auth_openidc-2.3.8.tar.gz' - } - ]) - end - - it 'includes the yaml representation of the source inside the resulting tarball' do - yaml_source = `tar xzf httpd-2.4.41-linux-x64.tgz sources.yml -O` - expect(YAML.safe_load(yaml_source)).to match_array([ - { - 'sha256' => '133d48298fe5315ae9366a0ec66282fa4040efa5d566174481077ade7d18ea40', - 'url' => 'https://archive.apache.org/dist/httpd/httpd-2.4.41.tar.bz2' - }, - { - 'sha256' => '48e9dbf45ae3fdc7b491259ffb6ccf7d63049ffacbc1c0977cced095e4c2d5a2', - 'url' => 'https://apache.osuosl.org/apr/apr-1.7.0.tar.gz' - }, - { - 'sha256' => 'ce94c7722ede927ce1e5a368675ace17d96d60ff9b8918df216ee5c1298c6a5e', - 'url' => 'https://apache.osuosl.org/apr/apr-iconv-1.2.2.tar.gz' - }, - { - 'sha256' => 'b65e40713da57d004123b6319828be7f1273fbc6490e145874ee1177e112c459', - 'url' => 'https://apache.osuosl.org/apr/apr-util-1.6.1.tar.gz' - }, - { - 'sha256' => '0f078444fed34085bc83e27eb3439556718f50dcea275307ffb66d498bdabb8f', - 'url' => 'https://github.com/zmartzone/mod_auth_openidc/releases/download/v2.3.8/mod_auth_openidc-2.3.8.tar.gz' - } - ]) - end - end -end diff --git a/cflinuxfs4/spec/spec_helper.rb b/cflinuxfs4/spec/spec_helper.rb deleted file mode 100644 index a8a79f56..00000000 --- a/cflinuxfs4/spec/spec_helper.rb +++ /dev/null @@ -1,143 +0,0 @@ -# frozen_string_literal: true - -require 'fileutils' -require 'open3' -require 'tmpdir' - -RSpec.configure do |config| - config.color = true - config.tty = true - - if RUBY_PLATFORM.include?('darwin') - DOCKER_CONTAINER_NAME = "test-suite-binary-builder-#{Time.now.to_i}".freeze - - config.before(:all, :integration) do - directory_mapping = "-v #{Dir.pwd}:/binary-builder" - setup_docker_container(DOCKER_CONTAINER_NAME, directory_mapping) - end - - config.after(:all, :integration) do - cleanup_docker_artifacts(DOCKER_CONTAINER_NAME) - end - - config.before(:all, :run_oracle_php_tests) do - dir_to_contain_oracle = File.join(Dir.pwd, 'oracle_client_libs') - FileUtils.mkdir_p(dir_to_contain_oracle) - setup_oracle_libs(dir_to_contain_oracle) - - oracle_dir = File.join(dir_to_contain_oracle, 'oracle') - directory_mapping = "-v #{Dir.pwd}:/binary-builder -v #{oracle_dir}:/oracle" - setup_docker_container(DOCKER_CONTAINER_NAME, directory_mapping) - end - - config.after(:all, :run_oracle_php_tests) do - cleanup_docker_artifacts(DOCKER_CONTAINER_NAME) - - dir_containing_oracle = File.join(Dir.pwd, 'oracle_client_libs') - FileUtils.rm_rf(dir_containing_oracle) - end - - config.before(:all, :run_geolite_php_tests) do - directory_mapping = "-v #{Dir.pwd}:/binary-builder" - setup_docker_container(DOCKER_CONTAINER_NAME, directory_mapping) - - file_to_enable_geolite_db = File.join(Dir.pwd, 'BUNDLE_GEOIP_LITE') - File.open(file_to_enable_geolite_db, 'w') { |f| f.puts 'true' } - end - - config.after(:all, :run_geolite_php_tests) do - cleanup_docker_artifacts(DOCKER_CONTAINER_NAME) - - file_to_enable_geolite_db = File.join(Dir.pwd, 'BUNDLE_GEOIP_LITE') - FileUtils.rm(file_to_enable_geolite_db) - end - else - config.before(:all, :run_oracle_php_tests) do - setup_oracle_libs('/') - end - - config.before(:all, :run_geolite_php_tests) do - file_to_enable_geolite_db = File.join(Dir.pwd, 'BUNDLE_GEOIP_LITE') - File.open(file_to_enable_geolite_db, 'w') { |f| f.puts 'true' } - end - - config.after(:all, :run_geolite_php_tests) do - file_to_enable_geolite_db = File.join(Dir.pwd, 'BUNDLE_GEOIP_LITE') - FileUtils.rm(file_to_enable_geolite_db) - end - end - - def cleanup_docker_artifacts(docker_container_name) - `docker stop #{docker_container_name}` - `docker rm #{docker_container_name}` - - Dir['*deb*'].each do |deb_file| - FileUtils.rm(deb_file) - end - end - - def setup_oracle_libs(dir_to_contain_oracle) - Dir.chdir(dir_to_contain_oracle) do - s3_bucket = ENV['ORACLE_LIBS_AWS_BUCKET'] - libs_filename = ENV['ORACLE_LIBS_FILENAME'] - - ## If AWS_ASSUME_ROLE_ARN is provides, switch to aws assume-role mode - if ENV['AWS_ASSUME_ROLE_ARN'] && !ENV['AWS_ASSUME_ROLE_ARN'].empty? - system <<-eof - uuid=$(cat /proc/sys/kernel/random/uuid) - RESULT="$(aws sts assume-role --role-arn "${AWS_ASSUME_ROLE_ARN}" --role-session-name "binary-builder-spec-${uuid}")" - export AWS_ACCESS_KEY_ID="$(echo "${RESULT}" |jq -r .Credentials.AccessKeyId)" - export AWS_SECRET_ACCESS_KEY="$(echo "${RESULT}" |jq -r .Credentials.SecretAccessKey)" - export AWS_SESSION_TOKEN="$(echo "${RESULT}" |jq -r .Credentials.SessionToken)" - aws s3 cp s3://#{s3_bucket}/#{libs_filename} . - eof - else - system "aws s3 cp s3://#{s3_bucket}/#{libs_filename} ." - end - system "tar -xvf #{libs_filename}" - end - end - - def setup_docker_container(docker_container_name, directory_mapping) - docker_image = "cloudfoundry/#{ENV.fetch('STACK', 'cflinuxfs3')}" - `docker run --name #{docker_container_name} -dit #{directory_mapping} -e CCACHE_DIR=/binary-builder/.ccache -w /binary-builder #{docker_image} sh -c 'env PATH=/usr/lib/ccache:$PATH bash'` - `docker exec #{docker_container_name} apt-get -y install ccache` - `docker exec #{docker_container_name} gem install bundler --no-ri --no-rdoc` - `docker exec #{docker_container_name} bundle install -j4` - end - - def run(cmd) - cmd = "docker exec #{DOCKER_CONTAINER_NAME} #{cmd}" if RUBY_PLATFORM.include?('darwin') - - Bundler.with_clean_env do - Open3.capture2e(cmd).tap do |output, status| - expect(status).to be_success, (lambda do - puts "command output: #{output}" - puts "expected command to return a success status code, got: #{status}" - end) - end - end - end - - def run_binary_builder(binary_name, binary_version, flags) - binary_builder_cmd = "bundle exec ./bin/binary-builder --name=#{binary_name} --version=#{binary_version} #{flags}" - run(binary_builder_cmd) - end - - def tar_contains_file(filename) - expect(@binary_tarball_location).to be - - o, status = Open3.capture2e("tar --wildcards --list --verbose -f #{@binary_tarball_location} #{filename}") - return false unless status.success? - - o.split(/[\r\n]+/).all? do |line| - m = line.match(/(\S+) -> (\S+)$/) - return true unless m - - oldfile, newfile = m[1, 2] - return false if newfile.start_with?('/') - - tar_contains_file("#{File.dirname(oldfile)}/#{newfile}") - end - end -end diff --git a/cflinuxfs4/spec/unit/archive_recipe_spec.rb b/cflinuxfs4/spec/unit/archive_recipe_spec.rb deleted file mode 100644 index 84b562e6..00000000 --- a/cflinuxfs4/spec/unit/archive_recipe_spec.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_relative '../../lib/archive_recipe' -require_relative '../../recipe/base' - -describe ArchiveRecipe do - class FakeRecipe < BaseRecipe - def url; end - - def archive_files - [1] - end - end - - context 'when the recipe has #setup_tar' do - it 'invokes' do - recipe = FakeRecipe.new('fake', '1.1.1') - def recipe.setup_tar; end - allow(YAMLPresenter).to receive(:new).and_return('') - - expect(recipe).to receive(:setup_tar) - described_class.new(recipe).compress! - end - end - - context 'when the recipe does not have #setup_tar' do - it 'does not invoke' do - recipe = FakeRecipe.new('fake', '1.1.1') - allow(YAMLPresenter).to receive(:new).and_return('') - - expect do - described_class.new(recipe).compress! - end.not_to raise_error - end - end -end diff --git a/cflinuxfs4/spec/unit/yaml_spec.rb b/cflinuxfs4/spec/unit/yaml_spec.rb deleted file mode 100644 index 7a5f19cb..00000000 --- a/cflinuxfs4/spec/unit/yaml_spec.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_relative '../../lib/yaml_presenter' - -describe YAMLPresenter do - it 'encodes the SHA256 as a raw string' do - recipe = double(:recipe, files_hashs: [ - { - local_path: File.expand_path(__FILE__) - } - ]) - presenter = described_class.new(recipe) - expect(presenter.to_yaml).not_to include "!binary |-\n" - end - - context 'the source is a github repo' do - it 'displays the git commit sha' do - recipe = double(:recipe, files_hashs: [ - { - git: { commit_sha: 'a_mocked_commit_sha' }, - local_path: File.expand_path(__FILE__) - } - ]) - presenter = described_class.new(recipe) - expect(presenter.to_yaml).not_to include "!binary |-\n" - expect(presenter.to_yaml).to include 'a_mocked_commit_sha' - end - end -end diff --git a/cmd/binary-builder/main.go b/cmd/binary-builder/main.go new file mode 100644 index 00000000..72574fda --- /dev/null +++ b/cmd/binary-builder/main.go @@ -0,0 +1,371 @@ +// Command binary-builder builds a single dependency for a given CF stack. +// +// Two input modes are supported: +// +// Mode 1 — Direct flags (manual/local use): +// +// binary-builder build \ +// --stack cflinuxfs4 \ +// --name ruby \ +// --version 3.3.6 \ +// --url https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.6.tar.gz \ +// --sha256 8dc48f... +// +// Mode 2 — Source file (CI use, depwatcher data.json): +// +// binary-builder build \ +// --stack cflinuxfs4 \ +// --source-file source/data.json +// +// Selection logic: +// - If --name is provided → build source.Input directly from flags +// (--version is required; --url/--sha256/--sha512 are optional) +// - Else if --source-file is explicitly given OR source/data.json exists +// at the default path → read from file +// - Else → error: provide either --name/--version or --source-file +// +// The tool compiles the dependency inside a temp directory and writes the +// final artifact to the current working directory using the canonical filename: +// +// _____. +// +// On success it writes a JSON summary to --output-file (default: summary.json): +// +// { +// "artifact_path": "ruby_3.3.6_linux_x64_cflinuxfs4_abcdef01.tgz", +// "version": "3.3.6", +// "sha256": "abcdef01...", +// "url": "https://buildpacks.cloudfoundry.org/dependencies/ruby/ruby_3.3.6_...", +// "source": {"url": "...", "sha256": "...", ...}, +// "sub_dependencies": {...} +// } +// +// All build subprocess output goes to stdout/stderr and is visible in logs. +// The JSON summary is always written to a file, never to stdout, so that +// build noise from compilers and make does not corrupt the structured output. +// +// All artifact renaming, dep-metadata writing, builds-artifacts JSON, and git +// commits are the responsibility of the CI task that wraps this tool. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/artifact" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/fileutil" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/recipe" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +const defaultOutputFile = "summary.json" + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "binary-builder: %v\n", err) + os.Exit(1) + } +} + +func run() error { + if len(os.Args) < 2 || os.Args[1] != "build" { + return fmt.Errorf("usage: binary-builder build [flags]") + } + + fs := flag.NewFlagSet("build", flag.ExitOnError) + + // Required. + stackName := fs.String("stack", "", "Stack name (e.g. cflinuxfs4, cflinuxfs5) [required]") + + // Mode 1 — direct flags (manual/local use). + name := fs.String("name", "", "Dependency name (e.g. ruby); triggers direct-input mode") + version := fs.String("version", "", "Version string (required when --name is set)") + url := fs.String("url", "", "Source tarball URL (optional with --name)") + sha256 := fs.String("sha256", "", "SHA256 of the source tarball (optional with --name)") + sha512 := fs.String("sha512", "", "SHA512 of the source tarball (optional with --name)") + + // Mode 2 — source file (CI / depwatcher use). + sourceFile := fs.String("source-file", "source/data.json", "Path to depwatcher data.json") + + stacksDir := fs.String("stacks-dir", "stacks", "Directory containing stack YAML files") + + // Output — JSON summary is always written to a file, never to stdout. + // Build subprocess output (compilers, make, etc.) flows to stdout/stderr + // so it is visible in logs without corrupting the structured JSON output. + outputFile := fs.String("output-file", defaultOutputFile, "Path to write the JSON build summary") + + if err := fs.Parse(os.Args[2:]); err != nil { + return err + } + + if *stackName == "" { + return fmt.Errorf("--stack is required") + } + + // Determine whether --source-file was explicitly passed. + sourceFileExplicit := false + fs.Visit(func(f *flag.Flag) { + if f.Name == "source-file" { + sourceFileExplicit = true + } + }) + + // Resolve source input using the agreed mode-selection logic. + var src *source.Input + switch { + case *name != "": + // Mode 1: build source.Input directly from flags. + if *version == "" { + return fmt.Errorf("--version is required when --name is set") + } + src = &source.Input{ + Name: *name, + Version: *version, + URL: *url, + SHA256: *sha256, + SHA512: *sha512, + } + + case sourceFileExplicit: + // Mode 2a: --source-file was explicitly provided. + var err error + src, err = source.FromFile(*sourceFile) + if err != nil { + return fmt.Errorf("loading source file: %w", err) + } + + default: + // Mode 2b: check whether the default path exists on disk. + if _, statErr := os.Stat(*sourceFile); statErr == nil { + var err error + src, err = source.FromFile(*sourceFile) + if err != nil { + return fmt.Errorf("loading source file: %w", err) + } + } else { + return fmt.Errorf("provide either --name (with --version) or --source-file") + } + } + + // Load stack config. + s, err := stack.Load(*stacksDir, *stackName) + if err != nil { + return fmt.Errorf("loading stack %q: %w", *stackName, err) + } + + // Look up the recipe. + reg := buildRegistry() + rec, err := reg.Get(src.Name) + if err != nil { + return fmt.Errorf("no recipe for %q — registered: %v", src.Name, reg.Names()) + } + + // Prepare output data seeded from the source input. + outData := output.NewOutData(src) + + // Run the build. + ctx := context.Background() + r := &runner.RealRunner{} + + fmt.Fprintf(os.Stderr, "[binary-builder] building %s %s for %s\n", src.Name, src.Version, *stackName) + + if err := rec.Build(ctx, s, src, r, outData); err != nil { + return fmt.Errorf("building %s: %w", src.Name, err) + } + + // Miniconda sets outData.URL directly (passthrough — no compiled artifact). + // All other recipes produce an intermediate file in the working directory. + if outData.URL == "" { + if err := finalizeArtifact(src, rec, s, outData); err != nil { + return err + } + } + + // Write the JSON summary to the output file. + // Build subprocess output has already flowed to stdout/stderr (visible in + // logs), so the output file contains only the clean structured JSON. + f, err := os.Create(*outputFile) + if err != nil { + return fmt.Errorf("creating output file %s: %w", *outputFile, err) + } + defer f.Close() + + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + if err := enc.Encode(buildSummary(outData)); err != nil { + return fmt.Errorf("writing JSON summary: %w", err) + } + + fmt.Fprintf(os.Stderr, "[binary-builder] summary written to %s\n", *outputFile) + return nil +} + +// buildSummaryOutput is the JSON struct emitted to stdout. +type buildSummaryOutput struct { + ArtifactPath string `json:"artifact_path"` + Version string `json:"version"` + SHA256 string `json:"sha256,omitempty"` + URL string `json:"url,omitempty"` + Source output.OutDataSource `json:"source"` + SubDependencies map[string]output.SubDependency `json:"sub_dependencies,omitempty"` + GitCommitSHA string `json:"git_commit_sha,omitempty"` +} + +func buildSummary(outData *output.OutData) buildSummaryOutput { + return buildSummaryOutput{ + ArtifactPath: filepath.Base(outData.URL), + Version: outData.Version, + SHA256: outData.SHA256, + URL: outData.URL, + Source: outData.Source, + SubDependencies: outData.SubDependencies, + GitCommitSHA: outData.GitCommitSHA, + } +} + +// finalizeArtifact finds the intermediate artifact file written to CWD by the +// recipe, computes its SHA256, renames it to the canonical filename, and +// populates outData.SHA256 and outData.URL. +func finalizeArtifact(src *source.Input, rec recipe.Recipe, s *stack.Stack, outData *output.OutData) error { + meta := rec.Artifact() + + // Resolve the effective stack label for the artifact filename. + stackLabel := meta.Stack + if stackLabel == "" { + stackLabel = s.Name + } + + // Use ArtifactVersion when set (e.g. jruby uses "9.4.14.0-ruby-3.1"); + // otherwise fall back to the raw source version. + artifactVersion := outData.ArtifactVersion + if artifactVersion == "" { + artifactVersion = outData.Version + } + + // Find the intermediate artifact produced by the recipe. + intermediatePath, err := findIntermediateArtifact(src.Name, artifactVersion) + if err != nil { + return err + } + + ext := artifact.ExtFromPath(intermediatePath) + + // Compute SHA256. + sha256hex, err := artifact.SHA256File(intermediatePath) + if err != nil { + return fmt.Errorf("computing SHA256 of %s: %w", intermediatePath, err) + } + + // Build canonical filename and move artifact to CWD. + a := artifact.Artifact{ + Name: src.Name, + Version: artifactVersion, + OS: meta.OS, + Arch: meta.Arch, + Stack: stackLabel, + } + finalFilename := a.Filename(sha256hex, ext) + finalPath := filepath.Join(".", finalFilename) + + // Use cross-device-safe move in case the recipe wrote to os.TempDir. + if err := fileutil.MoveFile(intermediatePath, finalPath); err != nil { + return fmt.Errorf("moving artifact to %s: %w", finalPath, err) + } + + outData.SHA256 = sha256hex + outData.URL = a.S3URL(finalFilename) + + fmt.Fprintf(os.Stderr, "[binary-builder] artifact: %s\n", finalFilename) + return nil +} + +// buildRegistry constructs and populates the full recipe registry. +func buildRegistry() *recipe.Registry { + f := fetch.NewHTTPFetcher() + reg := recipe.NewRegistry() + + // Compiled recipes. + reg.Register(&recipe.RubyRecipe{Fetcher: f}) + reg.Register(&recipe.BundlerRecipe{Fetcher: f}) + reg.Register(&recipe.PythonRecipe{Fetcher: f}) + reg.Register(&recipe.NodeRecipe{Fetcher: f}) + reg.Register(&recipe.GoRecipe{Fetcher: f}) + reg.Register(&recipe.NginxRecipe{Fetcher: f}) + reg.Register(&recipe.NginxStaticRecipe{Fetcher: f}) + reg.Register(&recipe.OpenrestyRecipe{Fetcher: f}) + reg.Register(&recipe.HTTPDRecipe{Fetcher: f}) + reg.Register(&recipe.JRubyRecipe{Fetcher: f}) + reg.Register(&recipe.RRecipe{Fetcher: f}) + reg.Register(&recipe.LibunwindRecipe{}) + reg.Register(&recipe.LibgdiplusRecipe{}) + reg.Register(&recipe.DepRecipe{Fetcher: f}) + reg.Register(&recipe.GlideRecipe{Fetcher: f}) + reg.Register(&recipe.GodepRecipe{Fetcher: f}) + reg.Register(&recipe.HWCRecipe{Fetcher: f}) + + // PHP recipe. + reg.Register(&recipe.PHPRecipe{Fetcher: f}) + + // Simple / repack recipes. + reg.Register(&recipe.PipRecipe{Fetcher: f}) + reg.Register(&recipe.PipenvRecipe{Fetcher: f}) + reg.Register(&recipe.BowerRecipe{Fetcher: f}) + reg.Register(&recipe.YarnRecipe{Fetcher: f}) + reg.Register(&recipe.RubygemsRecipe{Fetcher: f}) + reg.Register(&recipe.MinicondaRecipe{Fetcher: f}) + reg.Register(&recipe.DotnetSDKRecipe{}) + reg.Register(&recipe.DotnetRuntimeRecipe{}) + reg.Register(&recipe.DotnetAspnetcoreRecipe{}) + + // Passthrough recipes. + for _, r := range recipe.NewPassthroughRecipes(f) { + reg.Register(r) + } + + return reg +} + +// findIntermediateArtifact searches CWD (and os.TempDir as fallback) for the +// artifact file produced by a recipe. Extensions are tried in priority order. +func findIntermediateArtifact(name, version string) (string, error) { + // Extensions in priority order (matches ArtifactOutput.ext in the old Ruby code). + exts := []string{"tgz", "tar.gz", "zip", "tar.xz", "tar.bz2", "sh", "phar", "txt"} + + // Recipes that cannot write to CWD (e.g. pip, pipenv) write to os.TempDir. + searchDirs := []string{".", os.TempDir()} + + for _, dir := range searchDirs { + // name-version prefix first (most specific). + for _, ext := range exts { + pattern := filepath.Join(dir, fmt.Sprintf("%s-%s*.%s", name, version, ext)) + matches, err := filepath.Glob(pattern) + if err != nil { + return "", fmt.Errorf("globbing %s: %w", pattern, err) + } + if len(matches) > 0 { + return matches[0], nil + } + } + // Fallback: just name prefix. + for _, ext := range exts { + pattern := filepath.Join(dir, fmt.Sprintf("%s-*.%s", name, ext)) + matches, err := filepath.Glob(pattern) + if err != nil { + return "", fmt.Errorf("globbing %s: %w", pattern, err) + } + if len(matches) > 0 { + return matches[0], nil + } + } + } + + return "", fmt.Errorf("no intermediate artifact file found for %s %s", name, version) +} diff --git a/go-version.yml b/go-version.yml deleted file mode 100644 index 8a3be269..00000000 --- a/go-version.yml +++ /dev/null @@ -1,3 +0,0 @@ -go: - - version: 1.20.1 - sha256: 000a5b1fca4f75895f78befeb2eecf10bfff3c428597f3f1e69133b63b911b02 diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..7203cdbe --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/cloudfoundry/binary-builder + +go 1.25.7 + +require ( + github.com/stretchr/testify v1.11.1 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..c4c1710c --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/apt/apt.go b/internal/apt/apt.go new file mode 100644 index 00000000..45eb545a --- /dev/null +++ b/internal/apt/apt.go @@ -0,0 +1,84 @@ +// Package apt provides a wrapper around apt-get for installing packages. +package apt + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/runner" +) + +// APT wraps apt-get operations using an injected Runner. +type APT struct { + Runner runner.Runner +} + +// New creates an APT instance with the given runner. +func New(r runner.Runner) *APT { + return &APT{Runner: r} +} + +// Update runs apt-get update. +func (a *APT) Update(_ context.Context) error { + return a.Runner.RunWithEnv( + map[string]string{"DEBIAN_FRONTEND": "noninteractive"}, + "apt-get", "update", + ) +} + +// Install runs apt-get install -y for the given packages. +// Does nothing if no packages are provided. +func (a *APT) Install(_ context.Context, packages ...string) error { + if len(packages) == 0 { + return nil + } + + args := append([]string{"install", "-y"}, packages...) + return a.Runner.RunWithEnv( + map[string]string{"DEBIAN_FRONTEND": "noninteractive"}, + "apt-get", args..., + ) +} + +// AddPPA adds a PPA repository and runs apt-get update. +// If ppa is empty, this is a no-op (cflinuxfs5 does not need PPAs). +func (a *APT) AddPPA(_ context.Context, ppa string) error { + if ppa == "" { + return nil + } + + if err := a.Runner.Run("add-apt-repository", "-y", ppa); err != nil { + return fmt.Errorf("adding PPA %s: %w", ppa, err) + } + + return a.Runner.RunWithEnv( + map[string]string{"DEBIAN_FRONTEND": "noninteractive"}, + "apt-get", "update", + ) +} + +// InstallReinstall runs apt-get -d install --reinstall to download .deb files +// without installing them. Used by the Python recipe for tcl/tk debs. +// +// When useForceYes is true, passes --force-yes (cflinuxfs4 compatibility). +// When false, passes --yes (cflinuxfs5 / modern apt). +func (a *APT) InstallReinstall(_ context.Context, useForceYes bool, packages ...string) error { + if len(packages) == 0 { + return nil + } + + var forceFlag string + if useForceYes { + forceFlag = "--force-yes" + } else { + forceFlag = "--yes" + } + + args := []string{forceFlag, "-d", "install", "--reinstall"} + args = append(args, packages...) + + return a.Runner.RunWithEnv( + map[string]string{"DEBIAN_FRONTEND": "noninteractive"}, + "apt-get", args..., + ) +} diff --git a/internal/apt/apt_test.go b/internal/apt/apt_test.go new file mode 100644 index 00000000..37510147 --- /dev/null +++ b/internal/apt/apt_test.go @@ -0,0 +1,111 @@ +package apt_test + +import ( + "context" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInstallPackages(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + err := a.Install(context.Background(), "pkg1", "pkg2") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "apt-get", f.Calls[0].Name) + assert.Equal(t, []string{"install", "-y", "pkg1", "pkg2"}, f.Calls[0].Args) + assert.Equal(t, "noninteractive", f.Calls[0].Env["DEBIAN_FRONTEND"]) +} + +func TestInstallNoPackages(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + err := a.Install(context.Background()) + require.NoError(t, err) + + assert.Empty(t, f.Calls) +} + +func TestUpdate(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + err := a.Update(context.Background()) + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "apt-get", f.Calls[0].Name) + assert.Equal(t, []string{"update"}, f.Calls[0].Args) + assert.Equal(t, "noninteractive", f.Calls[0].Env["DEBIAN_FRONTEND"]) +} + +func TestAddPPANonEmpty(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + err := a.AddPPA(context.Background(), "ppa:ubuntu-toolchain-r/test") + require.NoError(t, err) + + require.Len(t, f.Calls, 2) + // First call: add-apt-repository + assert.Equal(t, "add-apt-repository", f.Calls[0].Name) + assert.Equal(t, []string{"-y", "ppa:ubuntu-toolchain-r/test"}, f.Calls[0].Args) + // Second call: apt-get update + assert.Equal(t, "apt-get", f.Calls[1].Name) + assert.Equal(t, []string{"update"}, f.Calls[1].Args) +} + +func TestAddPPAEmpty(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + err := a.AddPPA(context.Background(), "") + require.NoError(t, err) + + assert.Empty(t, f.Calls, "empty PPA should be a no-op") +} + +func TestInstallReinstallWithForceYes(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + err := a.InstallReinstall(context.Background(), true, "libtcl8.6", "libtk8.6", "libxss1") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "apt-get", f.Calls[0].Name) + assert.Equal(t, []string{"--force-yes", "-d", "install", "--reinstall", "libtcl8.6", "libtk8.6", "libxss1"}, f.Calls[0].Args) +} + +func TestInstallReinstallWithoutForceYes(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + err := a.InstallReinstall(context.Background(), false, "libtcl8.6", "libtk8.6", "libxss1") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "apt-get", f.Calls[0].Name) + assert.Equal(t, []string{"--yes", "-d", "install", "--reinstall", "libtcl8.6", "libtk8.6", "libxss1"}, f.Calls[0].Args) + // Verify --force-yes is NOT present + for _, arg := range f.Calls[0].Args { + assert.NotEqual(t, "--force-yes", arg) + } +} + +func TestInstallReinstallNoPackages(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + err := a.InstallReinstall(context.Background(), true) + require.NoError(t, err) + + assert.Empty(t, f.Calls) +} diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 00000000..b558029e --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,334 @@ +// Package archive provides helpers for creating and manipulating tar/zip archives. +package archive + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/runner" +) + +// Pack creates a gzipped tarball at outputPath from the given directory. +// pathName is the top-level directory name inside the tarball ("" for flat pack of dir contents). +func Pack(r runner.Runner, outputPath, sourceDir, pathName string) error { + args := []string{"czf", outputPath} + if pathName != "" { + args = append(args, "-C", sourceDir, pathName) + } else { + args = append(args, "-C", sourceDir, ".") + } + + return r.Run("tar", args...) +} + +// PackWithDereference creates a gzipped tarball with --hard-dereference. +// Used by the Python recipe to resolve symlinks. +func PackWithDereference(r runner.Runner, outputPath, sourceDir string) error { + return r.RunInDir(sourceDir, "tar", "zcvf", outputPath, "--hard-dereference", ".") +} + +// PackXZ creates an xz-compressed tarball. Used by dotnet recipes. +func PackXZ(r runner.Runner, outputPath, sourceDir string) error { + return r.RunInDir(sourceDir, "tar", "-Jcf", outputPath, ".") +} + +// PackZip creates a zip archive from the given directory. +func PackZip(r runner.Runner, outputPath, sourceDir string) error { + return r.RunInDir(sourceDir, "zip", outputPath, "-r", ".") +} + +// StripTopLevelDir re-archives a tarball without its top-level directory. +// e.g. "node-v20.11.0/bin/node" → "./bin/node" +// +// The output uses "./" prefixed paths and includes directory entries, +// matching the output of `tar -czf out.tgz -C dir .` which is what +// the Ruby builder produces. +// +// Strategy: extract to a temp dir, then re-archive with `tar -C dir .` +// so that directory entries are synthesised for every subdirectory. +func StripTopLevelDir(path string) error { + // Step 1: extract the tarball into a temp directory, stripping the top-level dir. + tmpDir, err := os.MkdirTemp("", "strip-top-level-*") + if err != nil { + return fmt.Errorf("creating temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + + gr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("opening gzip %s: %w", path, err) + } + defer gr.Close() + + tr := tar.NewReader(gr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("reading tar entry from %s: %w", path, err) + } + + // Strip the first real path component. + // Entries may start with "./" (e.g. produced by `tar -C dir .`), so + // normalise by removing any leading "./" before splitting. + name := strings.TrimPrefix(hdr.Name, "./") + parts := strings.SplitN(name, "/", 2) + if len(parts) < 2 || parts[1] == "" { + // Top-level directory entry itself — skip. + continue + } + stripped := parts[1] + + target := filepath.Join(tmpDir, filepath.FromSlash(stripped)) + + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, os.FileMode(hdr.Mode)|0700); err != nil { + return fmt.Errorf("mkdir %s: %w", target, err) + } + case tar.TypeReg, tar.TypeRegA: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return fmt.Errorf("mkdir parent of %s: %w", target, err) + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)|0600) + if err != nil { + return fmt.Errorf("creating %s: %w", target, err) + } + if _, err := io.Copy(f, tr); err != nil { + f.Close() + return fmt.Errorf("writing %s: %w", target, err) + } + f.Close() + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return fmt.Errorf("mkdir parent of symlink %s: %w", target, err) + } + if err := os.Symlink(hdr.Linkname, target); err != nil && !os.IsExist(err) { + return fmt.Errorf("symlink %s → %s: %w", target, hdr.Linkname, err) + } + case tar.TypeLink: + // Strip the top-level directory from the hard link target too. + linkName := strings.TrimPrefix(hdr.Linkname, "./") + linkParts := strings.SplitN(linkName, "/", 2) + var strippedLink string + if len(linkParts) >= 2 { + strippedLink = linkParts[1] + } else { + strippedLink = linkName + } + linkTarget := filepath.Join(tmpDir, filepath.FromSlash(strippedLink)) + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return fmt.Errorf("mkdir parent of hardlink %s: %w", target, err) + } + if err := os.Link(linkTarget, target); err != nil && !os.IsExist(err) { + return fmt.Errorf("hardlink %s → %s: %w", target, linkTarget, err) + } + } + } + + // Step 2: re-archive with `tar -czf -C .` + // This produces "./" prefixed paths and emits directory entries for every + // subdirectory — identical to what Ruby's Archive.strip_top_level_directory_from_tar does. + absPath, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("resolving output path: %w", err) + } + + cmd := exec.Command("tar", "-czf", absPath, "-C", tmpDir, ".") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("re-archiving %s: %w", path, err) + } + + return nil +} + +// StripTopLevelDirFromZip re-archives a zip without its top-level directory. +// Used by setuptools which may ship as .zip. +func StripTopLevelDirFromZip(path string) error { + r, err := zip.OpenReader(path) + if err != nil { + return fmt.Errorf("opening zip %s: %w", path, err) + } + defer r.Close() + + var buf bytes.Buffer + w := zip.NewWriter(&buf) + + for _, f := range r.File { + parts := strings.SplitN(f.Name, "/", 2) + if len(parts) < 2 || parts[1] == "" { + continue + } + + newHeader := f.FileHeader + newHeader.Name = parts[1] + + writer, err := w.CreateHeader(&newHeader) + if err != nil { + return fmt.Errorf("creating zip entry %s: %w", newHeader.Name, err) + } + + if !f.FileInfo().IsDir() { + reader, err := f.Open() + if err != nil { + return fmt.Errorf("opening zip entry %s: %w", f.Name, err) + } + if _, err := io.Copy(writer, reader); err != nil { + reader.Close() + return fmt.Errorf("copying zip entry %s: %w", f.Name, err) + } + reader.Close() + } + } + + if err := w.Close(); err != nil { + return fmt.Errorf("closing zip writer: %w", err) + } + + return os.WriteFile(path, buf.Bytes(), 0644) +} + +// StripFiles removes files matching a glob pattern from inside a tarball. +func StripFiles(path string, pattern string) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + + gr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("opening gzip %s: %w", path, err) + } + defer gr.Close() + + tr := tar.NewReader(gr) + + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("reading tar entry: %w", err) + } + + matched, _ := filepath.Match(pattern, filepath.Base(hdr.Name)) + if matched { + // Skip this entry — strip it from the archive. + if hdr.Typeflag == tar.TypeReg { + io.Copy(io.Discard, tr) + } + continue + } + + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("writing header: %w", err) + } + if hdr.Typeflag == tar.TypeReg { + if _, err := io.Copy(tw, tr); err != nil { + return fmt.Errorf("copying data: %w", err) + } + } + } + + if err := tw.Close(); err != nil { + return err + } + if err := gw.Close(); err != nil { + return err + } + + return os.WriteFile(path, buf.Bytes(), 0644) +} + +// StripIncorrectWordsYAML removes incorrect_words.yaml from a tarball +// and from any nested .jar files within it. +// Used by ruby and jruby recipes. +func StripIncorrectWordsYAML(path string) error { + return StripFiles(path, "incorrect_words.yaml") +} + +// InjectFile adds a file with the given name and content into an existing +// gzipped tarball. The file is appended at the archive root (no directory +// prefix). Typically used to inject sources.yml into an artifact tarball. +func InjectFile(tarPath, filename string, content []byte) error { + data, err := os.ReadFile(tarPath) + if err != nil { + return fmt.Errorf("reading %s: %w", tarPath, err) + } + + gr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("opening gzip %s: %w", tarPath, err) + } + defer gr.Close() + + tr := tar.NewReader(gr) + + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + + // Copy all existing entries. + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("reading tar entry: %w", err) + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("writing header: %w", err) + } + if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA { + if _, err := io.Copy(tw, tr); err != nil { + return fmt.Errorf("copying data: %w", err) + } + } + } + + // Append the new file at the archive root. + hdr := &tar.Header{ + Name: "./" + filename, + Mode: 0644, + Size: int64(len(content)), + Typeflag: tar.TypeReg, + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("writing injected header: %w", err) + } + if _, err := tw.Write(content); err != nil { + return fmt.Errorf("writing injected content: %w", err) + } + + if err := tw.Close(); err != nil { + return err + } + if err := gw.Close(); err != nil { + return err + } + + return os.WriteFile(tarPath, buf.Bytes(), 0644) +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go new file mode 100644 index 00000000..3b2671cb --- /dev/null +++ b/internal/archive/archive_test.go @@ -0,0 +1,278 @@ +package archive_test + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "io" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func createTestTarball(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0644, + Size: int64(len(content)), + Typeflag: tar.TypeReg, + } + require.NoError(t, tw.WriteHeader(hdr)) + _, err := tw.Write([]byte(content)) + require.NoError(t, err) + } + + require.NoError(t, tw.Close()) + require.NoError(t, gw.Close()) + return buf.Bytes() +} + +func listTarEntries(t *testing.T, path string) []string { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + + gr, err := gzip.NewReader(bytes.NewReader(data)) + require.NoError(t, err) + defer gr.Close() + + tr := tar.NewReader(gr) + var names []string + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + names = append(names, hdr.Name) + } + sort.Strings(names) + return names +} + +func TestStripTopLevelDir(t *testing.T) { + files := map[string]string{ + "node-v20.11.0/bin/node": "binary", + "node-v20.11.0/lib/libv8.a": "library", + "node-v20.11.0/README.md": "readme", + } + + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "test.tgz") + require.NoError(t, os.WriteFile(path, createTestTarball(t, files), 0644)) + + err := archive.StripTopLevelDir(path) + require.NoError(t, err) + + entries := listTarEntries(t, path) + // Expect "./" root entry, explicit dir entries, then "./" prefixed files — + // matching `tar -czf out.tgz -C dir .` output (what Ruby builder produces). + assert.Equal(t, []string{"./", "./README.md", "./bin/", "./bin/node", "./lib/", "./lib/libv8.a"}, entries) +} + +func TestStripTopLevelDirDotSlashPrefix(t *testing.T) { + // Simulate `tar czf out.tgz -C destDir .` where destDir contains `nginx/`. + // This produces entries like `./nginx/sbin/nginx` — the `./` is NOT a component + // to strip; `nginx/` is the real top-level dir. + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + + tw.WriteHeader(&tar.Header{Name: "./", Typeflag: tar.TypeDir, Mode: 0755}) + tw.WriteHeader(&tar.Header{Name: "./nginx/", Typeflag: tar.TypeDir, Mode: 0755}) + tw.WriteHeader(&tar.Header{Name: "./nginx/sbin/", Typeflag: tar.TypeDir, Mode: 0755}) + tw.WriteHeader(&tar.Header{Name: "./nginx/sbin/nginx", Typeflag: tar.TypeReg, Mode: 0755, Size: 6}) + tw.Write([]byte("binary")) + tw.WriteHeader(&tar.Header{Name: "./nginx/conf/", Typeflag: tar.TypeDir, Mode: 0755}) + tw.WriteHeader(&tar.Header{Name: "./nginx/modules/", Typeflag: tar.TypeDir, Mode: 0755}) + + tw.Close() + gw.Close() + + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "test.tgz") + require.NoError(t, os.WriteFile(path, buf.Bytes(), 0644)) + + err := archive.StripTopLevelDir(path) + require.NoError(t, err) + + entries := listTarEntries(t, path) + // After stripping `nginx/`, expect sbin/, conf/, modules/ at top-level. + assert.Contains(t, entries, "./sbin/nginx") + assert.NotContains(t, entries, "./nginx/") + assert.NotContains(t, entries, "./nginx/sbin/nginx") +} + +func TestStripTopLevelDirSkipsTopDir(t *testing.T) { + // Include the directory entry itself. + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + + // Directory entry. + tw.WriteHeader(&tar.Header{Name: "top/", Typeflag: tar.TypeDir, Mode: 0755}) + // File entry. + tw.WriteHeader(&tar.Header{Name: "top/file.txt", Typeflag: tar.TypeReg, Mode: 0644, Size: 5}) + tw.Write([]byte("hello")) + + tw.Close() + gw.Close() + + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "test.tgz") + require.NoError(t, os.WriteFile(path, buf.Bytes(), 0644)) + + err := archive.StripTopLevelDir(path) + require.NoError(t, err) + + entries := listTarEntries(t, path) + // Expect "./" root entry + "./"-prefixed file, matching Ruby tar output. + assert.Equal(t, []string{"./", "./file.txt"}, entries) +} + +func TestStripFiles(t *testing.T) { + files := map[string]string{ + "bin/ruby": "binary", + "lib/ruby/gems/foo.rb": "gem", + "incorrect_words.yaml": "should be removed", + "lib/incorrect_words.yaml": "also removed", + } + + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "test.tgz") + require.NoError(t, os.WriteFile(path, createTestTarball(t, files), 0644)) + + err := archive.StripFiles(path, "incorrect_words.yaml") + require.NoError(t, err) + + entries := listTarEntries(t, path) + assert.Contains(t, entries, "bin/ruby") + assert.Contains(t, entries, "lib/ruby/gems/foo.rb") + assert.NotContains(t, entries, "incorrect_words.yaml") + assert.NotContains(t, entries, "lib/incorrect_words.yaml") +} + +func TestStripIncorrectWordsYAML(t *testing.T) { + files := map[string]string{ + "bin/ruby": "binary", + "incorrect_words.yaml": "should be removed", + } + + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "test.tgz") + require.NoError(t, os.WriteFile(path, createTestTarball(t, files), 0644)) + + err := archive.StripIncorrectWordsYAML(path) + require.NoError(t, err) + + entries := listTarEntries(t, path) + assert.Contains(t, entries, "bin/ruby") + assert.NotContains(t, entries, "incorrect_words.yaml") +} + +func TestStripTopLevelDirFromZip(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "test.zip") + + // Create a zip with top-level directory. + f, err := os.Create(path) + require.NoError(t, err) + + w := zip.NewWriter(f) + fw, err := w.Create("setuptools-69.0.3/setup.py") + require.NoError(t, err) + fw.Write([]byte("setup code")) + + fw2, err := w.Create("setuptools-69.0.3/README.md") + require.NoError(t, err) + fw2.Write([]byte("readme")) + + require.NoError(t, w.Close()) + require.NoError(t, f.Close()) + + err = archive.StripTopLevelDirFromZip(path) + require.NoError(t, err) + + // Verify the zip contents. + r, err := zip.OpenReader(path) + require.NoError(t, err) + defer r.Close() + + var names []string + for _, f := range r.File { + names = append(names, f.Name) + } + sort.Strings(names) + assert.Equal(t, []string{"README.md", "setup.py"}, names) +} + +func TestPackUsesRunner(t *testing.T) { + f := runner.NewFakeRunner() + + err := archive.Pack(f, "/tmp/out.tgz", "/tmp/src", "mydir") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "tar", f.Calls[0].Name) + assert.Contains(t, f.Calls[0].Args, "czf") + assert.Contains(t, f.Calls[0].Args, "/tmp/out.tgz") + assert.Contains(t, f.Calls[0].Args, "mydir") +} + +func TestPackFlatUsesRunner(t *testing.T) { + f := runner.NewFakeRunner() + + err := archive.Pack(f, "/tmp/out.tgz", "/tmp/src", "") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Contains(t, f.Calls[0].Args, ".") +} + +func TestPackWithDereferenceUsesRunner(t *testing.T) { + f := runner.NewFakeRunner() + + err := archive.PackWithDereference(f, "/tmp/out.tgz", "/tmp/src") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "tar", f.Calls[0].Name) + assert.Contains(t, f.Calls[0].Args, "--hard-dereference") + assert.Equal(t, "/tmp/src", f.Calls[0].Dir) +} + +func TestPackXZUsesRunner(t *testing.T) { + f := runner.NewFakeRunner() + + err := archive.PackXZ(f, "/tmp/out.tar.xz", "/tmp/src") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "tar", f.Calls[0].Name) + assert.Contains(t, f.Calls[0].Args, "-Jcf") + assert.Equal(t, "/tmp/src", f.Calls[0].Dir) +} + +func TestPackZipUsesRunner(t *testing.T) { + f := runner.NewFakeRunner() + + err := archive.PackZip(f, "/tmp/out.zip", "/tmp/src") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "zip", f.Calls[0].Name) + assert.Equal(t, "/tmp/src", f.Calls[0].Dir) +} diff --git a/internal/artifact/artifact.go b/internal/artifact/artifact.go new file mode 100644 index 00000000..ea344581 --- /dev/null +++ b/internal/artifact/artifact.go @@ -0,0 +1,95 @@ +// Package artifact handles artifact naming, SHA256 computation, and S3 URL construction. +package artifact + +import ( + "crypto/sha256" + "fmt" + "io" + "os" + "strings" +) + +const s3BaseURL = "https://buildpacks.cloudfoundry.org/dependencies" + +// Artifact represents a built dependency artifact with its naming components. +type Artifact struct { + Name string // dependency name, e.g. "ruby", "php" + Version string // version string, e.g. "3.3.6", "11.0.22+7" + OS string // "linux" or "windows" + Arch string // "x64", "noarch", "x86-64" + Stack string // "cflinuxfs4", "cflinuxfs5", "any-stack" +} + +// Filename returns the canonical artifact filename: +// "name_version_os_arch_stack_sha256prefix.ext" +// +// The sha256 parameter is the full hex-encoded SHA256 of the artifact file. +// Only the first 8 characters are used in the filename. +func (a Artifact) Filename(sha256hex string, ext string) string { + prefix := a.FilenamePrefix() + sha8 := sha256hex + if len(sha8) > 8 { + sha8 = sha8[:8] + } + return fmt.Sprintf("%s_%s.%s", prefix, sha8, ext) +} + +// FilenamePrefix returns the artifact filename without the SHA prefix and extension: +// "name_version_os_arch_stack" +func (a Artifact) FilenamePrefix() string { + return fmt.Sprintf("%s_%s_%s_%s_%s", a.Name, a.Version, a.OS, a.Arch, a.Stack) +} + +// S3URL returns the canonical S3 URL for the artifact. +// +// The filename is URL-safe-encoded: '+' is replaced with '%2B' to prevent +// AWS S3 permission denied errors (S3 interprets unencoded '+' as space). +// See: https://github.com/cloudfoundry/buildpacks-ci/pull/553 +func (a Artifact) S3URL(filename string) string { + encoded := strings.ReplaceAll(filename, "+", "%2B") + return fmt.Sprintf("%s/%s/%s", s3BaseURL, a.Name, encoded) +} + +// SHA256File computes the SHA256 hex digest of a file. +func SHA256File(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("opening %s for SHA256: %w", path, err) + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", fmt.Errorf("reading %s for SHA256: %w", path, err) + } + + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// ExtFromPath extracts the file extension from a path, normalizing +// "tar.gz" to "tgz" to match the existing artifact naming convention. +func ExtFromPath(path string) string { + base := strings.ToLower(path) + + extensions := []struct { + suffix string + ext string + }{ + {".tar.gz", "tgz"}, + {".tgz", "tgz"}, + {".tar.xz", "tar.xz"}, + {".tar.bz2", "tar.bz2"}, + {".zip", "zip"}, + {".phar", "phar"}, + {".sh", "sh"}, + {".txt", "txt"}, + } + + for _, e := range extensions { + if strings.HasSuffix(base, e.suffix) { + return e.ext + } + } + + return "tgz" // default +} diff --git a/internal/artifact/artifact_test.go b/internal/artifact/artifact_test.go new file mode 100644 index 00000000..73ffe4ae --- /dev/null +++ b/internal/artifact/artifact_test.go @@ -0,0 +1,152 @@ +package artifact_test + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/artifact" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFilenameLinuxDep(t *testing.T) { + a := artifact.Artifact{ + Name: "ruby", + Version: "3.3.6", + OS: "linux", + Arch: "x64", + Stack: "cflinuxfs4", + } + + filename := a.Filename("e4311262abcdef01", "tgz") + assert.Equal(t, "ruby_3.3.6_linux_x64_cflinuxfs4_e4311262.tgz", filename) +} + +func TestFilenameWindowsDep(t *testing.T) { + a := artifact.Artifact{ + Name: "hwc", + Version: "2.0.0", + OS: "windows", + Arch: "x86-64", + Stack: "any-stack", + } + + filename := a.Filename("abcd1234deadbeef", "zip") + assert.Equal(t, "hwc_2.0.0_windows_x86-64_any-stack_abcd1234.zip", filename) +} + +func TestFilenameNoarchDep(t *testing.T) { + a := artifact.Artifact{ + Name: "bundler", + Version: "2.5.0", + OS: "linux", + Arch: "noarch", + Stack: "cflinuxfs4", + } + + filename := a.Filename("abcd1234deadbeef", "tgz") + assert.Equal(t, "bundler_2.5.0_linux_noarch_cflinuxfs4_abcd1234.tgz", filename) +} + +func TestFilenamePrefixOnly8CharsSHA(t *testing.T) { + a := artifact.Artifact{ + Name: "ruby", + Version: "3.3.6", + OS: "linux", + Arch: "x64", + Stack: "cflinuxfs4", + } + + fullSHA := "e4311262abcdef0123456789abcdef0123456789abcdef0123456789abcdef01" + filename := a.Filename(fullSHA, "tgz") + assert.Contains(t, filename, "_e4311262.") + assert.NotContains(t, filename, "abcdef01") +} + +func TestS3URL(t *testing.T) { + a := artifact.Artifact{ + Name: "ruby", + } + + url := a.S3URL("ruby_3.3.6_linux_x64_cflinuxfs4_e4311262.tgz") + assert.Equal(t, "https://buildpacks.cloudfoundry.org/dependencies/ruby/ruby_3.3.6_linux_x64_cflinuxfs4_e4311262.tgz", url) +} + +func TestS3URLEncodesPlus(t *testing.T) { + // PR #553: '+' in version strings (semver v2) must be encoded as %2B + // to prevent AWS S3 permission denied errors. + a := artifact.Artifact{ + Name: "openjdk", + } + + url := a.S3URL("openjdk_11.0.22+7_linux_x64_cflinuxfs4_abcd1234.tgz") + assert.Equal(t, "https://buildpacks.cloudfoundry.org/dependencies/openjdk/openjdk_11.0.22%2B7_linux_x64_cflinuxfs4_abcd1234.tgz", url) + assert.NotContains(t, url, "+") +} + +func TestS3URLNoEncodingNeeded(t *testing.T) { + a := artifact.Artifact{ + Name: "ruby", + } + + url := a.S3URL("ruby_3.3.6_linux_x64_cflinuxfs4_e4311262.tgz") + // No '+' in filename, URL should be unchanged. + assert.Equal(t, "https://buildpacks.cloudfoundry.org/dependencies/ruby/ruby_3.3.6_linux_x64_cflinuxfs4_e4311262.tgz", url) +} + +func TestSHA256File(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "test.txt") + content := []byte("hello world") + err := os.WriteFile(path, content, 0644) + require.NoError(t, err) + + expected := fmt.Sprintf("%x", sha256.Sum256(content)) + actual, err := artifact.SHA256File(path) + require.NoError(t, err) + assert.Equal(t, expected, actual) +} + +func TestSHA256FileMissing(t *testing.T) { + _, err := artifact.SHA256File("/nonexistent/file.txt") + require.Error(t, err) + assert.Contains(t, err.Error(), "opening") +} + +func TestExtFromPath(t *testing.T) { + tests := []struct { + path string + expected string + }{ + {"ruby-3.3.6.tar.gz", "tgz"}, + {"ruby-3.3.6.tgz", "tgz"}, + {"dotnet-sdk.tar.xz", "tar.xz"}, + {"appdynamics.tar.bz2", "tar.bz2"}, + {"hwc.zip", "zip"}, + {"composer.phar", "phar"}, + {"install.sh", "sh"}, + {"version.txt", "txt"}, + {"unknown.bin", "tgz"}, // default + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + assert.Equal(t, tt.expected, artifact.ExtFromPath(tt.path)) + }) + } +} + +func TestFilenamePrefix(t *testing.T) { + a := artifact.Artifact{ + Name: "python", + Version: "3.12.0", + OS: "linux", + Arch: "x64", + Stack: "cflinuxfs5", + } + + assert.Equal(t, "python_3.12.0_linux_x64_cflinuxfs5", a.FilenamePrefix()) +} diff --git a/internal/autoconf/autoconf.go b/internal/autoconf/autoconf.go new file mode 100644 index 00000000..41afe3bf --- /dev/null +++ b/internal/autoconf/autoconf.go @@ -0,0 +1,243 @@ +// Package autoconf provides a hook-based build engine for software that uses +// the standard autoconf configure/make/make-install cycle. +// +// Recipe is a pure build engine — it does not implement the recipe.Recipe +// interface directly. Thin wrappers in internal/recipe/ embed or delegate to +// Recipe.Build and expose the Name/Artifact methods required by recipe.Recipe. +// This avoids an import cycle between internal/recipe and internal/autoconf. +// +// Hook fields are all func types; nil means "use default behaviour". +package autoconf + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// Hooks holds all optional customisation points for Recipe. A nil hook means +// "use the default behaviour described in the field comment". +type Hooks struct { + // AptPackages returns the list of apt packages to install before building. + // Default: s.AptPackages["{name}_build"] + AptPackages func(s *stack.Stack) []string + + // SourceProvider downloads or otherwise prepares the source tree and returns + // the path to the extracted source directory. + // Default: fetch tarball from src.URL, extract to /tmp/- + SourceProvider func(ctx context.Context, src *source.Input, f fetch.Fetcher, r runner.Runner) (srcDir string, err error) + + // BeforeDownload runs before the source tarball is downloaded (or before + // SourceProvider is called). Typical use: GPG signature verification. + // Default: no-op + BeforeDownload func(ctx context.Context, src *source.Input, r runner.Runner) error + + // AfterExtract runs inside srcDir immediately after extraction. + // Typical use: autoreconf -i, autogen.sh. + // Default: no-op + AfterExtract func(ctx context.Context, srcDir string, prefix string, r runner.Runner) error + + // ConfigureArgs returns the full list of arguments for ./configure. + // Default: ["--prefix="] + ConfigureArgs func(srcDir, prefix string) []string + + // ConfigureEnv provides additional environment variables for ./configure and make. + // Default: nil (no extra env) + ConfigureEnv func() map[string]string + + // MakeArgs returns extra arguments for the make step. + // Default: nil (plain "make") + MakeArgs func() []string + + // InstallEnv provides extra environment variables for make install. + // Default: nil (no extra env, same as ConfigureEnv result) + InstallEnv func(prefix string) map[string]string + + // AfterInstall runs after make install, inside the prefix directory. + // Typical use: remove runtime dirs, move/rename files. + // Default: no-op + AfterInstall func(ctx context.Context, prefix string, r runner.Runner) error + + // PackDirs lists the sub-directories of prefix to pack into the artifact tarball. + // Default: ["."] (pack the entire prefix) + PackDirs func() []string + + // AfterPack runs after the artifact tarball is created. + // Typical use: archive.StripTopLevelDir for nginx. + // Default: no-op + AfterPack func(artifactPath string) error +} + +// Recipe is a hook-based build engine for autoconf-based dependencies. +// It does NOT implement recipe.Recipe; use a thin wrapper in internal/recipe/. +type Recipe struct { + DepName string + Fetcher fetch.Fetcher + Hooks Hooks +} + +func (r *Recipe) Name() string { return r.DepName } + +// Build runs the full configure/make/make-install cycle with hook customisation. +func (r *Recipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, _ *output.OutData) error { + name := r.DepName + version := src.Version + + // ── Step 1: apt install build dependencies ──────────────────────────────── + var pkgs []string + if r.Hooks.AptPackages != nil { + pkgs = r.Hooks.AptPackages(s) + } else { + pkgs = s.AptPackages[fmt.Sprintf("%s_build", name)] + } + if len(pkgs) > 0 { + a := apt.New(run) + if err := a.Install(ctx, pkgs...); err != nil { + return fmt.Errorf("%s: apt install %s_build: %w", name, name, err) + } + } + + // ── Step 2: before-download hook (e.g. GPG verification) ───────────────── + if r.Hooks.BeforeDownload != nil { + if err := r.Hooks.BeforeDownload(ctx, src, run); err != nil { + return fmt.Errorf("%s: before_download: %w", name, err) + } + } + + // ── Step 3: provide source ──────────────────────────────────────────────── + builtPath := fmt.Sprintf("/tmp/%s-built-%s", name, version) + prefix := builtPath + + var srcDir string + if r.Hooks.SourceProvider != nil { + var err error + srcDir, err = r.Hooks.SourceProvider(ctx, src, r.Fetcher, run) + if err != nil { + return fmt.Errorf("%s: source provider: %w", name, err) + } + } else { + srcDir = fmt.Sprintf("/tmp/%s-%s", name, version) + srcTarball := fmt.Sprintf("/tmp/%s-%s.tar.gz", name, version) + if err := r.Fetcher.Download(ctx, src.URL, srcTarball, src.PrimaryChecksum()); err != nil { + return fmt.Errorf("%s: downloading source: %w", name, err) + } + if err := run.Run("tar", "xzf", srcTarball, "-C", "/tmp"); err != nil { + return fmt.Errorf("%s: extracting source: %w", name, err) + } + } + + if err := run.Run("mkdir", "-p", builtPath); err != nil { + return fmt.Errorf("%s: mkdir prefix: %w", name, err) + } + + // ── Step 4: after-extract hook ──────────────────────────────────────────── + if r.Hooks.AfterExtract != nil { + if err := r.Hooks.AfterExtract(ctx, srcDir, prefix, run); err != nil { + return fmt.Errorf("%s: after_extract: %w", name, err) + } + } + + // ── Step 5: configure ───────────────────────────────────────────────────── + var configureArgs []string + if r.Hooks.ConfigureArgs != nil { + configureArgs = r.Hooks.ConfigureArgs(srcDir, prefix) + } else { + configureArgs = []string{fmt.Sprintf("--prefix=%s", prefix)} + } + + var configureEnv map[string]string + if r.Hooks.ConfigureEnv != nil { + configureEnv = r.Hooks.ConfigureEnv() + } + + if configureEnv != nil { + if err := run.RunInDirWithEnv(srcDir, configureEnv, "./configure", configureArgs...); err != nil { + return fmt.Errorf("%s: configure: %w", name, err) + } + } else { + if err := run.RunInDir(srcDir, "./configure", configureArgs...); err != nil { + return fmt.Errorf("%s: configure: %w", name, err) + } + } + + // ── Step 6: make ────────────────────────────────────────────────────────── + makeArgs := []string{} + if r.Hooks.MakeArgs != nil { + makeArgs = r.Hooks.MakeArgs() + } + + if configureEnv != nil { + if err := run.RunInDirWithEnv(srcDir, configureEnv, "make", makeArgs...); err != nil { + return fmt.Errorf("%s: make: %w", name, err) + } + } else { + if err := run.RunInDir(srcDir, "make", makeArgs...); err != nil { + return fmt.Errorf("%s: make: %w", name, err) + } + } + + // ── Step 7: make install ────────────────────────────────────────────────── + var installEnv map[string]string + if r.Hooks.InstallEnv != nil { + installEnv = r.Hooks.InstallEnv(prefix) + } else if configureEnv != nil { + installEnv = configureEnv + } + + if installEnv != nil { + if err := run.RunInDirWithEnv(srcDir, installEnv, "make", "install"); err != nil { + return fmt.Errorf("%s: make install: %w", name, err) + } + } else { + if err := run.RunInDir(srcDir, "make", "install"); err != nil { + return fmt.Errorf("%s: make install: %w", name, err) + } + } + + // ── Step 8: after-install hook ──────────────────────────────────────────── + if r.Hooks.AfterInstall != nil { + if err := r.Hooks.AfterInstall(ctx, prefix, run); err != nil { + return fmt.Errorf("%s: after_install: %w", name, err) + } + } + + // ── Step 9: pack artifact ───────────────────────────────────────────────── + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("%s-%s-linux-x64.tgz", name, version)) + + var packDirs []string + if r.Hooks.PackDirs != nil { + packDirs = r.Hooks.PackDirs() + } else { + packDirs = []string{"."} + } + + tarArgs := append([]string{"czf", artifactPath}, packDirs...) + if err := run.RunInDir(prefix, "tar", tarArgs...); err != nil { + return fmt.Errorf("%s: packing artifact: %w", name, err) + } + + // ── Step 10: after-pack hook (e.g. StripTopLevelDir) ───────────────────── + if r.Hooks.AfterPack != nil { + if err := r.Hooks.AfterPack(artifactPath); err != nil { + return fmt.Errorf("%s: after_pack: %w", name, err) + } + } + + return nil +} + +// mustCwd returns the current working directory, panicking on error. +func mustCwd() string { + cwd, err := filepath.Abs(".") + if err != nil { + panic(fmt.Sprintf("autoconf: getting cwd: %v", err)) + } + return cwd +} diff --git a/internal/autoconf/autoconf_test.go b/internal/autoconf/autoconf_test.go new file mode 100644 index 00000000..7f97f631 --- /dev/null +++ b/internal/autoconf/autoconf_test.go @@ -0,0 +1,668 @@ +package autoconf_test + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/autoconf" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ── fakeFetcher ─────────────────────────────────────────────────────────────── + +// fakeFetcher satisfies fetch.Fetcher without making any network calls. +type fakeFetcher struct { + downloaded []string + errMap map[string]error +} + +func newFakeFetcher() *fakeFetcher { + return &fakeFetcher{errMap: make(map[string]error)} +} + +func (f *fakeFetcher) Download(_ context.Context, url, _ string, _ source.Checksum) error { + f.downloaded = append(f.downloaded, url) + if err, ok := f.errMap[url]; ok { + return err + } + return nil +} + +func (f *fakeFetcher) ReadBody(_ context.Context, url string) ([]byte, error) { + if err, ok := f.errMap[url]; ok { + return nil, err + } + return []byte("fake"), nil +} + +// Ensure fakeFetcher satisfies the fetch.Fetcher interface at compile time. +var _ fetch.Fetcher = (*fakeFetcher)(nil) + +// ── helpers ─────────────────────────────────────────────────────────────────── + +func newStack() *stack.Stack { + return &stack.Stack{ + Name: "cflinuxfs4", + AptPackages: map[string][]string{ + "mylib_build": {"libfoo-dev", "libbar-dev"}, + }, + } +} + +func newInput(name, version, url string) *source.Input { + return &source.Input{ + Name: name, + Version: version, + URL: url, + SHA256: "deadbeef", + } +} + +func anyCallContains(calls []runner.Call, name string) bool { + for _, c := range calls { + if c.Name == name { + return true + } + } + return false +} + +func anyArgsContain(calls []runner.Call, target string) bool { + for _, c := range calls { + for _, arg := range c.Args { + if strings.Contains(arg, target) { + return true + } + } + } + return false +} + +func hasCallMatching(calls []runner.Call, name, argSubstr string) bool { + for _, c := range calls { + if c.Name == name { + joined := strings.Join(c.Args, " ") + if argSubstr == "" || strings.Contains(joined, argSubstr) { + return true + } + } + } + return false +} + +func hasCallWithEnv(calls []runner.Call, name, envKey string) bool { + for _, c := range calls { + if c.Name == name && c.Env != nil { + if _, ok := c.Env[envKey]; ok { + return true + } + } + } + return false +} + +// ── Name / Artifact ────────────────────────────────────────────────────────── + +func TestRecipeName(t *testing.T) { + r := &autoconf.Recipe{DepName: "mylib"} + assert.Equal(t, "mylib", r.Name()) +} + +// ── default apt packages ───────────────────────────────────────────────────── + +func TestDefaultAptPackagesFromStack(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + // No AptPackages hook → default key is "mylib_build" + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + // apt-get install must have been called with the packages from "mylib_build". + assert.True(t, hasCallMatching(run.Calls, "apt-get", "libfoo-dev"), + "default apt packages should come from s.AptPackages['mylib_build']") + assert.True(t, hasCallMatching(run.Calls, "apt-get", "libbar-dev"), + "default apt packages should come from s.AptPackages['mylib_build']") +} + +func TestAptPackagesHookOverridesDefault(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + AptPackages: func(_ *stack.Stack) []string { + return []string{"custom-pkg"} + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hasCallMatching(run.Calls, "apt-get", "custom-pkg"), + "AptPackages hook result should be used for install") + assert.False(t, hasCallMatching(run.Calls, "apt-get", "libfoo-dev"), + "default packages must not be used when hook overrides") +} + +// ── default source download ─────────────────────────────────────────────────── + +func TestDefaultSourceDownloadsAndExtracts(t *testing.T) { + run := runner.NewFakeRunner() + f := newFakeFetcher() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: f, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + // Fetcher should have downloaded the source URL. + require.Len(t, f.downloaded, 1) + assert.Equal(t, src.URL, f.downloaded[0]) + + // Runner should have extracted the tarball. + assert.True(t, hasCallMatching(run.Calls, "tar", "xzf"), + "should run tar xzf to extract source") +} + +// ── before-download hook ────────────────────────────────────────────────────── + +func TestBeforeDownloadHookIsCalledBeforeSource(t *testing.T) { + run := runner.NewFakeRunner() + f := newFakeFetcher() + hookCalled := false + downloadCalledBeforeHook := false + + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: f, + Hooks: autoconf.Hooks{ + BeforeDownload: func(_ context.Context, _ *source.Input, _ runner.Runner) error { + // Fetcher should not have been called yet. + if len(f.downloaded) > 0 { + downloadCalledBeforeHook = true + } + hookCalled = true + return nil + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hookCalled, "BeforeDownload hook must be called") + assert.False(t, downloadCalledBeforeHook, "download must not occur before BeforeDownload hook") +} + +func TestBeforeDownloadErrorIsPropagated(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + BeforeDownload: func(_ context.Context, _ *source.Input, _ runner.Runner) error { + return errors.New("gpg verification failed") + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "gpg verification failed") + assert.Contains(t, err.Error(), "before_download") +} + +func TestSourceProviderHookOverridesDownload(t *testing.T) { + run := runner.NewFakeRunner() + f := newFakeFetcher() + providerCalled := false + + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: f, + Hooks: autoconf.Hooks{ + SourceProvider: func(_ context.Context, _ *source.Input, _ fetch.Fetcher, _ runner.Runner) (string, error) { + providerCalled = true + return "/tmp/custom-src", nil + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, providerCalled, "SourceProvider hook must be called") + // Fetcher must NOT be called when SourceProvider is set. + assert.Empty(t, f.downloaded, "Fetcher.Download must not be called when SourceProvider is set") +} + +func TestSourceProviderErrorIsPropagated(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + SourceProvider: func(_ context.Context, _ *source.Input, _ fetch.Fetcher, _ runner.Runner) (string, error) { + return "", errors.New("provider failed") + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "provider failed") + assert.Contains(t, err.Error(), "source provider") +} + +// ── after-extract hook ──────────────────────────────────────────────────────── + +func TestAfterExtractHookIsCalledBeforeConfigure(t *testing.T) { + run := runner.NewFakeRunner() + hookCalled := false + configureCallIdx := -1 + + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + AfterExtract: func(_ context.Context, _, _ string, _ runner.Runner) error { + // Record that the hook ran — configure should not have been called yet. + for _, c := range run.Calls { + if c.Name == "./configure" { + configureCallIdx = len(run.Calls) - 1 + } + } + hookCalled = true + return nil + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hookCalled, "AfterExtract hook must be called") + assert.Equal(t, -1, configureCallIdx, + "configure must not be called before AfterExtract hook returns") +} + +func TestAfterExtractErrorIsPropagated(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + AfterExtract: func(_ context.Context, _, _ string, _ runner.Runner) error { + return errors.New("autoreconf failed") + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "autoreconf failed") + assert.Contains(t, err.Error(), "after_extract") +} + +// ── configure ──────────────────────────────────────────────────────────────── + +func TestDefaultConfigureUsesPrefix(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hasCallMatching(run.Calls, "./configure", "--prefix="), + "default configure should use --prefix=") + assert.True(t, anyArgsContain(run.Calls, "--prefix=/tmp/mylib-built-1.0"), + "prefix should contain dep name and version") +} + +func TestConfigureArgsHookOverridesDefault(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + ConfigureArgs: func(_, prefix string) []string { + return []string{ + "--prefix=" + prefix, + "--enable-shared", + "--disable-static", + } + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hasCallMatching(run.Calls, "./configure", "--enable-shared"), + "configure should include custom arg from hook") + assert.True(t, hasCallMatching(run.Calls, "./configure", "--disable-static"), + "configure should include custom arg from hook") +} + +func TestConfigureEnvHookPassesEnvToMake(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + ConfigureEnv: func() map[string]string { + return map[string]string{"CFLAGS": "-O2 -fPIC"} + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + // Both configure and make should be called with the env. + assert.True(t, hasCallWithEnv(run.Calls, "./configure", "CFLAGS"), + "configure should have CFLAGS in env when ConfigureEnv hook is set") + assert.True(t, hasCallWithEnv(run.Calls, "make", "CFLAGS"), + "make should inherit CFLAGS env from ConfigureEnv hook") +} + +// ── make args ───────────────────────────────────────────────────────────────── + +func TestMakeArgsHookPassesExtraArgs(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + MakeArgs: func() []string { return []string{"-j2"} }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hasCallMatching(run.Calls, "make", "-j2"), + "make should be called with -j2 from MakeArgs hook") +} + +// ── install env ────────────────────────────────────────────────────────────── + +func TestInstallEnvHookPassesDifferentEnvToMakeInstall(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + InstallEnv: func(prefix string) map[string]string { + return map[string]string{"DESTDIR": prefix + "/staging"} + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hasCallWithEnv(run.Calls, "make", "DESTDIR"), + "make install should have DESTDIR env when InstallEnv hook is set") +} + +func TestInstallEnvFallsBackToConfigureEnvWhenNil(t *testing.T) { + // When InstallEnv is nil but ConfigureEnv is set, the configure env is + // reused for make install (needed for libgdiplus which uses CFLAGS/CXXFLAGS + // for all three steps). + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + ConfigureEnv: func() map[string]string { + return map[string]string{"CFLAGS": "-g"} + }, + // InstallEnv is nil + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + // make install must still have CFLAGS (from ConfigureEnv fallback). + assert.True(t, hasCallWithEnv(run.Calls, "make", "CFLAGS"), + "make install should fall back to ConfigureEnv when InstallEnv is nil") +} + +// ── after-install hook ──────────────────────────────────────────────────────── + +func TestAfterInstallHookIsCalledAfterMakeInstall(t *testing.T) { + run := runner.NewFakeRunner() + hookCalled := false + makeInstallSeen := false + + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + AfterInstall: func(_ context.Context, _ string, _ runner.Runner) error { + // At this point make install should have already been called. + for _, c := range run.Calls { + if c.Name == "make" { + for _, a := range c.Args { + if a == "install" { + makeInstallSeen = true + } + } + } + } + hookCalled = true + return nil + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hookCalled, "AfterInstall hook must be called") + assert.True(t, makeInstallSeen, "make install must be called before AfterInstall hook") +} + +func TestAfterInstallErrorIsPropagated(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + AfterInstall: func(_ context.Context, _ string, _ runner.Runner) error { + return errors.New("cleanup failed") + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "cleanup failed") + assert.Contains(t, err.Error(), "after_install") +} + +// ── pack dirs ──────────────────────────────────────────────────────────────── + +func TestDefaultPackDirIsFullPrefix(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + // Default: pack "." from inside prefix. + assert.True(t, anyCallContains(run.Calls, "tar"), + "should run tar to pack artifact") + assert.True(t, anyArgsContain(run.Calls, "."), + "default pack dir should be '.'") +} + +func TestPackDirsHookLimitsWhatIsPacked(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + PackDirs: func() []string { return []string{"include", "lib"} }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hasCallMatching(run.Calls, "tar", "include"), + "tar should include 'include' directory") + assert.True(t, hasCallMatching(run.Calls, "tar", "lib"), + "tar should include 'lib' directory") + // Must not pack the default "." + // (this is subtle — we just check both named dirs are present) +} + +// ── after-pack hook ─────────────────────────────────────────────────────────── + +func TestAfterPackHookIsCalledAfterTar(t *testing.T) { + run := runner.NewFakeRunner() + hookCalled := false + tarCalledBeforeHook := false + + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + AfterPack: func(_ string) error { + // tar must have been called before this hook. + for _, c := range run.Calls { + if c.Name == "tar" { + for _, a := range c.Args { + if a == "czf" { + tarCalledBeforeHook = true + } + } + } + } + hookCalled = true + return nil + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hookCalled, "AfterPack hook must be called") + assert.True(t, tarCalledBeforeHook, "tar czf must be called before AfterPack hook") +} + +func TestAfterPackErrorIsPropagated(t *testing.T) { + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + Hooks: autoconf.Hooks{ + AfterPack: func(_ string) error { + return errors.New("strip failed") + }, + }, + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "strip failed") + assert.Contains(t, err.Error(), "after_pack") +} + +// ── call ordering sanity ────────────────────────────────────────────────────── + +func TestBuildStepOrder(t *testing.T) { + // Verifies apt→configure→make→make-install→tar order is maintained. + run := runner.NewFakeRunner() + r := &autoconf.Recipe{ + DepName: "mylib", + Fetcher: newFakeFetcher(), + } + src := newInput("mylib", "1.0", "https://example.com/mylib-1.0.tar.gz") + + err := r.Build(context.Background(), newStack(), src, run, &output.OutData{}) + require.NoError(t, err) + + // Collect call names in order. + var names []string + for _, c := range run.Calls { + names = append(names, c.Name) + } + + aptIdx := -1 + configureIdx := -1 + makeIdx := -1 + makeInstallIdx := -1 + tarIdx := -1 + + for i, c := range run.Calls { + switch { + case c.Name == "apt-get" && aptIdx < 0: + aptIdx = i + case c.Name == "./configure" && configureIdx < 0: + configureIdx = i + case c.Name == "make" && len(c.Args) == 0 && makeIdx < 0: + makeIdx = i + case c.Name == "make" && len(c.Args) > 0 && c.Args[0] == "install" && makeInstallIdx < 0: + makeInstallIdx = i + case c.Name == "tar" && tarIdx < 0: + // Skip the extract tar; find the pack tar (czf). + for _, a := range c.Args { + if a == "czf" { + tarIdx = i + } + } + } + } + + assert.Greater(t, configureIdx, aptIdx, "configure must come after apt-get") + assert.Greater(t, makeIdx, configureIdx, "make must come after configure") + assert.Greater(t, makeInstallIdx, makeIdx, "make install must come after make") + assert.Greater(t, tarIdx, makeInstallIdx, "tar (pack) must come after make install") +} diff --git a/internal/compiler/compiler.go b/internal/compiler/compiler.go new file mode 100644 index 00000000..58ffc890 --- /dev/null +++ b/internal/compiler/compiler.go @@ -0,0 +1,121 @@ +// Package compiler provides GCC and gfortran setup helpers. +// All version numbers and paths come from the injected stack config — +// no hardcoded compiler versions in this package. +package compiler + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// GCC manages GCC/g++ installation and update-alternatives setup. +type GCC struct { + Config stack.GCCConfig + APT *apt.APT + Runner runner.Runner +} + +// NewGCC creates a GCC instance from stack config. +func NewGCC(config stack.GCCConfig, a *apt.APT, r runner.Runner) *GCC { + return &GCC{Config: config, APT: a, Runner: r} +} + +// Setup installs GCC, optionally adds a PPA, and sets up update-alternatives. +// On cflinuxfs4: installs tool_packages (software-properties-common), adds PPA, installs gcc-12/g++-12. +// On cflinuxfs5: installs tool_packages, skips PPA (empty string), installs gcc-14/g++-14 (native). +func (g *GCC) Setup(ctx context.Context) error { + // Install prerequisite tool packages required before add-apt-repository + // (e.g. software-properties-common). The list lives in stacks/*.yaml under + // compilers.gcc.tool_packages so it can be adjusted per stack without + // touching Go source. + if len(g.Config.ToolPackages) > 0 { + if err := g.APT.Install(ctx, g.Config.ToolPackages...); err != nil { + return fmt.Errorf("installing GCC tool packages: %w", err) + } + } + + // Add PPA only when configured (cflinuxfs4 needs it, cflinuxfs5 does not). + if err := g.APT.AddPPA(ctx, g.Config.PPA); err != nil { + return fmt.Errorf("adding GCC PPA: %w", err) + } + + // Install GCC packages. + if err := g.APT.Install(ctx, g.Config.Packages...); err != nil { + return fmt.Errorf("installing GCC packages: %w", err) + } + + // Set up update-alternatives so gcc/g++ point to the correct version. + gccBin := fmt.Sprintf("/usr/bin/gcc-%d", g.Config.Version) + gppBin := fmt.Sprintf("/usr/bin/g++-%d", g.Config.Version) + + return g.Runner.Run( + "update-alternatives", + "--install", "/usr/bin/gcc", "gcc", gccBin, "60", + "--slave", "/usr/bin/g++", "g++", gppBin, + ) +} + +// Gfortran manages gfortran installation and library copying. +type Gfortran struct { + Config stack.GfortranConfig + APT *apt.APT + Runner runner.Runner +} + +// NewGfortran creates a Gfortran instance from stack config. +func NewGfortran(config stack.GfortranConfig, a *apt.APT, r runner.Runner) *Gfortran { + return &Gfortran{Config: config, APT: a, Runner: r} +} + +// Setup installs gfortran packages for the stack. +func (g *Gfortran) Setup(ctx context.Context) error { + return g.APT.Install(ctx, g.Config.Packages...) +} + +// CopyLibs copies the stack-specific gfortran libraries into the target directory. +// targetLib receives .a and .so files; targetBin receives the gfortran binary and f951. +// +// On cflinuxfs4 (jammy): executables and libs share the same dir (lib_path). +// On cflinuxfs5 (noble): GCC moved executables to /usr/libexec/gcc/…; libs remain +// in /usr/lib/gcc/…. libexec_path in the stack YAML points to the executables dir. +func (g *Gfortran) CopyLibs(_ context.Context, targetLib, targetBin string) error { + libPath := g.Config.LibPath + + // execPath is where GCC executables (f951, cc1, etc.) live. + // Falls back to libPath when libexec_path is not set (jammy/cflinuxfs4). + execPath := g.Config.LibexecPath + if execPath == "" { + execPath = libPath + } + + // Copy gfortran binary. + if err := g.Runner.Run("cp", "-L", g.Config.Bin, fmt.Sprintf("%s/gfortran", targetBin)); err != nil { + return fmt.Errorf("copying gfortran binary: %w", err) + } + + // Copy f951 compiler frontend from the executables directory. + if err := g.Runner.Run("cp", "-L", fmt.Sprintf("%s/f951", execPath), fmt.Sprintf("%s/f951", targetBin)); err != nil { + return fmt.Errorf("copying f951: %w", err) + } + + // Copy libraries. + libs := []string{"libcaf_single.a", "libgfortran.a", "libgfortran.so"} + for _, lib := range libs { + src := fmt.Sprintf("%s/%s", libPath, lib) + dst := fmt.Sprintf("%s/%s", targetLib, lib) + if err := g.Runner.Run("cp", "-L", src, dst); err != nil { + return fmt.Errorf("copying %s: %w", lib, err) + } + } + + // Copy system libpcre2. + if err := g.Runner.Run("cp", "-L", "/usr/lib/x86_64-linux-gnu/libpcre2-8.so.0", fmt.Sprintf("%s/libpcre2-8.so.0", targetLib)); err != nil { + return fmt.Errorf("copying libpcre2: %w", err) + } + + return nil +} diff --git a/internal/compiler/compiler_test.go b/internal/compiler/compiler_test.go new file mode 100644 index 00000000..cca45491 --- /dev/null +++ b/internal/compiler/compiler_test.go @@ -0,0 +1,270 @@ +package compiler_test + +import ( + "context" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/compiler" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/stack" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGCCSetupCflinuxfs4(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + config := stack.GCCConfig{ + Version: 12, + Packages: []string{"gcc-12", "g++-12"}, + PPA: "ppa:ubuntu-toolchain-r/test", + ToolPackages: []string{"software-properties-common"}, + } + + gcc := compiler.NewGCC(config, a, f) + err := gcc.Setup(context.Background()) + require.NoError(t, err) + + // Expect: install software-properties-common (from ToolPackages), add-apt-repository, apt-get update, + // install gcc-12 g++-12, update-alternatives + var callNames []string + for _, c := range f.Calls { + callNames = append(callNames, c.Name) + } + + // Should see software-properties-common installed first (from ToolPackages). + require.NotEmpty(t, f.Calls) + firstAptInstall := f.Calls[0] + assert.Equal(t, "apt-get", firstAptInstall.Name) + assert.Contains(t, firstAptInstall.Args, "software-properties-common") + + // Should see add-apt-repository (PPA is non-empty). + assert.Contains(t, callNames, "add-apt-repository") + + // Should see update-alternatives. + assert.Contains(t, callNames, "update-alternatives") + + // Find the update-alternatives call and verify version. + for _, c := range f.Calls { + if c.Name == "update-alternatives" { + assert.Contains(t, c.Args, "/usr/bin/gcc-12") + assert.Contains(t, c.Args, "/usr/bin/g++-12") + break + } + } + + // Find the GCC install call. + for _, c := range f.Calls { + if c.Name == "apt-get" { + for _, arg := range c.Args { + if arg == "gcc-12" { + assert.Contains(t, c.Args, "g++-12") + break + } + } + } + } +} + +func TestGCCSetupCflinuxfs5(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + config := stack.GCCConfig{ + Version: 14, + Packages: []string{"gcc-14", "g++-14"}, + PPA: "", // No PPA needed on cflinuxfs5. + ToolPackages: []string{"software-properties-common"}, + } + + gcc := compiler.NewGCC(config, a, f) + err := gcc.Setup(context.Background()) + require.NoError(t, err) + + // ToolPackages should be installed even when PPA is empty. + require.NotEmpty(t, f.Calls) + firstAptInstall := f.Calls[0] + assert.Equal(t, "apt-get", firstAptInstall.Name) + assert.Contains(t, firstAptInstall.Args, "software-properties-common") + + // Should NOT see add-apt-repository (PPA is empty). + for _, c := range f.Calls { + assert.NotEqual(t, "add-apt-repository", c.Name, + "cflinuxfs5 should not add a PPA") + } + + // Should see update-alternatives with gcc-14. + for _, c := range f.Calls { + if c.Name == "update-alternatives" { + assert.Contains(t, c.Args, "/usr/bin/gcc-14") + assert.Contains(t, c.Args, "/usr/bin/g++-14") + break + } + } +} + +func TestGCCSetupNoToolPackages(t *testing.T) { + // When ToolPackages is empty, no tool install call should be made. + f := runner.NewFakeRunner() + a := apt.New(f) + + config := stack.GCCConfig{ + Version: 12, + Packages: []string{"gcc-12", "g++-12"}, + PPA: "", + ToolPackages: nil, // explicitly empty + } + + gcc := compiler.NewGCC(config, a, f) + require.NoError(t, gcc.Setup(context.Background())) + + // No apt-get call should contain "software-properties-common". + for _, c := range f.Calls { + if c.Name == "apt-get" { + assert.NotContains(t, c.Args, "software-properties-common", + "no tool package install expected when ToolPackages is empty") + } + } +} + +func TestGfortranSetupCflinuxfs4(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + config := stack.GfortranConfig{ + Version: 11, + Bin: "/usr/bin/x86_64-linux-gnu-gfortran-11", + LibPath: "/usr/lib/gcc/x86_64-linux-gnu/11", + Packages: []string{"gfortran", "libgfortran-12-dev"}, + } + + gf := compiler.NewGfortran(config, a, f) + err := gf.Setup(context.Background()) + require.NoError(t, err) + + // Should install gfortran packages. + require.Len(t, f.Calls, 1) + assert.Equal(t, "apt-get", f.Calls[0].Name) + assert.Contains(t, f.Calls[0].Args, "gfortran") + assert.Contains(t, f.Calls[0].Args, "libgfortran-12-dev") +} + +func TestGfortranSetupCflinuxfs5(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + config := stack.GfortranConfig{ + Version: 13, + Bin: "/usr/bin/x86_64-linux-gnu-gfortran-13", + LibPath: "/usr/lib/gcc/x86_64-linux-gnu/13", + LibexecPath: "/usr/libexec/gcc/x86_64-linux-gnu/13", + Packages: []string{"gfortran", "libgfortran-13-dev"}, + } + + gf := compiler.NewGfortran(config, a, f) + err := gf.Setup(context.Background()) + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Contains(t, f.Calls[0].Args, "libgfortran-13-dev") +} + +func TestGfortranCopyLibsCflinuxfs4(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + config := stack.GfortranConfig{ + Version: 11, + Bin: "/usr/bin/x86_64-linux-gnu-gfortran-11", + LibPath: "/usr/lib/gcc/x86_64-linux-gnu/11", + } + + gf := compiler.NewGfortran(config, a, f) + err := gf.CopyLibs(context.Background(), "/target/lib", "/target/bin") + require.NoError(t, err) + + // Verify copies from version 11 paths. + var cpSources []string + for _, c := range f.Calls { + if c.Name == "cp" { + cpSources = append(cpSources, c.Args[1]) // -L is args[0], source is args[1] + } + } + + assert.Contains(t, cpSources, "/usr/bin/x86_64-linux-gnu-gfortran-11") + assert.Contains(t, cpSources, "/usr/lib/gcc/x86_64-linux-gnu/11/f951") + assert.Contains(t, cpSources, "/usr/lib/gcc/x86_64-linux-gnu/11/libcaf_single.a") + assert.Contains(t, cpSources, "/usr/lib/gcc/x86_64-linux-gnu/11/libgfortran.a") + assert.Contains(t, cpSources, "/usr/lib/gcc/x86_64-linux-gnu/11/libgfortran.so") + assert.Contains(t, cpSources, "/usr/lib/x86_64-linux-gnu/libpcre2-8.so.0") +} + +func TestGfortranCopyLibsCflinuxfs5(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + config := stack.GfortranConfig{ + Version: 13, + Bin: "/usr/bin/x86_64-linux-gnu-gfortran-13", + LibPath: "/usr/lib/gcc/x86_64-linux-gnu/13", + LibexecPath: "/usr/libexec/gcc/x86_64-linux-gnu/13", + } + + gf := compiler.NewGfortran(config, a, f) + err := gf.CopyLibs(context.Background(), "/target/lib", "/target/bin") + require.NoError(t, err) + + // Verify copies from version 13 paths. + var cpSources []string + for _, c := range f.Calls { + if c.Name == "cp" { + cpSources = append(cpSources, c.Args[1]) + } + } + + assert.Contains(t, cpSources, "/usr/bin/x86_64-linux-gnu-gfortran-13") + // f951 comes from libexec_path on cflinuxfs5 (noble), not lib_path. + assert.Contains(t, cpSources, "/usr/libexec/gcc/x86_64-linux-gnu/13/f951") + assert.NotContains(t, cpSources, "/usr/lib/gcc/x86_64-linux-gnu/13/f951") + // Libs still come from lib_path. + assert.Contains(t, cpSources, "/usr/lib/gcc/x86_64-linux-gnu/13/libcaf_single.a") + assert.Contains(t, cpSources, "/usr/lib/gcc/x86_64-linux-gnu/13/libgfortran.a") + assert.Contains(t, cpSources, "/usr/lib/gcc/x86_64-linux-gnu/13/libgfortran.so") + + // Verify NO version 11 or 14 paths. + for _, src := range cpSources { + assert.NotContains(t, src, "/11/") + } +} + +func TestGfortranCopyLibsTargetPaths(t *testing.T) { + f := runner.NewFakeRunner() + a := apt.New(f) + + config := stack.GfortranConfig{ + Version: 11, + Bin: "/usr/bin/x86_64-linux-gnu-gfortran-11", + LibPath: "/usr/lib/gcc/x86_64-linux-gnu/11", + } + + gf := compiler.NewGfortran(config, a, f) + err := gf.CopyLibs(context.Background(), "/r/lib", "/r/bin") + require.NoError(t, err) + + // Verify target paths. + var cpDests []string + for _, c := range f.Calls { + if c.Name == "cp" { + cpDests = append(cpDests, c.Args[2]) // -L is args[0], source is args[1], dest is args[2] + } + } + + assert.Contains(t, cpDests, "/r/bin/gfortran") + assert.Contains(t, cpDests, "/r/bin/f951") + assert.Contains(t, cpDests, "/r/lib/libcaf_single.a") + assert.Contains(t, cpDests, "/r/lib/libgfortran.a") + assert.Contains(t, cpDests, "/r/lib/libgfortran.so") + assert.Contains(t, cpDests, "/r/lib/libpcre2-8.so.0") +} diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go new file mode 100644 index 00000000..36891aec --- /dev/null +++ b/internal/fetch/fetch.go @@ -0,0 +1,135 @@ +// Package fetch provides HTTP download with checksum verification. +package fetch + +import ( + "context" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "fmt" + "hash" + "io" + "net/http" + "os" + "time" + + "github.com/cloudfoundry/binary-builder/internal/source" +) + +// Fetcher is the interface for downloading files and reading URL bodies. +type Fetcher interface { + // Download fetches a URL to a local file, verifying the checksum. + // If checksum.Value is empty, no verification is performed. + Download(ctx context.Context, url, dest string, checksum source.Checksum) error + + // ReadBody fetches a URL and returns the response body as bytes. + ReadBody(ctx context.Context, url string) ([]byte, error) +} + +// HTTPFetcher implements Fetcher using net/http. +type HTTPFetcher struct { + Client *http.Client +} + +// NewHTTPFetcher creates a Fetcher with a 10-minute HTTP client timeout. +// A timeout prevents builds hanging indefinitely on stalled downloads. +func NewHTTPFetcher() *HTTPFetcher { + return &HTTPFetcher{Client: &http.Client{Timeout: 10 * time.Minute}} +} + +// Download fetches a URL to dest, following redirects, and verifies the checksum. +func (f *HTTPFetcher) Download(ctx context.Context, url, dest string, checksum source.Checksum) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("creating request for %s: %w", url, err) + } + + resp, err := f.Client.Do(req) + if err != nil { + return fmt.Errorf("downloading %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("downloading %s: HTTP %d", url, resp.StatusCode) + } + + out, err := os.Create(dest) + if err != nil { + return fmt.Errorf("creating %s: %w", dest, err) + } + defer out.Close() + + var h hash.Hash + if checksum.Value != "" { + h, err = newHash(checksum.Algorithm) + if err != nil { + return err + } + } + + var w io.Writer = out + if h != nil { + w = io.MultiWriter(out, h) + } + + if _, err := io.Copy(w, resp.Body); err != nil { + os.Remove(dest) + return fmt.Errorf("writing %s: %w", dest, err) + } + + if err := out.Close(); err != nil { + return fmt.Errorf("closing %s: %w", dest, err) + } + + if h != nil { + actual := fmt.Sprintf("%x", h.Sum(nil)) + if actual != checksum.Value { + os.Remove(dest) + return fmt.Errorf("%s digest does not match: expected %s, got %s", checksum.Algorithm, checksum.Value, actual) + } + } + + return nil +} + +// ReadBody fetches a URL and returns the response body. +func (f *HTTPFetcher) ReadBody(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("creating request for %s: %w", url, err) + } + + resp, err := f.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetching %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetching %s: HTTP %d", url, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading body from %s: %w", url, err) + } + + return body, nil +} + +func newHash(algorithm string) (hash.Hash, error) { + switch algorithm { + case "sha256": + return sha256.New(), nil + case "sha512": + return sha512.New(), nil + case "md5": + return md5.New(), nil + case "sha1": + return sha1.New(), nil + default: + return nil, fmt.Errorf("unsupported checksum algorithm: %s", algorithm) + } +} diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go new file mode 100644 index 00000000..0feee071 --- /dev/null +++ b/internal/fetch/fetch_test.go @@ -0,0 +1,195 @@ +package fetch_test + +import ( + "context" + "crypto/sha256" + "crypto/sha512" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDownloadCorrectSHA256(t *testing.T) { + body := []byte("hello world") + sha := fmt.Sprintf("%x", sha256.Sum256(body)) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(body) + })) + defer srv.Close() + + f := fetch.NewHTTPFetcher() + dest := filepath.Join(t.TempDir(), "file.tgz") + + err := f.Download(context.Background(), srv.URL, dest, source.Checksum{ + Algorithm: "sha256", + Value: sha, + }) + require.NoError(t, err) + + content, err := os.ReadFile(dest) + require.NoError(t, err) + assert.Equal(t, body, content) +} + +func TestDownloadWrongSHA256(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hello world")) + })) + defer srv.Close() + + f := fetch.NewHTTPFetcher() + dest := filepath.Join(t.TempDir(), "file.tgz") + + err := f.Download(context.Background(), srv.URL, dest, source.Checksum{ + Algorithm: "sha256", + Value: "0000000000000000000000000000000000000000000000000000000000000000", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "sha256 digest does not match") + + // File should be cleaned up on checksum failure. + _, statErr := os.Stat(dest) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestDownloadWrongSHA512(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hello world")) + })) + defer srv.Close() + + f := fetch.NewHTTPFetcher() + dest := filepath.Join(t.TempDir(), "file.tgz") + + err := f.Download(context.Background(), srv.URL, dest, source.Checksum{ + Algorithm: "sha512", + Value: "0000", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "sha512 digest does not match") +} + +func TestDownloadCorrectSHA512(t *testing.T) { + body := []byte("hello world") + h := sha512.Sum512(body) + sha := fmt.Sprintf("%x", h) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(body) + })) + defer srv.Close() + + f := fetch.NewHTTPFetcher() + dest := filepath.Join(t.TempDir(), "file.tgz") + + err := f.Download(context.Background(), srv.URL, dest, source.Checksum{ + Algorithm: "sha512", + Value: sha, + }) + require.NoError(t, err) +} + +func TestDownloadNoChecksum(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("data")) + })) + defer srv.Close() + + f := fetch.NewHTTPFetcher() + dest := filepath.Join(t.TempDir(), "file.tgz") + + err := f.Download(context.Background(), srv.URL, dest, source.Checksum{}) + require.NoError(t, err) +} + +func TestDownload404(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + f := fetch.NewHTTPFetcher() + dest := filepath.Join(t.TempDir(), "file.tgz") + + err := f.Download(context.Background(), srv.URL, dest, source.Checksum{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "404") +} + +func TestReadBodySuccess(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("response body")) + })) + defer srv.Close() + + f := fetch.NewHTTPFetcher() + body, err := f.ReadBody(context.Background(), srv.URL) + require.NoError(t, err) + assert.Equal(t, []byte("response body"), body) +} + +func TestReadBody404(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + f := fetch.NewHTTPFetcher() + _, err := f.ReadBody(context.Background(), srv.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "404") +} + +func TestDownloadFollowsRedirect(t *testing.T) { + body := []byte("final content") + sha := fmt.Sprintf("%x", sha256.Sum256(body)) + + mux := http.NewServeMux() + mux.HandleFunc("/redirect", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/final", http.StatusFound) + }) + mux.HandleFunc("/final", func(w http.ResponseWriter, r *http.Request) { + w.Write(body) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + f := fetch.NewHTTPFetcher() + dest := filepath.Join(t.TempDir(), "file.tgz") + + err := f.Download(context.Background(), srv.URL+"/redirect", dest, source.Checksum{ + Algorithm: "sha256", + Value: sha, + }) + require.NoError(t, err) + + content, err := os.ReadFile(dest) + require.NoError(t, err) + assert.Equal(t, body, content) +} + +func TestDownloadUnsupportedAlgorithm(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("data")) + })) + defer srv.Close() + + f := fetch.NewHTTPFetcher() + dest := filepath.Join(t.TempDir(), "file.tgz") + + err := f.Download(context.Background(), srv.URL, dest, source.Checksum{ + Algorithm: "crc32", + Value: "abc", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported checksum algorithm") +} diff --git a/internal/fileutil/fileutil.go b/internal/fileutil/fileutil.go new file mode 100644 index 00000000..de3b40af --- /dev/null +++ b/internal/fileutil/fileutil.go @@ -0,0 +1,42 @@ +// Package fileutil provides file-system utilities shared across packages. +package fileutil + +import ( + "errors" + "fmt" + "io" + "os" + "syscall" +) + +// MoveFile moves src to dst. It tries os.Rename first; if that fails with a +// cross-device link error (EXDEV) it falls back to copy-then-delete so that +// moves across filesystem boundaries succeed. +func MoveFile(src, dst string) error { + if err := os.Rename(src, dst); err == nil { + return nil + } else if !errors.Is(err, syscall.EXDEV) { + return err + } + + // Cross-device: copy then delete. + in, err := os.Open(src) + if err != nil { + return fmt.Errorf("opening source: %w", err) + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return fmt.Errorf("creating destination: %w", err) + } + defer out.Close() + + if _, err := io.Copy(out, in); err != nil { + return fmt.Errorf("copying: %w", err) + } + if err := out.Close(); err != nil { + return fmt.Errorf("closing destination: %w", err) + } + return os.Remove(src) +} diff --git a/internal/fileutil/fileutil_test.go b/internal/fileutil/fileutil_test.go new file mode 100644 index 00000000..4fd02dbf --- /dev/null +++ b/internal/fileutil/fileutil_test.go @@ -0,0 +1,43 @@ +package fileutil_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/fileutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMoveFileSameDevice(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src.txt") + dst := filepath.Join(dir, "dst.txt") + + require.NoError(t, os.WriteFile(src, []byte("hello"), 0644)) + + require.NoError(t, fileutil.MoveFile(src, dst)) + + content, err := os.ReadFile(dst) + require.NoError(t, err) + assert.Equal(t, "hello", string(content)) + + _, err = os.Stat(src) + assert.True(t, os.IsNotExist(err), "source file should have been removed") +} + +func TestMoveFileDestinationContainsContent(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src.bin") + dst := filepath.Join(dir, "dst.bin") + + payload := []byte("binary content \x00\x01\x02") + require.NoError(t, os.WriteFile(src, payload, 0644)) + + require.NoError(t, fileutil.MoveFile(src, dst)) + + got, err := os.ReadFile(dst) + require.NoError(t, err) + assert.Equal(t, payload, got) +} diff --git a/internal/gpg/gpg.go b/internal/gpg/gpg.go new file mode 100644 index 00000000..e6c71a39 --- /dev/null +++ b/internal/gpg/gpg.go @@ -0,0 +1,75 @@ +// Package gpg provides GPG signature verification for downloaded files. +// Used by nginx and nginx-static recipes. +package gpg + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/runner" +) + +// VerifySignature downloads a file and its .asc signature, imports all +// public keys, and runs gpg --verify. Returns an error if verification fails. +// +// If gpg is not found in PATH, it is installed via apt-get before proceeding. +// +// Design note — hardcoded "gpg" package name: +// Unlike most apt packages in this codebase (which are declared in +// stacks/*.yaml so they can differ per Ubuntu version), "gpg" is intentionally +// hardcoded here. The package name is identical across all supported Ubuntu +// versions (22.04, 24.04, …) and is a universal baseline tool rather than a +// stack-specific dependency. Threading *stack.Stack through VerifySignature and +// all its callers would add API complexity for zero practical benefit. If this +// assumption ever changes (e.g. a future stack ships gpg under a different +// package name), introduce a gpg_tools key in stacks/*.yaml and pass a +// *stack.Stack parameter to this function. +func VerifySignature(ctx context.Context, fileURL, signatureURL string, publicKeyURLs []string, r runner.Runner) error { + // Install gpg if not present — mirrors Ruby's GPGHelper behaviour. + if _, err := exec.LookPath("gpg"); err != nil { + if err := r.Run("apt-get", "update"); err != nil { + return fmt.Errorf("apt-get update before gpg install: %w", err) + } + if err := r.Run("apt-get", "install", "-y", "gpg"); err != nil { + return fmt.Errorf("installing gpg: %w", err) + } + } + + tmpDir, err := os.MkdirTemp("", "gpg-verify-*") + if err != nil { + return fmt.Errorf("creating temp dir for GPG verification: %w", err) + } + defer os.RemoveAll(tmpDir) + + // Download each public key and import it. + for i, keyURL := range publicKeyURLs { + keyPath := filepath.Join(tmpDir, fmt.Sprintf("key-%d.asc", i)) + if err := r.Run("wget", "-q", "-O", keyPath, keyURL); err != nil { + return fmt.Errorf("downloading GPG key %s: %w", keyURL, err) + } + if err := r.Run("gpg", "--import", keyPath); err != nil { + return fmt.Errorf("importing GPG key %s: %w", keyURL, err) + } + } + + // Download the file and its signature. + filePath := filepath.Join(tmpDir, "file") + sigPath := filepath.Join(tmpDir, "file.asc") + + if err := r.Run("wget", "-q", "-O", filePath, fileURL); err != nil { + return fmt.Errorf("downloading file %s: %w", fileURL, err) + } + if err := r.Run("wget", "-q", "-O", sigPath, signatureURL); err != nil { + return fmt.Errorf("downloading signature %s: %w", signatureURL, err) + } + + // Verify the signature. + if err := r.Run("gpg", "--verify", sigPath, filePath); err != nil { + return fmt.Errorf("GPG verification failed for %s: %w", fileURL, err) + } + + return nil +} diff --git a/internal/gpg/gpg_test.go b/internal/gpg/gpg_test.go new file mode 100644 index 00000000..abfaa943 --- /dev/null +++ b/internal/gpg/gpg_test.go @@ -0,0 +1,88 @@ +package gpg_test + +import ( + "context" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/gpg" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVerifySignature(t *testing.T) { + f := runner.NewFakeRunner() + + keyURLs := []string{ + "http://nginx.org/keys/maxim.key", + "http://nginx.org/keys/arut.key", + } + + err := gpg.VerifySignature( + context.Background(), + "http://nginx.org/download/nginx-1.25.3.tar.gz", + "http://nginx.org/download/nginx-1.25.3.tar.gz.asc", + keyURLs, + f, + ) + require.NoError(t, err) + + // Expect: wget key0, gpg import key0, wget key1, gpg import key1, + // wget file, wget sig, gpg verify + require.Len(t, f.Calls, 7) + + // Key 0: download + import + assert.Equal(t, "wget", f.Calls[0].Name) + assert.Contains(t, f.Calls[0].Args[len(f.Calls[0].Args)-1], "maxim.key") + assert.Equal(t, "gpg", f.Calls[1].Name) + assert.Equal(t, "--import", f.Calls[1].Args[0]) + + // Key 1: download + import + assert.Equal(t, "wget", f.Calls[2].Name) + assert.Contains(t, f.Calls[2].Args[len(f.Calls[2].Args)-1], "arut.key") + assert.Equal(t, "gpg", f.Calls[3].Name) + assert.Equal(t, "--import", f.Calls[3].Args[0]) + + // File download + assert.Equal(t, "wget", f.Calls[4].Name) + assert.Contains(t, f.Calls[4].Args[len(f.Calls[4].Args)-1], "nginx-1.25.3.tar.gz") + + // Signature download + assert.Equal(t, "wget", f.Calls[5].Name) + assert.Contains(t, f.Calls[5].Args[len(f.Calls[5].Args)-1], "nginx-1.25.3.tar.gz.asc") + + // GPG verify + assert.Equal(t, "gpg", f.Calls[6].Name) + assert.Equal(t, "--verify", f.Calls[6].Args[0]) +} + +func TestVerifySignatureMultipleKeys(t *testing.T) { + f := runner.NewFakeRunner() + + keyURLs := []string{ + "http://example.com/key1.asc", + "http://example.com/key2.asc", + "http://example.com/key3.asc", + } + + err := gpg.VerifySignature( + context.Background(), + "http://example.com/file.tar.gz", + "http://example.com/file.tar.gz.asc", + keyURLs, + f, + ) + require.NoError(t, err) + + // 3 keys × 2 calls (wget + import) + 2 (file wget + sig wget) + 1 (verify) = 9 + assert.Len(t, f.Calls, 9) + + // All keys imported before verify + importCount := 0 + for _, call := range f.Calls { + if call.Name == "gpg" && len(call.Args) > 0 && call.Args[0] == "--import" { + importCount++ + } + } + assert.Equal(t, 3, importCount) +} diff --git a/internal/output/output.go b/internal/output/output.go new file mode 100644 index 00000000..e44552b1 --- /dev/null +++ b/internal/output/output.go @@ -0,0 +1,162 @@ +// Package output handles writing build output JSON files and dep-metadata JSON files. +package output + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" +) + +// SubDependency represents a sub-dependency with its source and version. +type SubDependency struct { + Source *SubDepSource `json:"source,omitempty"` + Version string `json:"version"` +} + +// SubDepSource holds the source URL and checksum for a sub-dependency. +type SubDepSource struct { + URL string `json:"url,omitempty"` + SHA256 string `json:"sha256,omitempty"` +} + +// OutData is the canonical output data structure for a dependency build. +// It is written to both builds-artifacts JSON and dep-metadata JSON. +type OutData struct { + Version string `json:"version"` + Source OutDataSource `json:"source"` + URL string `json:"url,omitempty"` + SHA256 string `json:"sha256,omitempty"` + GitCommitSHA string `json:"git_commit_sha,omitempty"` + SubDependencies map[string]SubDependency `json:"sub_dependencies,omitempty"` + + // ArtifactVersion is the version string used for the artifact filename and + // intermediate-file lookup. It is NOT serialized to JSON. When set, it + // overrides Version for artifact purposes only — allowing the dep-metadata + // and builds JSON to carry the raw source version (e.g. "9.4.14.0") while + // the artifact filename uses the full version (e.g. "9.4.14.0-ruby-3.1"). + // If empty, Version is used for both. + ArtifactVersion string `json:"-"` +} + +// OutDataSource holds the source checksums. +// MD5 and SHA1 use pointer types so they serialize as JSON null when not set, +// matching the Ruby builder's output where unset checksum fields are null. +// SHA512 uses a plain string so an empty value is preserved as "" (Ruby outputs +// the raw depwatcher value which may be an empty string for non-applicable fields). +type OutDataSource struct { + URL string `json:"url"` + MD5 *string `json:"md5"` + SHA256 string `json:"sha256"` + SHA512 string `json:"sha512"` + SHA1 *string `json:"sha1"` +} + +// NewOutData creates an OutData from a source input. +func NewOutData(src *source.Input) *OutData { + return &OutData{ + Version: src.Version, + Source: OutDataSource{ + URL: src.URL, + MD5: nullableString(src.MD5), + SHA256: src.SHA256, + SHA512: src.SHA512, + SHA1: nullableString(src.SHA1), + }, + } +} + +// nullableString converts an empty string to nil (JSON null) and a non-empty +// string to a pointer. This matches Ruby's behavior where unset values are nil +// and serialize as JSON null. +func nullableString(s string) *string { + if s == "" { + return nil + } + return &s +} + +// BuildOutput writes build JSON files and commits them to git. +type BuildOutput struct { + BaseDir string + Runner runner.Runner +} + +// NewBuildOutput creates a BuildOutput for the given dependency name. +// Creates the directory structure: {baseDir}/binary-builds-new/{name}/ +func NewBuildOutput(name string, r runner.Runner, baseDir string) (*BuildOutput, error) { + dir := filepath.Join(baseDir, "binary-builds-new", name) + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("creating build output dir %s: %w", dir, err) + } + return &BuildOutput{BaseDir: dir, Runner: r}, nil +} + +// AddOutput writes a JSON file with the given data and stages it for commit. +func (b *BuildOutput) AddOutput(filename string, data *OutData) error { + path := filepath.Join(b.BaseDir, filename) + + jsonData, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("marshaling build output: %w", err) + } + + if err := os.WriteFile(path, jsonData, 0644); err != nil { + return fmt.Errorf("writing build output %s: %w", path, err) + } + + return nil +} + +// Commit stages and commits the output file with the given message. +// It skips the commit if there are no staged changes (safe-commit behaviour). +func (b *BuildOutput) Commit(msg string) error { + if err := b.Runner.RunInDir(b.BaseDir, "git", "add", "."); err != nil { + return fmt.Errorf("running git add: %w", err) + } + + if err := b.Runner.RunInDir(b.BaseDir, "git", "config", "user.email", "cf-buildpacks-eng@pivotal.io"); err != nil { + return fmt.Errorf("setting git email: %w", err) + } + if err := b.Runner.RunInDir(b.BaseDir, "git", "config", "user.name", "CF Buildpacks Team CI Server"); err != nil { + return fmt.Errorf("setting git name: %w", err) + } + + // safe_commit: only commit when there are staged changes. + // git diff --cached --quiet exits 0 when nothing is staged, 1 when changes exist. + // RunInDir returns an error on non-zero exit, so a nil error means nothing to commit. + if err := b.Runner.RunInDir(b.BaseDir, "git", "diff", "--cached", "--quiet"); err == nil { + // Exit 0: no staged changes — nothing to commit. + return nil + } + + return b.Runner.RunInDir(b.BaseDir, "git", "commit", "-m", msg) +} + +// DepMetadataOutput writes dep-metadata JSON files. +type DepMetadataOutput struct { + BaseDir string +} + +// NewDepMetadataOutput creates a DepMetadataOutput. +func NewDepMetadataOutput(baseDir string) *DepMetadataOutput { + return &DepMetadataOutput{BaseDir: baseDir} +} + +// WriteMetadata writes the metadata JSON file for a dependency artifact. +// The filename is "{artifactFilename}_metadata.json". +func (d *DepMetadataOutput) WriteMetadata(artifactFilename string, data *OutData) error { + basename := filepath.Base(artifactFilename) + metadataFilename := basename + "_metadata.json" + path := filepath.Join(d.BaseDir, metadataFilename) + + jsonData, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("marshaling dep metadata: %w", err) + } + + return os.WriteFile(path, jsonData, 0644) +} diff --git a/internal/output/output_test.go b/internal/output/output_test.go new file mode 100644 index 00000000..c7ec9916 --- /dev/null +++ b/internal/output/output_test.go @@ -0,0 +1,252 @@ +package output_test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewOutData(t *testing.T) { + src := &source.Input{ + Name: "ruby", + Version: "3.3.6", + URL: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.6.tar.gz", + SHA256: "abc123", + SHA512: "def456", + MD5: "md5val", + } + + data := output.NewOutData(src) + + assert.Equal(t, "3.3.6", data.Version) + assert.Equal(t, "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.6.tar.gz", data.Source.URL) + assert.Equal(t, "abc123", data.Source.SHA256) + assert.Equal(t, "def456", data.Source.SHA512) + require.NotNil(t, data.Source.MD5) + assert.Equal(t, "md5val", *data.Source.MD5) +} + +func TestNewOutDataEmptyChecksums(t *testing.T) { + src := &source.Input{ + Version: "1.0.0", + URL: "https://example.com/dep.tgz", + } + + data := output.NewOutData(src) + + assert.Equal(t, "1.0.0", data.Version) + assert.Empty(t, data.Source.SHA256) + assert.Empty(t, data.Source.SHA512) + assert.Nil(t, data.Source.MD5) + assert.Nil(t, data.Source.SHA1) +} + +func TestOutDataJSON(t *testing.T) { + data := &output.OutData{ + Version: "3.3.6", + Source: output.OutDataSource{ + URL: "https://example.com/ruby.tgz", + SHA256: "abc123", + }, + URL: "https://buildpacks.cloudfoundry.org/dependencies/ruby/ruby_3.3.6_linux_x64_cflinuxfs4_abc12345.tgz", + SHA256: "fullsha256", + } + + jsonData, err := json.MarshalIndent(data, "", " ") + require.NoError(t, err) + + var parsed map[string]interface{} + require.NoError(t, json.Unmarshal(jsonData, &parsed)) + + assert.Equal(t, "3.3.6", parsed["version"]) + assert.Equal(t, "fullsha256", parsed["sha256"]) + + src := parsed["source"].(map[string]interface{}) + assert.Equal(t, "abc123", src["sha256"]) +} + +func TestOutDataWithSubDependencies(t *testing.T) { + data := &output.OutData{ + Version: "8.3.0", + Source: output.OutDataSource{ + URL: "https://example.com/php.tgz", + }, + SubDependencies: map[string]output.SubDependency{ + "redis": {Version: "6.0.2"}, + "imagick": {Version: "3.7.0"}, + }, + } + + jsonData, err := json.MarshalIndent(data, "", " ") + require.NoError(t, err) + + var parsed map[string]interface{} + require.NoError(t, json.Unmarshal(jsonData, &parsed)) + + subDeps := parsed["sub_dependencies"].(map[string]interface{}) + redis := subDeps["redis"].(map[string]interface{}) + assert.Equal(t, "6.0.2", redis["version"]) +} + +func TestOutDataWithSubDependenciesAndSource(t *testing.T) { + data := &output.OutData{ + Version: "4.3.2", + GitCommitSHA: "deadbeef", + SubDependencies: map[string]output.SubDependency{ + "forecast": { + Source: &output.SubDepSource{URL: "https://cran.r-project.org/forecast.tar.gz", SHA256: "abc"}, + Version: "8.21.1", + }, + }, + } + + jsonData, err := json.MarshalIndent(data, "", " ") + require.NoError(t, err) + + var parsed map[string]interface{} + require.NoError(t, json.Unmarshal(jsonData, &parsed)) + + assert.Equal(t, "deadbeef", parsed["git_commit_sha"]) + subDeps := parsed["sub_dependencies"].(map[string]interface{}) + forecast := subDeps["forecast"].(map[string]interface{}) + assert.Equal(t, "8.21.1", forecast["version"]) + forecastSrc := forecast["source"].(map[string]interface{}) + assert.Equal(t, "abc", forecastSrc["sha256"]) +} + +func TestBuildOutputAddOutput(t *testing.T) { + tmpDir := t.TempDir() + f := runner.NewFakeRunner() + + bo, err := output.NewBuildOutput("ruby", f, tmpDir) + require.NoError(t, err) + + data := &output.OutData{ + Version: "3.3.6", + Source: output.OutDataSource{ + URL: "https://example.com/ruby.tgz", + SHA256: "abc123", + }, + URL: "https://buildpacks.cloudfoundry.org/dependencies/ruby/ruby_3.3.6.tgz", + SHA256: "fullsha", + } + + err = bo.AddOutput("3.3.6-cflinuxfs4.json", data) + require.NoError(t, err) + + // Verify file was written. + path := filepath.Join(tmpDir, "binary-builds-new", "ruby", "3.3.6-cflinuxfs4.json") + content, err := os.ReadFile(path) + require.NoError(t, err) + + var parsed output.OutData + require.NoError(t, json.Unmarshal(content, &parsed)) + assert.Equal(t, "3.3.6", parsed.Version) + assert.Equal(t, "fullsha", parsed.SHA256) + + // AddOutput only writes the file; no git commands should be run. + assert.Empty(t, f.Calls, "AddOutput should not invoke any git commands (git add is done in Commit)") +} + +func TestDepMetadataOutputWriteMetadata(t *testing.T) { + tmpDir := t.TempDir() + + dmo := output.NewDepMetadataOutput(tmpDir) + + data := &output.OutData{ + Version: "3.3.6", + URL: "https://buildpacks.cloudfoundry.org/dependencies/ruby/ruby_3.3.6_linux_x64_cflinuxfs4_abc12345.tgz", + SHA256: "fullsha", + } + + err := dmo.WriteMetadata("ruby_3.3.6_linux_x64_cflinuxfs4_abc12345.tgz", data) + require.NoError(t, err) + + // Verify file was written with correct name. + path := filepath.Join(tmpDir, "ruby_3.3.6_linux_x64_cflinuxfs4_abc12345.tgz_metadata.json") + content, err := os.ReadFile(path) + require.NoError(t, err) + + var parsed output.OutData + require.NoError(t, json.Unmarshal(content, &parsed)) + assert.Equal(t, "3.3.6", parsed.Version) + assert.Equal(t, "fullsha", parsed.SHA256) +} + +func TestNewBuildOutputCreatesDirectory(t *testing.T) { + tmpDir := t.TempDir() + f := runner.NewFakeRunner() + + bo, err := output.NewBuildOutput("python", f, tmpDir) + require.NoError(t, err) + + expectedDir := filepath.Join(tmpDir, "binary-builds-new", "python") + info, err := os.Stat(expectedDir) + require.NoError(t, err) + assert.True(t, info.IsDir()) + assert.Equal(t, expectedDir, bo.BaseDir) +} + +func TestOutDataOmitsEmptyFields(t *testing.T) { + data := &output.OutData{ + Version: "1.0.0", + Source: output.OutDataSource{ + URL: "https://example.com/dep.tgz", + }, + } + + jsonData, err := json.Marshal(data) + require.NoError(t, err) + + // Optional top-level fields should be omitted when empty. + assert.NotContains(t, string(jsonData), "git_commit_sha") + assert.NotContains(t, string(jsonData), "sub_dependencies") + // sha256 is always serialized (even as "") to match Ruby builder output. + assert.Contains(t, string(jsonData), `"sha256":""`) +} + +func TestBuildOutputCommitWithStagedChanges(t *testing.T) { + tmpDir := t.TempDir() + f := runner.NewFakeRunner() + // Simulate staged changes: git diff --cached --quiet exits non-zero. + f.ErrorMap["git diff --cached --quiet"] = fmt.Errorf("exit status 1") + + bo, err := output.NewBuildOutput("ruby", f, tmpDir) + require.NoError(t, err) + + require.NoError(t, bo.Commit("Build ruby 3.3.6 [cflinuxfs4]")) + + // Expect: git add, git config x2, git diff --cached --quiet, git commit. + require.Len(t, f.Calls, 5) + assert.Equal(t, "git add .", f.Calls[0].String()) + assert.Equal(t, "git config user.email cf-buildpacks-eng@pivotal.io", f.Calls[1].String()) + assert.Equal(t, "git config user.name CF Buildpacks Team CI Server", f.Calls[2].String()) + assert.Equal(t, "git diff --cached --quiet", f.Calls[3].String()) + assert.Equal(t, "git commit -m Build ruby 3.3.6 [cflinuxfs4]", f.Calls[4].String()) +} + +func TestBuildOutputCommitSkipsWhenNothingStaged(t *testing.T) { + tmpDir := t.TempDir() + f := runner.NewFakeRunner() + // git diff --cached --quiet returns exit 0 (no staged changes) — no error configured. + + bo, err := output.NewBuildOutput("ruby", f, tmpDir) + require.NoError(t, err) + + require.NoError(t, bo.Commit("Build ruby 3.3.6 [cflinuxfs4]")) + + // Expect: git add, git config x2, git diff --cached --quiet — but NO git commit. + require.Len(t, f.Calls, 4) + assert.Equal(t, "git diff --cached --quiet", f.Calls[3].String()) + for _, call := range f.Calls { + assert.NotEqual(t, "commit", call.Name, "git commit should not be run when nothing is staged") + } +} diff --git a/internal/php/assets/php8-base-extensions.yml b/internal/php/assets/php8-base-extensions.yml new file mode 100644 index 00000000..a84cf9a8 --- /dev/null +++ b/internal/php/assets/php8-base-extensions.yml @@ -0,0 +1,179 @@ +--- +native_modules: +- name: rabbitmq + version: 0.11.0 + md5: e7d9896577aea6351811d7c1d7f0a68a + klass: RabbitMQRecipe +- name: lua + version: 5.4.6 + md5: 25a429319dff20dfbfb9956c2b5be911 + klass: LuaRecipe +- name: hiredis + version: 1.2.0 + md5: 119767d178cfa79718a80c83e0d0e849 + klass: HiredisRecipe +- name: snmp + version: nil + md5: nil + klass: SnmpRecipe +- name: librdkafka + version: 2.3.0 + md5: afb9d41c50535ba0aa5025ea81d39617 + klass: LibRdKafkaRecipe +- name: libsodium + version: 1.0.19 + md5: faaff6e58b34c47794a2f659d6dfd7aa + klass: LibSodiumRecipe +extensions: +- name: apcu + version: 5.1.23 + md5: c6ed350a587cf2b376c1efeb31f68907 + klass: PeclRecipe +- name: igbinary + version: 3.2.15 + md5: de81e2f54bfbe741a7f2453bccf970e9 + klass: PeclRecipe +- name: gnupg + version: 1.5.1 + md5: c48f5de2f96ffebe2e18eaefff4917f9 + klass: PeclRecipe +- name: imagick + version: 3.7.0 + md5: '0687774a6126467d4e5ede02171e981d' + klass: PeclRecipe +- name: LZF + version: 1.7.0 + md5: e242c66e53c0d155982713cd78b3d480 + klass: PeclRecipe +- name: mailparse + version: 3.1.6 + md5: e3a71b27439ee08dd63272d7d290d136 + klass: PeclRecipe +- name: mongodb + version: 1.18.1 + md5: 8939b5f966fa3ba6bb8e25206b9dae0f + klass: PeclRecipe +- name: msgpack + version: 2.2.0 + md5: 703fb4e08deac7ba68e53ee1e52bbc64 + klass: PeclRecipe +- name: oauth + version: 2.0.7 + md5: 8ea6eb5ac6de8a4ed399980848c04c0c + klass: PeclRecipe +- name: odbc + version: nil + md5: nil + klass: OdbcRecipe +- name: pdo_odbc + version: nil + md5: nil + klass: PdoOdbcRecipe +- name: pdo_sqlsrv + version: 5.12.0 + md5: 0c06402f30a7f6f0b758ad55277ad950 + klass: PeclRecipe +- name: rdkafka + version: 6.0.3 + md5: fd10faec0c599e34837a9482d3c6c801 + klass: PeclRecipe +- name: redis + version: 6.0.2 + md5: 29f1f0ba367aef7e0313cd75aa1ea83f + klass: RedisPeclRecipe +- name: ssh2 + version: 1.4.1 + md5: 9d655fb9e83aec24fcd6f943ddf94000 + klass: PeclRecipe +- name: sqlsrv + version: 5.12.0 + md5: e62485cbbcb564f4a55ab8eae40df6a6 + klass: PeclRecipe +- name: stomp + version: 2.0.3 + md5: 30de4089ae5c17f32617cef35dbe53e5 + klass: PeclRecipe +- name: xdebug + version: 3.3.2 + md5: 3c5bca14b7c638893383646565d78e9b + klass: PeclRecipe +- name: yaf + version: 3.3.5 + md5: 128ecf6c84dd71d59c12d826cc51f0c4 + klass: PeclRecipe +- name: yaml + version: 2.2.3 + md5: 8d8d18bc1d033966083ec4d6e993d61a + klass: PeclRecipe +- name: memcached + version: 3.2.0 + md5: acc58fea7b7f456408a25ac927846ad0 + klass: MemcachedPeclRecipe +- name: sodium + version: nil + md5: nil + klass: SodiumRecipe +- name: tidy + version: nil + md5: nil + klass: FakePeclRecipe +- name: enchant + version: nil + md5: nil + klass: EnchantFakePeclRecipe +- name: pdo_firebird + version: nil + md5: nil + klass: FakePeclRecipe +- name: readline + version: nil + md5: nil + klass: FakePeclRecipe +- name: zip + version: nil + md5: nil + klass: FakePeclRecipe +- name: amqp + version: 2.1.2 + md5: addd05de32a74af7d3d332b5f58b8414 + klass: AmqpPeclRecipe +- name: maxminddb + version: 1.11.1 + md5: 1e020b49b8c320b43ebfee1f31d0017b + klass: MaxMindRecipe +- name: psr + version: 1.2.0 + md5: '07695da8376002babbce528205decf07' + klass: PeclRecipe +- name: phalcon + version: 5.6.2 + md5: 806c25f017c0842798d937caaecbc994 + klass: PeclRecipe +- name: phpiredis + version: 1.0.1 + md5: '09a9bdb347c70832d3e034655b604064' + klass: PHPIRedisRecipe +- name: tideways_xhprof + version: 5.0.4 + md5: 68b68cd9410e62b8481445e0d89220c0 + klass: TidewaysXhprofRecipe +- name: solr + version: 2.7.0 + md5: 3e9df94b06f8a026e33b3a8a3f02921b + klass: PeclRecipe +- name: oci8 + version: 3.3.0 + md5: bbbbb26f1791d1f27ffc05289abee2f3 + klass: OraclePeclRecipe +- name: pdo_oci + version: nil + md5: nil + klass: OraclePdoRecipe +- name: gd + version: nil + md5: nil + klass: Gd74FakePeclRecipe +- name: ioncube + version: 13.0.2 + md5: 0526cd3702ef25de119e4724d603d773 + klass: IonCubeRecipe diff --git a/internal/php/assets/php81-extensions-patch.yml b/internal/php/assets/php81-extensions-patch.yml new file mode 100644 index 00000000..344a48c6 --- /dev/null +++ b/internal/php/assets/php81-extensions-patch.yml @@ -0,0 +1,8 @@ +--- +extensions: + exclusions: + additions: + - name: oci8 + version: 3.2.1 + md5: 309190ef3ede2779a617c9375d32ea7a + klass: OraclePeclRecipe diff --git a/internal/php/assets/php82-extensions-patch.yml b/internal/php/assets/php82-extensions-patch.yml new file mode 100644 index 00000000..e84a33c2 --- /dev/null +++ b/internal/php/assets/php82-extensions-patch.yml @@ -0,0 +1,12 @@ +--- +extensions: + exclusions: + - name: yaf + version: 3.3.5 + md5: 128ecf6c84dd71d59c12d826cc51f0c4 + klass: PeclRecipe + additions: + - name: oci8 + version: 3.3.0 + md5: bbbbb26f1791d1f27ffc05289abee2f3 + klass: OraclePeclRecipe diff --git a/internal/php/assets/php83-extensions-patch.yml b/internal/php/assets/php83-extensions-patch.yml new file mode 100644 index 00000000..9c7c0450 --- /dev/null +++ b/internal/php/assets/php83-extensions-patch.yml @@ -0,0 +1,8 @@ +--- +extensions: + exclusions: + - name: yaf + version: 3.3.5 + md5: 128ecf6c84dd71d59c12d826cc51f0c4 + klass: PeclRecipe + additions: diff --git a/internal/php/extensions.go b/internal/php/extensions.go new file mode 100644 index 00000000..4d25082d --- /dev/null +++ b/internal/php/extensions.go @@ -0,0 +1,189 @@ +// Package php provides extension loading and recipe implementations for PHP builds. +// +// # Extension data files +// +// The YAML files that define which PHP extensions and native modules are built +// live in assets/ alongside the Go source: +// +// internal/php/assets/ +// php8-base-extensions.yml — full list for all PHP 8.x (native modules + PECL extensions) +// php81-extensions-patch.yml — overrides/exclusions specific to PHP 8.1.x +// php82-extensions-patch.yml — overrides/exclusions specific to PHP 8.2.x +// php83-extensions-patch.yml — overrides/exclusions specific to PHP 8.3.x +// +// # Adding a new PHP minor version (e.g. 8.4) +// +// 1. Create assets/php84-extensions-patch.yml with any additions or exclusions +// relative to the PHP 8 base (an empty patch is valid: "---\nextensions:\n"). +// +// No code changes are required — the file is discovered automatically via the +// embedded FS glob. +// +// # Adding a new PHP major version (e.g. 9) +// +// 1. Create assets/php9-base-extensions.yml with the full extension list for PHP 9.x. +// 2. Create a patch file for each shipped minor version (e.g. assets/php90-extensions-patch.yml). +// +// Again, no code changes are required. +// +// # File naming convention (drives auto-discovery) +// +// - Base files: php{major}-base-extensions.yml (e.g. php8-base-extensions.yml) +// - Patch files: php{major}{minor}-extensions-patch.yml (e.g. php84-extensions-patch.yml) +// +// Important: only .yml files are embedded (the glob is assets/*.yml). A file with a +// .yaml extension would be silently ignored. +package php + +import ( + "embed" + "fmt" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +//go:embed assets/*.yml +var assetsFS embed.FS + +var ( + // baseFileRE matches e.g. "assets/php8-base-extensions.yml" and captures the major version ("8"). + baseFileRE = regexp.MustCompile(`^assets/php(\d+)-base-extensions\.yml$`) + // patchFileRE matches e.g. "assets/php83-extensions-patch.yml" and captures major+minor ("83"). + // The \d{2,} quantifier is intentional: patch filenames must encode both the major AND minor + // digit together (e.g. "php81", "php90"), never a bare major digit. This prevents a file named + // "php8-extensions-patch.yml" from matching (that would be a base file), and ensures a future + // PHP 9 patch is named "php90-extensions-patch.yml", not "php9-extensions-patch.yml" (which + // would be silently ignored). + patchFileRE = regexp.MustCompile(`^assets/php(\d{2,})-extensions-patch\.yml$`) +) + +// embeddedBases maps PHP major version (e.g. "8") → base YAML bytes. +// embeddedPatches maps PHP major+minor (e.g. "83") → patch YAML bytes. +// Both are populated once at init() from the embedded FS. +var ( + embeddedBases map[string][]byte + embeddedPatches map[string][]byte +) + +func init() { + embeddedBases = make(map[string][]byte) + embeddedPatches = make(map[string][]byte) + + entries, err := assetsFS.ReadDir("assets") + if err != nil { + panic("php/extensions: cannot read embedded assets: " + err.Error()) + } + + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".yml") { + continue + } + path := "assets/" + e.Name() + data, err := assetsFS.ReadFile(path) + if err != nil { + panic("php/extensions: cannot read embedded file " + path + ": " + err.Error()) + } + + if m := baseFileRE.FindStringSubmatch(path); m != nil { + embeddedBases[m[1]] = data + } else if m := patchFileRE.FindStringSubmatch(path); m != nil { + embeddedPatches[m[1]] = data + } + } +} + +// Extension represents a single PHP extension or native module. +type Extension struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + MD5 string `yaml:"md5"` + Klass string `yaml:"klass"` +} + +// ExtensionSet holds the full set of native modules and extensions for a PHP build. +type ExtensionSet struct { + NativeModules []Extension `yaml:"native_modules"` + Extensions []Extension `yaml:"extensions"` +} + +// patchYAML is the structure of a php{major}{minor}-extensions-patch.yml file. +type patchYAML struct { + NativeModules *patchCategory `yaml:"native_modules"` + Extensions *patchCategory `yaml:"extensions"` +} + +type patchCategory struct { + Additions []Extension `yaml:"additions"` + Exclusions []Extension `yaml:"exclusions"` +} + +// Load returns the ExtensionSet for the given PHP major+minor version by reading +// the embedded YAML data compiled into this package. +// +// It loads the base file for the major version (e.g. "8" → php8-base-extensions.yml), +// then applies the patch file for the specific minor version if one exists +// (e.g. "8"+"2" → php82-extensions-patch.yml). +// +// Merge rules: +// - For each addition: if name already exists → replace; otherwise → append. +// - For each exclusion: remove by name. +func Load(phpMajor, phpMinor string) (*ExtensionSet, error) { + baseData, ok := embeddedBases[phpMajor] + if !ok { + return nil, fmt.Errorf("php/extensions: no base extensions file for PHP major version %q", phpMajor) + } + + var set ExtensionSet + if err := yaml.Unmarshal(baseData, &set); err != nil { + return nil, fmt.Errorf("php/extensions: parsing base file for PHP %s: %w", phpMajor, err) + } + + patchData, ok := embeddedPatches[phpMajor+phpMinor] + if !ok { + // No patch for this minor version — return base as-is. + return &set, nil + } + + var patch patchYAML + if err := yaml.Unmarshal(patchData, &patch); err != nil { + return nil, fmt.Errorf("php/extensions: parsing patch file for PHP %s.%s: %w", phpMajor, phpMinor, err) + } + + applyPatch(&set.NativeModules, patch.NativeModules) + applyPatch(&set.Extensions, patch.Extensions) + + return &set, nil +} + +// applyPatch applies additions (override by name or append) and exclusions +// (remove by name) from a patch category to a slice of extensions. +func applyPatch(list *[]Extension, cat *patchCategory) { + if cat == nil { + return + } + + for _, add := range cat.Additions { + if idx := indexByName(*list, add.Name); idx >= 0 { + (*list)[idx] = add + } else { + *list = append(*list, add) + } + } + + for _, excl := range cat.Exclusions { + if idx := indexByName(*list, excl.Name); idx >= 0 { + *list = append((*list)[:idx], (*list)[idx+1:]...) + } + } +} + +func indexByName(list []Extension, name string) int { + for i, e := range list { + if e.Name == name { + return i + } + } + return -1 +} diff --git a/internal/php/extensions_test.go b/internal/php/extensions_test.go new file mode 100644 index 00000000..682cc435 --- /dev/null +++ b/internal/php/extensions_test.go @@ -0,0 +1,141 @@ +package php_test + +import ( + "testing" + + "github.com/cloudfoundry/binary-builder/internal/php" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestLoad_PatchOverridesExisting verifies that an addition whose name already +// exists in the base replaces the existing entry (override path), and that the +// result contains exactly one entry for that name (no duplicates). +// php81-extensions-patch.yml overrides oci8, which is also present in the base. +// +// TODO: the pure-append path (patch adds a name not present in the base at all) +// has no integration-level coverage here; it would require a dedicated patch +// fixture that introduces a genuinely new extension name. +func TestLoad_PatchOverridesExisting(t *testing.T) { + // php81 patch adds oci8 (as an override — oci8 exists in base with a + // different version). Verify the result contains exactly one oci8 entry. + set, err := php.Load("8", "1") + require.NoError(t, err) + + var oci8Count int + for _, e := range set.Extensions { + if e.Name == "oci8" { + oci8Count++ + } + } + assert.Equal(t, 1, oci8Count, "oci8 should appear exactly once after patch") +} + +// TestLoad_PatchOverridesVersion verifies that an addition whose name already +// exists in the base replaces the existing entry (version override). +// php81-extensions-patch.yml overrides oci8 to version 3.2.1. +func TestLoad_PatchOverridesVersion(t *testing.T) { + set, err := php.Load("8", "1") + require.NoError(t, err) + + var oci8 *php.Extension + for i := range set.Extensions { + if set.Extensions[i].Name == "oci8" { + oci8 = &set.Extensions[i] + } + } + require.NotNil(t, oci8, "oci8 should be present after php81 patch") + assert.Equal(t, "3.2.1", oci8.Version) + assert.Equal(t, "309190ef3ede2779a617c9375d32ea7a", oci8.MD5) +} + +// TestLoad_PatchRemovesExclusion verifies that an exclusion in a patch file +// removes the named extension from the result. +// php82-extensions-patch.yml excludes yaf. +func TestLoad_PatchRemovesExclusion(t *testing.T) { + set, err := php.Load("8", "2") + require.NoError(t, err) + + for _, e := range set.Extensions { + assert.NotEqual(t, "yaf", e.Name, "yaf should be excluded in php82") + } +} + +// TestLoad_PatchNativeModuleAddition verifies that native modules are +// unaffected when a patch file has no native_modules section. +// php83-extensions-patch.yml has no native_modules section. +func TestLoad_PatchNativeModuleAddition(t *testing.T) { + set, err := php.Load("8", "3") + require.NoError(t, err) + + // Native modules from the base file should be present unchanged. + nativeNames := make([]string, len(set.NativeModules)) + for i, m := range set.NativeModules { + nativeNames[i] = m.Name + } + assert.Contains(t, nativeNames, "rabbitmq") + assert.Contains(t, nativeNames, "lua") +} + +// TestLoad_MissingBaseFile verifies that Load returns an error (mentioning +// "base") when no base file exists for the requested major version. +func TestLoad_MissingBaseFile(t *testing.T) { + _, err := php.Load("9", "0") + require.Error(t, err) + assert.Contains(t, err.Error(), "base") +} + +// --- Smoke tests against the real embedded YAML files --- + +func TestLoad_RealPhp8BaseFile(t *testing.T) { + // Smoke-test: load the real php8-base-extensions.yml embedded in this package. + set, err := php.Load("8", "4") + require.NoError(t, err) + + // The base file must have the expected native modules. + nativeNames := make([]string, len(set.NativeModules)) + for i, m := range set.NativeModules { + nativeNames[i] = m.Name + } + assert.Contains(t, nativeNames, "rabbitmq") + assert.Contains(t, nativeNames, "lua") + assert.Contains(t, nativeNames, "hiredis") + + // Must have a meaningful number of extensions. + assert.Greater(t, len(set.Extensions), 20) +} + +func TestLoad_RealPhp81Patch(t *testing.T) { + set, err := php.Load("8", "1") + require.NoError(t, err) + + // php81 patch overrides oci8 version to 3.2.1 + var oci8 *php.Extension + for i := range set.Extensions { + if set.Extensions[i].Name == "oci8" { + oci8 = &set.Extensions[i] + } + } + require.NotNil(t, oci8, "oci8 should be present") + assert.Equal(t, "3.2.1", oci8.Version) +} + +func TestLoad_RealPhp82Patch(t *testing.T) { + set, err := php.Load("8", "2") + require.NoError(t, err) + + // php82 patch removes yaf + for _, e := range set.Extensions { + assert.NotEqual(t, "yaf", e.Name, "yaf should be excluded in php82") + } +} + +func TestLoad_RealPhp83Patch(t *testing.T) { + set, err := php.Load("8", "3") + require.NoError(t, err) + + // php83 patch also removes yaf + for _, e := range set.Extensions { + assert.NotEqual(t, "yaf", e.Name, "yaf should be excluded in php83") + } +} diff --git a/internal/php/fake_pecl.go b/internal/php/fake_pecl.go new file mode 100644 index 00000000..663be488 --- /dev/null +++ b/internal/php/fake_pecl.go @@ -0,0 +1,203 @@ +package php + +import ( + "context" + "fmt" + "os" + + "github.com/cloudfoundry/binary-builder/internal/runner" +) + +// FakePeclRecipe builds built-in PHP extensions that live inside the PHP source tree +// under ext/{name}. It tars up the source directory and then uses the PECL build steps. +type FakePeclRecipe struct{} + +func (f *FakePeclRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + opts := []string{fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath)} + return buildFakePecl(ctx, ext.Name, ec, opts, run) +} + +// SodiumRecipe builds the sodium extension (built-in since PHP 7.2) against libsodium. +type SodiumRecipe struct{} + +func (s *SodiumRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + opts := []string{ + fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath), + fmt.Sprintf("--with-sodium=%s", ec.LibSodiumPath), + } + env := map[string]string{ + "LDFLAGS": fmt.Sprintf("-L%s/lib", ec.LibSodiumPath), + "PKG_CONFIG_PATH": fmt.Sprintf("%s/lib/pkgconfig", ec.LibSodiumPath), + } + if err := buildFakePeclWithEnv(ctx, ext.Name, ec, opts, env, run); err != nil { + return err + } + // Copy libsodium shared libs into the PHP prefix. + return run.Run("sh", "-c", fmt.Sprintf("cp -a %s/lib/libsodium.so* %s/lib/", ec.LibSodiumPath, ec.PHPPath)) +} + +// OdbcRecipe builds the ODBC extension (built-in) with unixODBC. +type OdbcRecipe struct{} + +func (o *OdbcRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + workDir := fmt.Sprintf("%s/ext/odbc", ec.PHPSourceDir) + // Patch config.m4 to add AC_DEFUN before contents. + patchScript := fmt.Sprintf( + `cd %s && echo 'AC_DEFUN([PHP_ALWAYS_SHARED],[])dnl' > temp.m4 && echo >> temp.m4 && cat config.m4 >> temp.m4 && mv temp.m4 config.m4`, + workDir, + ) + if err := run.Run("sh", "-c", patchScript); err != nil { + return fmt.Errorf("php/odbc: patch config.m4: %w", err) + } + + opts := []string{"--with-unixODBC=shared,/usr"} + if err := buildFakePeclFromDir(ctx, "odbc", workDir, ec, opts, run); err != nil { + return err + } + // Copy odbc libs into PHP prefix. + return run.Run("sh", "-c", fmt.Sprintf( + "cp -a /usr/lib/x86_64-linux-gnu/libodbc.so* %s/lib/ && cp -a /usr/lib/x86_64-linux-gnu/libodbcinst.so* %s/lib/", + ec.PHPPath, ec.PHPPath, + )) +} + +// PdoOdbcRecipe builds the PDO ODBC extension (built-in) with unixODBC. +type PdoOdbcRecipe struct{} + +func (p *PdoOdbcRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + opts := []string{"--with-pdo-odbc=unixODBC,/usr"} + if err := buildFakePecl(ctx, "pdo_odbc", ec, opts, run); err != nil { + return err + } + return run.Run("sh", "-c", fmt.Sprintf( + "cp -a /usr/lib/x86_64-linux-gnu/libodbc.so* %s/lib/ && cp -a /usr/lib/x86_64-linux-gnu/libodbcinst.so* %s/lib/", + ec.PHPPath, ec.PHPPath, + )) +} + +// SnmpRecipe copies SNMP system libraries and mibs into the PHP prefix. +// No external download or compilation — just copies from system paths. +type SnmpRecipe struct{} + +func (s *SnmpRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + script := fmt.Sprintf(` +cd %s +mkdir -p mibs +cp -a /usr/lib/x86_64-linux-gnu/libnetsnmp.so* lib/ +cp -rL /usr/share/snmp/mibs/iana mibs/ +cp -rL /usr/share/snmp/mibs/ietf mibs/ +cp /usr/share/snmp/mibs/*.txt mibs/ +cp /usr/bin/download-mibs bin/ +cp /usr/bin/smistrip bin/ +sed -i "s|^CONFDIR=/etc/snmp-mibs-downloader|CONFDIR=\$HOME/php/mibs/conf|" bin/download-mibs +sed -i "s|^SMISTRIP=/usr/bin/smistrip|SMISTRIP=\$HOME/php/bin/smistrip|" bin/download-mibs +cp -R /etc/snmp-mibs-downloader mibs/conf +cp -R /etc/snmp-mibs-downloader mibs/conf +sed -i "s|^DIR=/usr/share/doc|DIR=\$HOME/php/mibs/originals|" mibs/conf/iana.conf +sed -i "s|^DEST=iana|DEST=|" mibs/conf/iana.conf +sed -i "s|^DIR=/usr/share/doc|DIR=\$HOME/php/mibs/originals|" mibs/conf/ianarfc.conf +sed -i "s|^DEST=iana|DEST=|" mibs/conf/ianarfc.conf +sed -i "s|^DIR=/usr/share/doc|DIR=\$HOME/php/mibs/originals|" mibs/conf/rfc.conf +sed -i "s|^DEST=ietf|DEST=|" mibs/conf/rfc.conf +sed -i "s|^BASEDIR=/var/lib/mibs|BASEDIR=\$HOME/php/mibs|" mibs/conf/snmp-mibs-downloader.conf +`, ec.PHPPath) + if err := run.Run("sh", "-c", script); err != nil { + return fmt.Errorf("php/snmp: setup: %w", err) + } + return nil +} + +// Gd74FakePeclRecipe builds the GD extension (PHP 7.4+) using the system libgd. +type Gd74FakePeclRecipe struct{} + +func (g *Gd74FakePeclRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + opts := []string{"--with-external-gd"} + return buildFakePecl(ctx, "gd", ec, opts, run) +} + +// EnchantFakePeclRecipe builds the enchant extension with a source patch. +type EnchantFakePeclRecipe struct{} + +func (e *EnchantFakePeclRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + workDir := fmt.Sprintf("%s/ext/enchant", ec.PHPSourceDir) + // Patch the broken include path. + patchScript := fmt.Sprintf( + `cd %s && sed -i 's|#include "../spl/spl_exceptions.h"|#include |' enchant.c`, + workDir, + ) + if err := run.Run("sh", "-c", patchScript); err != nil { + return fmt.Errorf("php/enchant: patch: %w", err) + } + opts := []string{fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath)} + return buildFakePeclFromDir(ctx, "enchant", workDir, ec, opts, run) +} + +// buildFakePecl archives ec.PHPSourceDir/ext/{name}, then runs phpize+configure+make+install. +func buildFakePecl(ctx context.Context, name string, ec ExtensionContext, opts []string, run runner.Runner) error { + workDir := fmt.Sprintf("%s/ext/%s", ec.PHPSourceDir, name) + return buildFakePeclFromDir(ctx, name, workDir, ec, opts, run) +} + +// buildFakePeclWithEnv is like buildFakePecl but runs phpize/configure with extra env vars. +func buildFakePeclWithEnv(ctx context.Context, name string, ec ExtensionContext, opts []string, env map[string]string, run runner.Runner) error { + workDir := fmt.Sprintf("%s/ext/%s", ec.PHPSourceDir, name) + + // Merge PHP bin dir into PATH so phpize and php-config are found by name. + mergedEnv := mergePHPBinPath(ec.PHPPath, env) + + if err := run.RunInDirWithEnv(workDir, mergedEnv, fmt.Sprintf("%s/bin/phpize", ec.PHPPath)); err != nil { + return fmt.Errorf("php/%s: phpize: %w", name, err) + } + + configureArgs := append([]string{"./configure"}, opts...) + if err := run.RunInDirWithEnv(workDir, mergedEnv, "sh", configureArgs...); err != nil { + return fmt.Errorf("php/%s: configure: %w", name, err) + } + + if err := run.RunInDir(workDir, "make"); err != nil { + return fmt.Errorf("php/%s: make: %w", name, err) + } + + if err := run.RunInDir(workDir, "make", "install"); err != nil { + return fmt.Errorf("php/%s: make install: %w", name, err) + } + + return nil +} + +// buildFakePeclFromDir runs phpize + configure + make + make install in the given directory. +func buildFakePeclFromDir(ctx context.Context, name, workDir string, ec ExtensionContext, opts []string, run runner.Runner) error { + // Prepend PHP bin dir to PATH so phpize and php-config are found by name. + phpBinEnv := mergePHPBinPath(ec.PHPPath, nil) + + if err := run.RunInDirWithEnv(workDir, phpBinEnv, fmt.Sprintf("%s/bin/phpize", ec.PHPPath)); err != nil { + return fmt.Errorf("php/%s: phpize: %w", name, err) + } + + configureArgs := append([]string{"./configure"}, opts...) + if err := run.RunInDirWithEnv(workDir, phpBinEnv, "sh", configureArgs...); err != nil { + return fmt.Errorf("php/%s: configure: %w", name, err) + } + + if err := run.RunInDir(workDir, "make"); err != nil { + return fmt.Errorf("php/%s: make: %w", name, err) + } + + if err := run.RunInDir(workDir, "make", "install"); err != nil { + return fmt.Errorf("php/%s: make install: %w", name, err) + } + + return nil +} + +// mergePHPBinPath returns an env map with PATH prepended with phpPath/bin. +// If extra is non-nil, its entries are merged in too. +func mergePHPBinPath(phpPath string, extra map[string]string) map[string]string { + env := map[string]string{ + "PATH": fmt.Sprintf("%s/bin:%s", phpPath, os.Getenv("PATH")), + } + for k, v := range extra { + env[k] = v + } + return env +} diff --git a/internal/php/native.go b/internal/php/native.go new file mode 100644 index 00000000..69bdee20 --- /dev/null +++ b/internal/php/native.go @@ -0,0 +1,36 @@ +package php + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" +) + +// LuaRecipe downloads and builds Lua from lua.org. +// Compiles with `make linux MYCFLAGS=-fPIC` and installs with `make install INSTALL_TOP={path}`. +type LuaRecipe struct{} + +func (l *LuaRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("http://www.lua.org/ftp/lua-%s.tar.gz", ext.Version) + archiveName := fmt.Sprintf("lua-%s.tar.gz", ext.Version) + dest := fmt.Sprintf("/tmp/%s", archiveName) + srcDir := fmt.Sprintf("/tmp/lua-%s", ext.Version) + installPath := fmt.Sprintf("/tmp/lua-install-%s", ext.Version) + + if err := ec.Fetcher.Download(ctx, url, dest, source.Checksum{}); err != nil { + return fmt.Errorf("php/lua: download: %w", err) + } + if err := run.Run("tar", "xzf", dest, "-C", "/tmp/"); err != nil { + return fmt.Errorf("php/lua: extract: %w", err) + } + if err := run.RunInDir(srcDir, "bash", "-c", "make linux MYCFLAGS=-fPIC"); err != nil { + return fmt.Errorf("php/lua: make: %w", err) + } + installCmd := fmt.Sprintf("make install INSTALL_TOP=%s", installPath) + if err := run.RunInDir(srcDir, "bash", "-c", installCmd); err != nil { + return fmt.Errorf("php/lua: make install: %w", err) + } + return nil +} diff --git a/internal/php/pecl.go b/internal/php/pecl.go new file mode 100644 index 00000000..084faeac --- /dev/null +++ b/internal/php/pecl.go @@ -0,0 +1,156 @@ +package php + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" +) + +// PeclRecipe downloads an extension from pecl.php.net and builds it with +// phpize + ./configure + make + make install. +// +// URL: http://pecl.php.net/get/{name}-{version}.tgz +type PeclRecipe struct{} + +func (p *PeclRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("http://pecl.php.net/get/%s-%s.tgz", ext.Name, ext.Version) + checksum := source.Checksum{Algorithm: "md5", Value: ext.MD5} + return buildPecl(ctx, ext.Name, ext.Version, url, checksum, ec, p.configureOptions(ec), run) +} + +func (p *PeclRecipe) configureOptions(ec ExtensionContext) []string { + return []string{fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath)} +} + +// AmqpPeclRecipe builds the amqp extension against the rabbitmq-c library. +type AmqpPeclRecipe struct{} + +func (a *AmqpPeclRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("http://pecl.php.net/get/%s-%s.tgz", ext.Name, ext.Version) + checksum := source.Checksum{Algorithm: "md5", Value: ext.MD5} + opts := []string{ + fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath), + "--with-amqp", + fmt.Sprintf("--with-librabbitmq-dir=%s", ec.RabbitMQPath), + } + return buildPecl(ctx, ext.Name, ext.Version, url, checksum, ec, opts, run) +} + +// MaxMindRecipe builds the maxminddb extension; the work dir is maxminddb-{version}/ext. +type MaxMindRecipe struct{} + +func (m *MaxMindRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("http://pecl.php.net/get/%s-%s.tgz", ext.Name, ext.Version) + checksum := source.Checksum{Algorithm: "md5", Value: ext.MD5} + opts := []string{fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath)} + return buildPeclInSubdir(ctx, ext.Name, ext.Version, url, checksum, fmt.Sprintf("maxminddb-%s/ext", ext.Version), ec, opts, run) +} + +// RedisPeclRecipe builds the redis extension with igbinary and lzf support. +type RedisPeclRecipe struct{} + +func (r *RedisPeclRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("http://pecl.php.net/get/%s-%s.tgz", ext.Name, ext.Version) + checksum := source.Checksum{Algorithm: "md5", Value: ext.MD5} + opts := []string{ + fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath), + "--enable-redis-igbinary", + "--enable-redis-lzf", + "--with-liblzf=no", + } + return buildPecl(ctx, ext.Name, ext.Version, url, checksum, ec, opts, run) +} + +// MemcachedPeclRecipe builds the memcached extension with all optional features. +type MemcachedPeclRecipe struct{} + +func (m *MemcachedPeclRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("http://pecl.php.net/get/%s-%s.tgz", ext.Name, ext.Version) + checksum := source.Checksum{Algorithm: "md5", Value: ext.MD5} + opts := []string{ + fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath), + "--with-libmemcached-dir", + "--enable-memcached-sasl", + "--enable-memcached-msgpack", + "--enable-memcached-igbinary", + "--enable-memcached-json", + } + return buildPecl(ctx, ext.Name, ext.Version, url, checksum, ec, opts, run) +} + +// TidewaysXhprofRecipe builds the tideways_xhprof extension from GitHub. +type TidewaysXhprofRecipe struct{} + +func (t *TidewaysXhprofRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("https://github.com/tideways/php-xhprof-extension/archive/v%s.tar.gz", ext.Version) + opts := []string{fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath)} + // GitHub archive extracts as "php-xhprof-extension-{version}", not "tideways_xhprof-{version}". + // No MD5 available for GitHub archive downloads — checksum.Value is empty, so verification is skipped. + return buildPeclInSubdir(ctx, ext.Name, ext.Version, url, source.Checksum{}, fmt.Sprintf("php-xhprof-extension-%s", ext.Version), ec, opts, run) +} + +// PHPIRedisRecipe builds the phpiredis extension from GitHub against hiredis. +type PHPIRedisRecipe struct{} + +func (p *PHPIRedisRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("https://github.com/nrk/phpiredis/archive/v%s.tar.gz", ext.Version) + opts := []string{ + fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath), + "--enable-phpiredis", + fmt.Sprintf("--with-hiredis-dir=%s", ec.HiredisPath), + } + // No MD5 available for GitHub archive downloads — checksum.Value is empty, so verification is skipped. + return buildPecl(ctx, ext.Name, ext.Version, url, source.Checksum{}, ec, opts, run) +} + +// buildPecl is the shared PECL build helper: download → phpize → configure → make → make install. +func buildPecl(ctx context.Context, name, version, url string, checksum source.Checksum, ec ExtensionContext, opts []string, run runner.Runner) error { + return buildPeclInSubdir(ctx, name, version, url, checksum, fmt.Sprintf("%s-%s", name, version), ec, opts, run) +} + +// buildPeclInSubdir is like buildPecl but uses a custom subdirectory inside the extracted archive. +func buildPeclInSubdir(ctx context.Context, name, version, url string, checksum source.Checksum, subdir string, ec ExtensionContext, opts []string, run runner.Runner) error { + archiveName := fmt.Sprintf("%s-%s.tgz", name, version) + workDir := fmt.Sprintf("/tmp/php-ext-build/%s", subdir) + + // Download with checksum verification via Fetcher. + // If checksum.Value is empty (GitHub archives without MD5), verification is skipped. + if err := ec.Fetcher.Download(ctx, url, fmt.Sprintf("/tmp/%s", archiveName), checksum); err != nil { + return fmt.Errorf("php/%s: download: %w", name, err) + } + + // Ensure the extraction directory exists. + if err := run.Run("mkdir", "-p", "/tmp/php-ext-build/"); err != nil { + return fmt.Errorf("php/%s: mkdir ext-build: %w", name, err) + } + + // Extract. + if err := run.Run("tar", "xzf", fmt.Sprintf("/tmp/%s", archiveName), "-C", "/tmp/php-ext-build/"); err != nil { + return fmt.Errorf("php/%s: extract: %w", name, err) + } + + // phpize. + if err := run.RunInDir(workDir, fmt.Sprintf("%s/bin/phpize", ec.PHPPath)); err != nil { + return fmt.Errorf("php/%s: phpize: %w", name, err) + } + + // configure. + configureArgs := append([]string{"./configure"}, opts...) + if err := run.RunInDir(workDir, "sh", configureArgs...); err != nil { + return fmt.Errorf("php/%s: configure: %w", name, err) + } + + // make. + if err := run.RunInDir(workDir, "make"); err != nil { + return fmt.Errorf("php/%s: make: %w", name, err) + } + + // make install. + if err := run.RunInDir(workDir, "make", "install"); err != nil { + return fmt.Errorf("php/%s: make install: %w", name, err) + } + + return nil +} diff --git a/internal/php/pkgconfig.go b/internal/php/pkgconfig.go new file mode 100644 index 00000000..0120b5d4 --- /dev/null +++ b/internal/php/pkgconfig.go @@ -0,0 +1,120 @@ +package php + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" +) + +// HiredisRecipe downloads and builds the hiredis C library from GitHub. +// Uses LIBRARY_PATH=lib PREFIX={path} make install (no autoconf configure). +type HiredisRecipe struct{} + +func (h *HiredisRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("https://github.com/redis/hiredis/archive/v%s.tar.gz", ext.Version) + archiveName := fmt.Sprintf("hiredis-%s.tar.gz", ext.Version) + dest := fmt.Sprintf("/tmp/%s", archiveName) + srcDir := fmt.Sprintf("/tmp/hiredis-%s", ext.Version) + installPath := fmt.Sprintf("/tmp/hiredis-install-%s", ext.Version) + + if err := ec.Fetcher.Download(ctx, url, dest, source.Checksum{}); err != nil { + return fmt.Errorf("php/hiredis: download: %w", err) + } + if err := run.Run("tar", "xzf", dest, "-C", "/tmp/"); err != nil { + return fmt.Errorf("php/hiredis: extract: %w", err) + } + installCmd := fmt.Sprintf("LIBRARY_PATH=lib PREFIX='%s' make install", installPath) + if err := run.RunInDir(srcDir, "bash", "-c", installCmd); err != nil { + return fmt.Errorf("php/hiredis: make install: %w", err) + } + // Expose install path via ec (caller sets ec.HiredisPath = installPath). + _ = installPath + return nil +} + +// RabbitMQRecipe downloads and builds rabbitmq-c from GitHub using cmake. +type RabbitMQRecipe struct{} + +func (r *RabbitMQRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("https://github.com/alanxz/rabbitmq-c/archive/v%s.tar.gz", ext.Version) + archiveName := fmt.Sprintf("rabbitmq-%s.tar.gz", ext.Version) + dest := fmt.Sprintf("/tmp/%s", archiveName) + srcDir := fmt.Sprintf("/tmp/rabbitmq-c-%s", ext.Version) + + if err := ec.Fetcher.Download(ctx, url, dest, source.Checksum{}); err != nil { + return fmt.Errorf("php/rabbitmq: download: %w", err) + } + if err := run.Run("tar", "xzf", dest, "-C", "/tmp/"); err != nil { + return fmt.Errorf("php/rabbitmq: extract: %w", err) + } + for _, step := range [][]string{ + {"cmake", "."}, + {"cmake", "--build", "."}, + {"cmake", "-DCMAKE_INSTALL_PREFIX=/usr/local", "."}, + {"cmake", "--build", ".", "--target", "install"}, + } { + if err := run.RunInDir(srcDir, step[0], step[1:]...); err != nil { + return fmt.Errorf("php/rabbitmq: cmake step %v: %w", step, err) + } + } + return nil +} + +// LibRdKafkaRecipe downloads and builds librdkafka from GitHub. +// Uses ./configure --prefix=/usr then make + make install. +type LibRdKafkaRecipe struct{} + +func (l *LibRdKafkaRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("https://github.com/edenhill/librdkafka/archive/v%s.tar.gz", ext.Version) + archiveName := fmt.Sprintf("librdkafka-%s.tar.gz", ext.Version) + dest := fmt.Sprintf("/tmp/%s", archiveName) + srcDir := fmt.Sprintf("/tmp/librdkafka-%s", ext.Version) + + if err := ec.Fetcher.Download(ctx, url, dest, source.Checksum{}); err != nil { + return fmt.Errorf("php/librdkafka: download: %w", err) + } + if err := run.Run("tar", "xzf", dest, "-C", "/tmp/"); err != nil { + return fmt.Errorf("php/librdkafka: extract: %w", err) + } + if err := run.RunInDir(srcDir, "bash", "./configure", "--prefix=/usr"); err != nil { + return fmt.Errorf("php/librdkafka: configure: %w", err) + } + if err := run.RunInDir(srcDir, "make"); err != nil { + return fmt.Errorf("php/librdkafka: make: %w", err) + } + if err := run.RunInDir(srcDir, "make", "install"); err != nil { + return fmt.Errorf("php/librdkafka: make install: %w", err) + } + return nil +} + +// LibSodiumRecipe downloads and builds libsodium from GitHub. +// Uses ./configure + make + make install (standard autoconf). +type LibSodiumRecipe struct{} + +func (l *LibSodiumRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("https://github.com/jedisct1/libsodium/archive/%s-RELEASE.tar.gz", ext.Version) + archiveName := fmt.Sprintf("libsodium-%s.tar.gz", ext.Version) + dest := fmt.Sprintf("/tmp/%s", archiveName) + srcDir := fmt.Sprintf("/tmp/libsodium-%s-RELEASE", ext.Version) + installPath := fmt.Sprintf("/tmp/libsodium-install-%s", ext.Version) + + if err := ec.Fetcher.Download(ctx, url, dest, source.Checksum{}); err != nil { + return fmt.Errorf("php/libsodium: download: %w", err) + } + if err := run.Run("tar", "xzf", dest, "-C", "/tmp/"); err != nil { + return fmt.Errorf("php/libsodium: extract: %w", err) + } + if err := run.RunInDir(srcDir, "sh", "./configure", fmt.Sprintf("--prefix=%s", installPath)); err != nil { + return fmt.Errorf("php/libsodium: configure: %w", err) + } + if err := run.RunInDir(srcDir, "make"); err != nil { + return fmt.Errorf("php/libsodium: make: %w", err) + } + if err := run.RunInDir(srcDir, "make", "install"); err != nil { + return fmt.Errorf("php/libsodium: make install: %w", err) + } + return nil +} diff --git a/internal/php/recipe.go b/internal/php/recipe.go new file mode 100644 index 00000000..cf991110 --- /dev/null +++ b/internal/php/recipe.go @@ -0,0 +1,83 @@ +package php + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/runner" +) + +// ExtensionContext holds the runtime paths resolved during the PHP build +// and passed to each extension recipe. +type ExtensionContext struct { + PHPPath string // e.g. /app/vendor/php-8.3.x + PHPSourceDir string // e.g. /tmp/php-8.3.x (unpacked PHP source, used by FakePecl) + HiredisPath string // set after hiredis native module builds + LibSodiumPath string // set after libsodium native module builds + LuaPath string // set after lua native module builds + RabbitMQPath string // set after rabbitmq native module builds + IonCubePath string // set after ioncube download + PHPMajor string // e.g. "8" + PHPMinor string // e.g. "3" + Fetcher fetch.Fetcher +} + +// ExtensionRecipe is the interface implemented by every PHP extension builder. +type ExtensionRecipe interface { + // Build performs the full build cycle for the given extension. + Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error +} + +// RecipeFor returns the ExtensionRecipe implementation for the given klass name. +// Returns an error for unknown klass names. +func RecipeFor(klass string) (ExtensionRecipe, error) { + switch klass { + case "PeclRecipe": + return &PeclRecipe{}, nil + case "FakePeclRecipe": + return &FakePeclRecipe{}, nil + case "AmqpPeclRecipe": + return &AmqpPeclRecipe{}, nil + case "MaxMindRecipe": + return &MaxMindRecipe{}, nil + case "HiredisRecipe": + return &HiredisRecipe{}, nil + case "LibSodiumRecipe": + return &LibSodiumRecipe{}, nil + case "IonCubeRecipe": + return &IonCubeRecipe{}, nil + case "LuaRecipe": + return &LuaRecipe{}, nil + case "MemcachedPeclRecipe": + return &MemcachedPeclRecipe{}, nil + case "OdbcRecipe": + return &OdbcRecipe{}, nil + case "PdoOdbcRecipe": + return &PdoOdbcRecipe{}, nil + case "SodiumRecipe": + return &SodiumRecipe{}, nil + case "OraclePeclRecipe": + return &OraclePeclRecipe{}, nil + case "OraclePdoRecipe": + return &OraclePdoRecipe{}, nil + case "PHPIRedisRecipe": + return &PHPIRedisRecipe{}, nil + case "RabbitMQRecipe": + return &RabbitMQRecipe{}, nil + case "RedisPeclRecipe": + return &RedisPeclRecipe{}, nil + case "SnmpRecipe": + return &SnmpRecipe{}, nil + case "TidewaysXhprofRecipe": + return &TidewaysXhprofRecipe{}, nil + case "LibRdKafkaRecipe": + return &LibRdKafkaRecipe{}, nil + case "Gd74FakePeclRecipe": + return &Gd74FakePeclRecipe{}, nil + case "EnchantFakePeclRecipe": + return &EnchantFakePeclRecipe{}, nil + default: + return nil, fmt.Errorf("php: unknown extension klass %q", klass) + } +} diff --git a/internal/php/recipe_test.go b/internal/php/recipe_test.go new file mode 100644 index 00000000..1a1bf2d2 --- /dev/null +++ b/internal/php/recipe_test.go @@ -0,0 +1,66 @@ +package php_test + +import ( + "testing" + + "github.com/cloudfoundry/binary-builder/internal/php" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRecipeFor_AllKnownKlasses(t *testing.T) { + knownKlasses := []string{ + "PeclRecipe", + "FakePeclRecipe", + "AmqpPeclRecipe", + "MaxMindRecipe", + "HiredisRecipe", + "LibSodiumRecipe", + "IonCubeRecipe", + "LuaRecipe", + "MemcachedPeclRecipe", + "OdbcRecipe", + "PdoOdbcRecipe", + "SodiumRecipe", + "OraclePeclRecipe", + "OraclePdoRecipe", + "PHPIRedisRecipe", + "RabbitMQRecipe", + "RedisPeclRecipe", + "SnmpRecipe", + "TidewaysXhprofRecipe", + "LibRdKafkaRecipe", + "Gd74FakePeclRecipe", + "EnchantFakePeclRecipe", + } + + for _, klass := range knownKlasses { + t.Run(klass, func(t *testing.T) { + recipe, err := php.RecipeFor(klass) + require.NoError(t, err, "RecipeFor(%q) should not error", klass) + assert.NotNil(t, recipe, "RecipeFor(%q) should return non-nil recipe", klass) + }) + } +} + +func TestRecipeFor_UnknownKlass(t *testing.T) { + _, err := php.RecipeFor("NonExistentRecipe") + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown extension klass") + assert.Contains(t, err.Error(), "NonExistentRecipe") +} + +func TestLoad_AllKlassesInBaseFileAreKnown(t *testing.T) { + // Verify every klass in the real base extensions file maps to a known recipe. + set, err := php.Load("8", "4") + require.NoError(t, err) + + all := append(set.NativeModules, set.Extensions...) + for _, ext := range all { + if ext.Klass == "" { + continue + } + _, err := php.RecipeFor(ext.Klass) + assert.NoError(t, err, "klass %q from base extensions should be known", ext.Klass) + } +} diff --git a/internal/php/special.go b/internal/php/special.go new file mode 100644 index 00000000..2f6e303c --- /dev/null +++ b/internal/php/special.go @@ -0,0 +1,98 @@ +package php + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" +) + +// IonCubeRecipe downloads a pre-built ioncube loader binary (no compile step). +// The loader is placed at {ioncubePath}/ioncube/ioncube_loader_lin_{major}.so +// and later copied to {ztsPath}/ioncube.so by PHPRecipe.setup_tar. +type IonCubeRecipe struct{} + +func (i *IonCubeRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("http://downloads3.ioncube.com/loader_downloads/ioncube_loaders_lin_x86-64_%s.tar.gz", ext.Version) + archiveName := fmt.Sprintf("ioncube-%s.tar.gz", ext.Version) + installPath := fmt.Sprintf("/tmp/ioncube-%s", ext.Version) + + // IonCube provides no checksum — skip verification (checksum.Value == ""). + if err := ec.Fetcher.Download(ctx, url, fmt.Sprintf("/tmp/%s", archiveName), source.Checksum{}); err != nil { + return fmt.Errorf("php/ioncube: download: %w", err) + } + if err := run.Run("mkdir", "-p", installPath); err != nil { + return fmt.Errorf("php/ioncube: mkdir: %w", err) + } + if err := run.Run("tar", "xzf", fmt.Sprintf("/tmp/%s", archiveName), "-C", installPath); err != nil { + return fmt.Errorf("php/ioncube: extract: %w", err) + } + return nil +} + +// OraclePeclRecipe builds the oci8 PECL extension against Oracle Instant Client. +// Requires /oracle to be mounted with the Oracle SDK. +type OraclePeclRecipe struct{} + +func (o *OraclePeclRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + url := fmt.Sprintf("http://pecl.php.net/get/%s-%s.tgz", ext.Name, ext.Version) + opts := []string{ + fmt.Sprintf("--with-php-config=%s/bin/php-config", ec.PHPPath), + "--with-oci8=shared,instantclient,/oracle", + } + checksum := source.Checksum{Algorithm: "md5", Value: ext.MD5} + if err := buildPecl(ctx, ext.Name, ext.Version, url, checksum, ec, opts, run); err != nil { + return err + } + // Copy Oracle libs into PHP prefix. + return run.Run("sh", "-c", fmt.Sprintf(` +cp -an /oracle/libclntshcore.so.12.1 %s/lib +cp -an /oracle/libclntsh.so %s/lib +cp -an /oracle/libclntsh.so.12.1 %s/lib +cp -an /oracle/libipc1.so %s/lib +cp -an /oracle/libmql1.so %s/lib +cp -an /oracle/libnnz12.so %s/lib +cp -an /oracle/libociicus.so %s/lib +cp -an /oracle/libons.so %s/lib +`, ec.PHPPath, ec.PHPPath, ec.PHPPath, ec.PHPPath, ec.PHPPath, ec.PHPPath, ec.PHPPath, ec.PHPPath)) +} + +// OraclePdoRecipe builds the pdo_oci extension (FakePecl) against Oracle Instant Client. +// Detects the Oracle version from /oracle/libclntsh.so.*. +type OraclePdoRecipe struct{} + +func (o *OraclePdoRecipe) Build(ctx context.Context, ext Extension, ec ExtensionContext, run runner.Runner) error { + // Detect oracle version from the libclntsh.so.{version} symlink. + oracleVersion, err := detectOracleVersion(run) + if err != nil { + return fmt.Errorf("php/pdo_oci: detect oracle version: %w", err) + } + + opts := []string{ + fmt.Sprintf("--with-pdo-oci=shared,instantclient,/oracle,%s", oracleVersion), + } + if err := buildFakePecl(ctx, "pdo_oci", ec, opts, run); err != nil { + return err + } + // Copy Oracle libs into PHP prefix. + return run.Run("sh", "-c", fmt.Sprintf(` +cp -an /oracle/libclntshcore.so.12.1 %s/lib +cp -an /oracle/libclntsh.so %s/lib +cp -an /oracle/libclntsh.so.12.1 %s/lib +cp -an /oracle/libipc1.so %s/lib +cp -an /oracle/libmql1.so %s/lib +cp -an /oracle/libnnz12.so %s/lib +cp -an /oracle/libociicus.so %s/lib +cp -an /oracle/libons.so %s/lib +`, ec.PHPPath, ec.PHPPath, ec.PHPPath, ec.PHPPath, ec.PHPPath, ec.PHPPath, ec.PHPPath, ec.PHPPath)) +} + +// detectOracleVersion returns the version suffix from the first libclntsh.so.{version} file found. +func detectOracleVersion(run runner.Runner) (string, error) { + out, err := run.Output("sh", "-c", `ls /oracle/libclntsh.so.* 2>/dev/null | head -1 | sed 's|.*libclntsh\.so\.||'`) + if err != nil { + return "", fmt.Errorf("listing oracle libs: %w", err) + } + return out, nil +} diff --git a/internal/portile/portile.go b/internal/portile/portile.go new file mode 100644 index 00000000..a7c014f3 --- /dev/null +++ b/internal/portile/portile.go @@ -0,0 +1,174 @@ +// Package portile provides a Go equivalent of mini_portile2. +// It manages the download → extract → configure → compile → install +// lifecycle for autoconf-based software (./configure && make && make install). +package portile + +import ( + "context" + "fmt" + "path/filepath" + "runtime" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" +) + +// Portile manages the build lifecycle for an autoconf-based dependency. +type Portile struct { + Name string + Version string + URL string + + // Checksum for verifying the downloaded source. + Checksum source.Checksum + + // Prefix is the --prefix= value for ./configure. + // Defaults to TmpPath()/port if empty. + Prefix string + + // Options are extra flags passed to ./configure after --prefix. + Options []string + + // Jobs is the -j flag for make. Defaults to 4. + Jobs int + + // ExtractedDirName overrides the assumed "{name}-{version}" directory name + // that the tarball extracts to. Use this when the tarball extracts to a + // differently-named directory (e.g. node tarballs extract to "node-v{version}" + // rather than "node-{version}"). + // When empty, defaults to "{name}-{version}". + ExtractedDirName string + + // InstallArgs are extra arguments appended to "make install". + // Used for recipes that need DESTDIR=... or PORTABLE=1 etc. + InstallArgs []string + + Runner runner.Runner + Fetcher fetch.Fetcher +} + +// TmpPath returns the temporary build directory: +// /tmp/{arch}/ports/{name}/{version} +func (p *Portile) TmpPath() string { + return filepath.Join("/tmp", runtime.GOARCH, "ports", p.Name, p.Version) +} + +// PortPath returns the extracted source directory inside TmpPath. +func (p *Portile) PortPath() string { + return filepath.Join(p.TmpPath(), "port") +} + +func (p *Portile) prefix() string { + if p.Prefix != "" { + return p.Prefix + } + return p.PortPath() +} + +func (p *Portile) jobs() int { + if p.Jobs > 0 { + return p.Jobs + } + return 4 +} + +// Cook performs the full build lifecycle: download, extract, configure, compile, install. +func (p *Portile) Cook(ctx context.Context) error { + if err := p.download(ctx); err != nil { + return fmt.Errorf("portile %s %s download: %w", p.Name, p.Version, err) + } + + if err := p.extract(ctx); err != nil { + return fmt.Errorf("portile %s %s extract: %w", p.Name, p.Version, err) + } + + if err := p.configure(ctx); err != nil { + return fmt.Errorf("portile %s %s configure: %w", p.Name, p.Version, err) + } + + if err := p.compile(ctx); err != nil { + return fmt.Errorf("portile %s %s compile: %w", p.Name, p.Version, err) + } + + if err := p.install(ctx); err != nil { + return fmt.Errorf("portile %s %s install: %w", p.Name, p.Version, err) + } + + return nil +} + +func (p *Portile) tarballPath() string { + // Extract filename from URL. + parts := strings.Split(p.URL, "/") + filename := parts[len(parts)-1] + // Strip query parameters. + if idx := strings.Index(filename, "?"); idx >= 0 { + filename = filename[:idx] + } + return filepath.Join(p.TmpPath(), filename) +} + +func (p *Portile) download(ctx context.Context) error { + // Create the tmp directory. + if err := p.Runner.Run("mkdir", "-p", p.TmpPath()); err != nil { + return err + } + + return p.Fetcher.Download(ctx, p.URL, p.tarballPath(), p.Checksum) +} + +func (p *Portile) extractedDirName() string { + if p.ExtractedDirName != "" { + return p.ExtractedDirName + } + return fmt.Sprintf("%s-%s", p.Name, p.Version) +} + +// ExtractFlag returns an explicit tar compression flag derived from the +// filename extension. This avoids relying on tar's auto-detect heuristic, +// which misidentifies .tar.gz files as zstd-compressed in some environments +// (e.g. cflinuxfs4) where zstd is not installed. +func ExtractFlag(filename string) string { + lower := strings.ToLower(filename) + switch { + case strings.HasSuffix(lower, ".tar.gz"), strings.HasSuffix(lower, ".tgz"): + return "xzf" + case strings.HasSuffix(lower, ".tar.bz2"): + return "xjf" + case strings.HasSuffix(lower, ".tar.xz"): + return "xJf" + default: + return "xf" + } +} + +func (p *Portile) extract(_ context.Context) error { + srcDir := filepath.Join(p.TmpPath(), p.extractedDirName()) + + flag := ExtractFlag(p.tarballPath()) + if err := p.Runner.Run("tar", flag, p.tarballPath(), "-C", p.TmpPath()); err != nil { + return err + } + + // mini_portile2 renames the extracted directory to "port". + return p.Runner.Run("mv", srcDir, p.PortPath()) +} + +func (p *Portile) configure(_ context.Context) error { + args := []string{fmt.Sprintf("--prefix=%s", p.prefix())} + args = append(args, p.Options...) + + return p.Runner.RunInDir(p.PortPath(), "./configure", args...) +} + +func (p *Portile) compile(_ context.Context) error { + return p.Runner.RunInDir(p.PortPath(), "make", fmt.Sprintf("-j%d", p.jobs())) +} + +func (p *Portile) install(_ context.Context) error { + args := []string{"install"} + args = append(args, p.InstallArgs...) + return p.Runner.RunInDir(p.PortPath(), "make", args...) +} diff --git a/internal/portile/portile_test.go b/internal/portile/portile_test.go new file mode 100644 index 00000000..3bed0c3b --- /dev/null +++ b/internal/portile/portile_test.go @@ -0,0 +1,247 @@ +package portile_test + +import ( + "context" + "fmt" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/portile" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeFetcher records download calls without performing them. +type fakeFetcher struct { + Downloads []downloadCall +} + +type downloadCall struct { + URL string + Dest string + Checksum source.Checksum +} + +func (f *fakeFetcher) Download(_ context.Context, url, dest string, checksum source.Checksum) error { + f.Downloads = append(f.Downloads, downloadCall{URL: url, Dest: dest, Checksum: checksum}) + return nil +} + +func (f *fakeFetcher) ReadBody(_ context.Context, url string) ([]byte, error) { + return nil, fmt.Errorf("not implemented") +} + +func TestTmpPath(t *testing.T) { + p := &portile.Portile{ + Name: "ruby", + Version: "3.3.6", + } + + path := p.TmpPath() + assert.Contains(t, path, "/tmp/") + assert.Contains(t, path, "ports/ruby/3.3.6") +} + +func TestPortPath(t *testing.T) { + p := &portile.Portile{ + Name: "ruby", + Version: "3.3.6", + } + + path := p.PortPath() + assert.Contains(t, path, "ports/ruby/3.3.6/port") +} + +func TestCookSequence(t *testing.T) { + f := runner.NewFakeRunner() + ff := &fakeFetcher{} + + p := &portile.Portile{ + Name: "ruby", + Version: "3.3.6", + URL: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.6.tar.gz", + Checksum: source.Checksum{ + Algorithm: "sha256", + Value: "abc123", + }, + Prefix: "/usr/local", + Options: []string{"--enable-shared", "--disable-install-doc"}, + Runner: f, + Fetcher: ff, + } + + err := p.Cook(context.Background()) + require.NoError(t, err) + + // Verify download was called. + require.Len(t, ff.Downloads, 1) + assert.Equal(t, "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.6.tar.gz", ff.Downloads[0].URL) + assert.Equal(t, "sha256", ff.Downloads[0].Checksum.Algorithm) + + // Verify the sequence of runner calls: + // 1. mkdir -p (create tmp dir) + // 2. tar xf (extract) + // 3. mv (rename to port) + // 4. ./configure (in port dir) + // 5. make -j4 (in port dir) + // 6. make install (in port dir) + require.Len(t, f.Calls, 6) + + assert.Equal(t, "mkdir", f.Calls[0].Name) + assert.Equal(t, "-p", f.Calls[0].Args[0]) + + assert.Equal(t, "tar", f.Calls[1].Name) + assert.Equal(t, "xzf", f.Calls[1].Args[0]) // .tar.gz → explicit gzip flag + + assert.Equal(t, "mv", f.Calls[2].Name) + + assert.Equal(t, "./configure", f.Calls[3].Name) + assert.Contains(t, f.Calls[3].Args, "--prefix=/usr/local") + assert.Contains(t, f.Calls[3].Args, "--enable-shared") + assert.Contains(t, f.Calls[3].Args, "--disable-install-doc") + assert.NotEmpty(t, f.Calls[3].Dir, "configure should run in port dir") + + assert.Equal(t, "make", f.Calls[4].Name) + assert.Equal(t, "-j4", f.Calls[4].Args[0]) + assert.NotEmpty(t, f.Calls[4].Dir) + + assert.Equal(t, "make", f.Calls[5].Name) + assert.Equal(t, "install", f.Calls[5].Args[0]) + assert.NotEmpty(t, f.Calls[5].Dir) +} + +func TestCookWithExtraOptions(t *testing.T) { + f := runner.NewFakeRunner() + ff := &fakeFetcher{} + + p := &portile.Portile{ + Name: "nginx", + Version: "1.25.3", + URL: "https://nginx.org/download/nginx-1.25.3.tar.gz", + Options: []string{"--with-http_ssl_module", "--with-http_v2_module"}, + Runner: f, + Fetcher: ff, + } + + err := p.Cook(context.Background()) + require.NoError(t, err) + + // Find the configure call. + var configureCall runner.Call + for _, c := range f.Calls { + if c.Name == "./configure" { + configureCall = c + break + } + } + + assert.Contains(t, configureCall.Args, "--with-http_ssl_module") + assert.Contains(t, configureCall.Args, "--with-http_v2_module") +} + +func TestCookFailureOnMake(t *testing.T) { + f := runner.NewFakeRunner() + ff := &fakeFetcher{} + + // Make the "make" command fail. + f.ErrorMap["make -j4"] = fmt.Errorf("compilation failed") + + p := &portile.Portile{ + Name: "ruby", + Version: "3.3.6", + URL: "https://example.com/ruby-3.3.6.tar.gz", + Runner: f, + Fetcher: ff, + } + + err := p.Cook(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "compile") + + // Verify install was NOT called after make failed. + for _, c := range f.Calls { + if c.Name == "make" && len(c.Args) > 0 && c.Args[0] == "install" { + t.Fatal("make install should not be called after make fails") + } + } +} + +func TestCookDefaultPrefix(t *testing.T) { + f := runner.NewFakeRunner() + ff := &fakeFetcher{} + + p := &portile.Portile{ + Name: "ruby", + Version: "3.3.6", + URL: "https://example.com/ruby-3.3.6.tar.gz", + Runner: f, + Fetcher: ff, + } + + err := p.Cook(context.Background()) + require.NoError(t, err) + + // When Prefix is empty, it defaults to PortPath(). + var configureCall runner.Call + for _, c := range f.Calls { + if c.Name == "./configure" { + configureCall = c + break + } + } + + assert.Contains(t, configureCall.Args[0], "--prefix=") + assert.Contains(t, configureCall.Args[0], "ports/ruby/3.3.6/port") +} + +func TestCookCustomJobs(t *testing.T) { + f := runner.NewFakeRunner() + ff := &fakeFetcher{} + + p := &portile.Portile{ + Name: "ruby", + Version: "3.3.6", + URL: "https://example.com/ruby-3.3.6.tar.gz", + Jobs: 2, + Runner: f, + Fetcher: ff, + } + + err := p.Cook(context.Background()) + require.NoError(t, err) + + var makeCall runner.Call + for _, c := range f.Calls { + if c.Name == "make" && len(c.Args) > 0 && c.Args[0] != "install" { + makeCall = c + break + } + } + + assert.Equal(t, "-j2", makeCall.Args[0]) +} + +func TestExtractFlag(t *testing.T) { + cases := []struct { + filename string + want string + }{ + {"ruby-3.3.6.tar.gz", "xzf"}, + {"R-4.4.2.tar.gz", "xzf"}, + {"something.tgz", "xzf"}, + {"archive.tar.bz2", "xjf"}, + {"archive.tar.xz", "xJf"}, + {"archive.tar.zst", "xf"}, // unknown: fall through to auto-detect + {"archive.tar", "xf"}, + // Case-insensitive. + {"ARCHIVE.TAR.GZ", "xzf"}, + {"ARCHIVE.TAR.BZ2", "xjf"}, + {"ARCHIVE.TAR.XZ", "xJf"}, + } + for _, tc := range cases { + t.Run(tc.filename, func(t *testing.T) { + assert.Equal(t, tc.want, portile.ExtractFlag(tc.filename)) + }) + } +} diff --git a/internal/recipe/bundle.go b/internal/recipe/bundle.go new file mode 100644 index 00000000..95c42a60 --- /dev/null +++ b/internal/recipe/bundle.go @@ -0,0 +1,91 @@ +package recipe + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// BundleRecipe downloads multiple pip packages and bundles them into a tarball. +// It covers pip and pipenv, which both follow the pattern: +// +// 1. setupPythonAndPip (fixed first step — both always need it) +// 2. mkdir -p tmpDir +// 3. pip3 download main package into tmpDir +// 4. [optional] ExtraSteps — e.g. pip's source-tarball strip + extract +// 5. pip3 download each ExtraDeps into tmpDir +// 6. tar zcvf OutputPath(version) from tmpDir +type BundleRecipe struct { + DepName string + Meta ArtifactMeta + Fetcher fetch.Fetcher + // MainPackage returns the pip package specifier for the main dep, e.g. "pip==24.0". + MainPackage func(version string) string + // DownloadArgs are extra args passed to the pip3 download command for the main package. + // e.g. ["--no-binary", ":all:"] or ["--no-cache-dir", "--no-binary", ":all:"] + DownloadArgs []string + // ExtraDeps is the list of additional packages to bundle (each gets its own pip3 download). + ExtraDeps []string + // ExtraSteps runs inside tmpDir after the main package download and before ExtraDeps. + // Used for pip's source-tarball-strip-and-extract step. + // May be nil. + ExtraSteps func(ctx context.Context, tmpDir string, src *source.Input, f fetch.Fetcher, r runner.Runner) error + // OutputPath returns the artifact path from version. + // Default: "/tmp/-.tgz" + OutputPath func(version string) string +} + +func (b *BundleRecipe) Name() string { return b.DepName } +func (b *BundleRecipe) Artifact() ArtifactMeta { return b.Meta } + +func (b *BundleRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, r runner.Runner, _ *output.OutData) error { + name := b.DepName + version := src.Version + + // Fixed first step: both pip and pipenv always need python + pip setup. + if err := setupPythonAndPip(ctx, s, r); err != nil { + return fmt.Errorf("%s: setup python: %w", name, err) + } + + outputPath := fmt.Sprintf("/tmp/%s-%s.tgz", name, version) + if b.OutputPath != nil { + outputPath = b.OutputPath(version) + } + + tmpDir := fmt.Sprintf("/tmp/%s-build-%s", name, version) + if err := r.Run("mkdir", "-p", tmpDir); err != nil { + return fmt.Errorf("%s: mkdir: %w", name, err) + } + + // Download the main package. + mainArgs := append(append([]string{"download"}, b.DownloadArgs...), b.MainPackage(version)) + if err := r.RunInDir(tmpDir, "/usr/bin/pip3", mainArgs...); err != nil { + return fmt.Errorf("%s: pip3 download %s: %w", name, b.MainPackage(version), err) + } + + // Run recipe-specific extra steps (e.g. pip's source tarball strip + extract). + if b.ExtraSteps != nil { + if err := b.ExtraSteps(ctx, tmpDir, src, b.Fetcher, r); err != nil { + return fmt.Errorf("%s: extra steps: %w", name, err) + } + } + + // Download each extra dependency. + for _, dep := range b.ExtraDeps { + if err := r.RunInDir(tmpDir, "/usr/bin/pip3", "download", "--no-binary", ":all:", dep); err != nil { + return fmt.Errorf("%s: pip3 download %s: %w", name, dep, err) + } + } + + // Bundle everything. + if err := r.RunInDir(tmpDir, "tar", "zcvf", outputPath, "."); err != nil { + return fmt.Errorf("%s: creating tarball: %w", name, err) + } + + return nil +} diff --git a/internal/recipe/bundler.go b/internal/recipe/bundler.go new file mode 100644 index 00000000..9e0ad5e5 --- /dev/null +++ b/internal/recipe/bundler.go @@ -0,0 +1,126 @@ +package recipe + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// BundlerRecipe builds bundler by bootstrapping a pre-built Ruby, then running +// `gem install bundler -v {version} --no-document --env-shebang`, tarring the +// result, and replacing shebangs — matching the Ruby builder's behavior exactly. +type BundlerRecipe struct { + Fetcher fetch.Fetcher +} + +func (b *BundlerRecipe) Name() string { return "bundler" } +func (b *BundlerRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: ""} +} + +func (b *BundlerRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, _ *output.OutData) error { + bootstrap := s.Bootstrap.Ruby + rubyTarball := filepath.Join("/tmp", filepath.Base(bootstrap.URL)) + + // Download the pre-built Ruby bootstrap binary with SHA256 verification. + if err := b.Fetcher.Download(ctx, bootstrap.URL, rubyTarball, source.Checksum{Algorithm: "sha256", Value: bootstrap.SHA256}); err != nil { + return fmt.Errorf("bundler: downloading ruby bootstrap: %w", err) + } + defer os.Remove(rubyTarball) + + // Extract Ruby bootstrap to its install dir. + if err := run.Run("mkdir", "-p", bootstrap.InstallDir); err != nil { + return err + } + if err := run.Run("tar", "xzf", rubyTarball, "-C", bootstrap.InstallDir, "--strip-components=1"); err != nil { + return fmt.Errorf("bundler: extracting ruby bootstrap: %w", err) + } + + // Use the full path to gem to avoid PATH resolution issues. + gemBin := filepath.Join(bootstrap.InstallDir, "bin", "gem") + + // Create a tmpdir to act as GEM_HOME/GEM_PATH — mirrors the Ruby recipe's + // `in_gem_env` block which sets these to a temp directory so the gem + // installs there without touching system gems. + gemHome, err := os.MkdirTemp("", "bundler-gemhome-*") + if err != nil { + return fmt.Errorf("bundler: creating gem tmpdir: %w", err) + } + defer os.RemoveAll(gemHome) + + // gem install bundler --version X --no-document --env-shebang + // GEM_HOME and GEM_PATH point to gemHome so all files land there. + if err := run.RunWithEnv( + map[string]string{ + "GEM_HOME": gemHome, + "GEM_PATH": gemHome, + "RUBYOPT": "", + }, + gemBin, "install", "bundler", "--version", src.Version, "--no-document", "--env-shebang", + ); err != nil { + return fmt.Errorf("bundler: gem install: %w", err) + } + + // Replace shebangs in bin/ scripts: #!/path/to/ruby → #!/usr/bin/env ruby + // Mirrors the Ruby recipe's replace_shebangs method. + // The bin/ dir may not exist when running under a fake runner in tests. + binDir := filepath.Join(gemHome, "bin") + if _, statErr := os.Stat(binDir); os.IsNotExist(statErr) { + // No bin/ dir — nothing to replace (e.g. fake runner in unit tests). + } else if err := replaceShebangs(binDir); err != nil { + return fmt.Errorf("bundler: replacing shebangs: %w", err) + } + + // Remove the .gem cache file (Ruby recipe does `rm -f bundler-X.gem` and + // `rm -rf cache/bundler-X.gem`). + os.Remove(filepath.Join(gemHome, fmt.Sprintf("bundler-%s.gem", src.Version))) + os.RemoveAll(filepath.Join(gemHome, "cache", fmt.Sprintf("bundler-%s.gem", src.Version))) + + // Tar gemHome contents into bundler-{version}.tgz in the CWD. + // The Ruby recipe does `tar czvf {current_dir}/{archive_filename} .` from gemHome. + archiveName := fmt.Sprintf("bundler-%s.tgz", src.Version) + if err := run.RunInDir(gemHome, "tar", "czf", filepath.Join(mustCwd(), archiveName), "."); err != nil { + return fmt.Errorf("bundler: creating archive: %w", err) + } + + return nil +} + +// replaceShebangs replaces Ruby interpreter shebangs in bin/ scripts with +// the portable `#!/usr/bin/env ruby` form. +func replaceShebangs(binDir string) error { + return filepath.WalkDir(binDir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + + data, err := os.ReadFile(path) + if err != nil { + return err + } + + content := string(data) + lines := strings.SplitN(content, "\n", 2) + if len(lines) == 0 || !strings.HasPrefix(lines[0], "#!") { + return nil + } + + shebang := lines[0] + // Only replace Ruby shebangs (lines containing "ruby"). + if !strings.Contains(shebang, "ruby") { + return nil + } + + newContent := strings.Replace(content, shebang, "#!/usr/bin/env ruby", 1) + return os.WriteFile(path, []byte(newContent), 0755) + }) +} diff --git a/internal/recipe/dep.go b/internal/recipe/dep.go new file mode 100644 index 00000000..8b3b8468 --- /dev/null +++ b/internal/recipe/dep.go @@ -0,0 +1,213 @@ +package recipe + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// GoToolRecipe implements the common pattern for building Go CLI tools (dep, glide, godep): +// +// 1. Download the source tarball to /tmp/-.tar.gz +// 2. mkdir -p +// 3. tar xzf to +// 4. Rename /-* → / +// 5. Run BuildCmd (sh -c "cd && GOPATH=... go get/build ...") +// 6. Move binary + license to /tmp +// 7. [optional] extra staging step (e.g. glide's mkdir /tmp/bin + copy) +// 8. tar czf artifact from PackFiles +type GoToolRecipe struct { + ToolName string // "dep", "glide", "godep" + OrgPath string // GitHub org path, e.g. "github.com/golang" + LicenseName string // "LICENSE" or "License" (godep uses capital-L, no-E) + // BuildCmd returns the shell command string executed via sh -c inside srcDir. + BuildCmd func(srcDir, version string) string + // PackFiles returns the paths to pack into the artifact tarball, relative to /tmp. + PackFiles func(name string) []string + // ExtraStaging runs after move and before packing; used by glide to stage /tmp/bin. + // May be nil. + ExtraStaging func(ctx context.Context, name string, run runner.Runner) error + Fetcher fetch.Fetcher +} + +func (g *GoToolRecipe) Name() string { return g.ToolName } +func (g *GoToolRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (g *GoToolRecipe) Build(ctx context.Context, _ *stack.Stack, src *source.Input, run runner.Runner, _ *output.OutData) error { + name := g.ToolName + version := src.Version + tmpPath := fmt.Sprintf("/tmp/src/%s", g.OrgPath) + srcDir := fmt.Sprintf("%s/%s", tmpPath, name) + srcTarball := fmt.Sprintf("/tmp/%s-%s.tar.gz", name, version) + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("%s-v%s-linux-x64.tgz", name, version)) + + if err := g.Fetcher.Download(ctx, src.URL, srcTarball, src.PrimaryChecksum()); err != nil { + return fmt.Errorf("%s: downloading source: %w", name, err) + } + if err := run.Run("mkdir", "-p", tmpPath); err != nil { + return fmt.Errorf("%s: mkdir: %w", name, err) + } + if err := run.Run("tar", "xzf", srcTarball, "-C", tmpPath); err != nil { + return fmt.Errorf("%s: extracting source: %w", name, err) + } + if err := run.Run("sh", "-c", + fmt.Sprintf("mv %s/%s-* %s", tmpPath, name, srcDir)); err != nil { + return fmt.Errorf("%s: renaming source dir: %w", name, err) + } + if err := run.Run("sh", "-c", g.BuildCmd(srcDir, version)); err != nil { + return fmt.Errorf("%s: build: %w", name, err) + } + + // Move binary to /tmp/. + if err := run.Run("mv", fmt.Sprintf("%s/%s", srcDir, name), fmt.Sprintf("/tmp/%s", name)); err != nil { + return fmt.Errorf("%s: moving binary: %w", name, err) + } + // Move license file to /tmp/. + if err := run.Run("mv", fmt.Sprintf("%s/%s", srcDir, g.LicenseName), fmt.Sprintf("/tmp/%s", g.LicenseName)); err != nil { + return fmt.Errorf("%s: moving license: %w", name, err) + } + + if g.ExtraStaging != nil { + if err := g.ExtraStaging(ctx, name, run); err != nil { + return fmt.Errorf("%s: extra staging: %w", name, err) + } + } + + packArgs := append([]string{"czf", artifactPath}, g.PackFiles(name)...) + if err := run.RunInDir("/tmp", "tar", packArgs...); err != nil { + return fmt.Errorf("%s: packing artifact: %w", name, err) + } + return nil +} + +// DepRecipe builds the `dep` Go dependency manager tool. +// +// Ruby layout (dep.rb): +// +// tmp_path = /tmp/src/github.com/golang +// source extracted to {tmp_path}/dep-VERSION/, renamed to {tmp_path}/dep/ +// GOPATH = {tmp_path}/dep/deps/_workspace:/tmp +// `go get -asmflags -trimpath ./...` run inside {tmp_path}/dep/ +// binary lands at /tmp/bin/dep (second GOPATH entry /tmp → /tmp/bin/) +// archive_files = ['/tmp/bin/dep', '/tmp/LICENSE'] +// archive_path_name = 'bin' → artifact contains bin/dep + bin/LICENSE +type DepRecipe struct { + Fetcher fetch.Fetcher +} + +func (d *DepRecipe) Name() string { return "dep" } +func (d *DepRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (d *DepRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, out *output.OutData) error { + return (&GoToolRecipe{ + ToolName: "dep", + OrgPath: "github.com/golang", + LicenseName: "LICENSE", + BuildCmd: func(srcDir, _ string) string { + gopath := fmt.Sprintf("%s/deps/_workspace:/tmp", srcDir) + return fmt.Sprintf("cd %s && GOPATH=%s /usr/local/go/bin/go get -asmflags -trimpath ./...", srcDir, gopath) + }, + // dep: binary lands at /tmp/bin/dep via GOPATH; pack directly from /tmp. + PackFiles: func(name string) []string { + return []string{"bin/dep", "bin/LICENSE"} + }, + Fetcher: d.Fetcher, + }).Build(ctx, s, src, run, out) +} + +// GlideRecipe builds the `glide` Go package manager tool. +// +// Ruby layout (glide.rb): +// +// tmp_path = /tmp/src/github.com/Masterminds +// source extracted to {tmp_path}/glide-VERSION/, renamed to {tmp_path}/glide/ +// GOPATH = /tmp +// `go build` run inside {tmp_path}/glide/ +// binary built at {tmp_path}/glide/glide, moved to /tmp/glide +// LICENSE moved to /tmp/LICENSE +// archive_files = ['/tmp/glide', '/tmp/LICENSE'] +// archive_path_name = 'bin' → artifact contains bin/glide + bin/LICENSE +type GlideRecipe struct { + Fetcher fetch.Fetcher +} + +func (g *GlideRecipe) Name() string { return "glide" } +func (g *GlideRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (g *GlideRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, out *output.OutData) error { + return (&GoToolRecipe{ + ToolName: "glide", + OrgPath: "github.com/Masterminds", + LicenseName: "LICENSE", + BuildCmd: func(srcDir, _ string) string { + return fmt.Sprintf("cd %s && GOPATH=/tmp /usr/local/go/bin/go build", srcDir) + }, + PackFiles: func(name string) []string { + return []string{"bin/glide", "bin/LICENSE"} + }, + // glide: binary built in srcDir, not GOPATH bin; must stage manually. + ExtraStaging: func(_ context.Context, name string, run runner.Runner) error { + if err := run.Run("mkdir", "-p", "/tmp/bin"); err != nil { + return err + } + if err := run.Run("cp", fmt.Sprintf("/tmp/%s", name), fmt.Sprintf("/tmp/bin/%s", name)); err != nil { + return err + } + if err := run.Run("cp", "/tmp/LICENSE", "/tmp/bin/LICENSE"); err != nil { + return err + } + return nil + }, + Fetcher: g.Fetcher, + }).Build(ctx, s, src, run, out) +} + +// GodepRecipe builds the `godep` Go vendoring tool. +// +// Ruby layout (godep.rb): +// +// tmp_path = /tmp/src/github.com/tools +// source extracted to {tmp_path}/godep-VERSION/, renamed to {tmp_path}/godep/ +// GOPATH = {tmp_path}/godep/Godeps/_workspace:/tmp +// `go get ./...` run inside {tmp_path}/godep/ +// binary lands at /tmp/bin/godep +// License (capital L, no E) moved to /tmp/License +// archive_files = ['/tmp/bin/godep', '/tmp/License'] +// archive_path_name = 'bin' → artifact contains bin/godep + bin/License +type GodepRecipe struct { + Fetcher fetch.Fetcher +} + +func (g *GodepRecipe) Name() string { return "godep" } +func (g *GodepRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (g *GodepRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, out *output.OutData) error { + return (&GoToolRecipe{ + ToolName: "godep", + OrgPath: "github.com/tools", + LicenseName: "License", + BuildCmd: func(srcDir, _ string) string { + gopath := fmt.Sprintf("%s/Godeps/_workspace:/tmp", srcDir) + return fmt.Sprintf("cd %s && GOPATH=%s /usr/local/go/bin/go get ./...", srcDir, gopath) + }, + // godep: binary lands at /tmp/bin/godep via GOPATH; pack directly from /tmp. + PackFiles: func(_ string) []string { + return []string{"bin/godep", "bin/License"} + }, + Fetcher: g.Fetcher, + }).Build(ctx, s, src, run, out) +} diff --git a/internal/recipe/dotnet.go b/internal/recipe/dotnet.go new file mode 100644 index 00000000..2b29b109 --- /dev/null +++ b/internal/recipe/dotnet.go @@ -0,0 +1,125 @@ +package recipe + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// DotnetSDKRecipe builds dotnet-sdk: download, prune ./shared/*, inject RuntimeVersion.txt, xz compress. +type DotnetSDKRecipe struct{} + +func (d *DotnetSDKRecipe) Name() string { return "dotnet-sdk" } +func (d *DotnetSDKRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (d *DotnetSDKRecipe) Build(ctx context.Context, _ *stack.Stack, src *source.Input, r runner.Runner, _ *output.OutData) error { + return pruneDotnetFiles(r, src, []string{"./shared/*"}, true) +} + +// DotnetRuntimeRecipe builds dotnet-runtime: download, prune ./dotnet, xz compress. +type DotnetRuntimeRecipe struct{} + +func (d *DotnetRuntimeRecipe) Name() string { return "dotnet-runtime" } +func (d *DotnetRuntimeRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (d *DotnetRuntimeRecipe) Build(ctx context.Context, _ *stack.Stack, src *source.Input, r runner.Runner, _ *output.OutData) error { + return pruneDotnetFiles(r, src, []string{"./dotnet"}, false) +} + +// DotnetAspnetcoreRecipe builds dotnet-aspnetcore: download, prune ./dotnet + ./shared/Microsoft.NETCore.App, xz compress. +type DotnetAspnetcoreRecipe struct{} + +func (d *DotnetAspnetcoreRecipe) Name() string { return "dotnet-aspnetcore" } +func (d *DotnetAspnetcoreRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (d *DotnetAspnetcoreRecipe) Build(ctx context.Context, _ *stack.Stack, src *source.Input, r runner.Runner, _ *output.OutData) error { + return pruneDotnetFiles(r, src, []string{"./dotnet", "./shared/Microsoft.NETCore.App"}, false) +} + +// pruneDotnetFiles extracts a dotnet tarball excluding specified paths, +// optionally writes RuntimeVersion.txt, and re-compresses with xz. +// +// The dotnet source tarball is pre-downloaded by Concourse into source/*.tar.gz. +// We use filepath.Glob to resolve the actual path before passing it to tar, +// since the runner does not invoke a shell (no glob expansion). +// +// The output artifact is written to the CWD using dash-separated naming +// (e.g. dotnet-runtime-8.0.21-linux-x64.tar.xz) so that findIntermediateArtifact +// can locate it via the standard glob patterns. +func pruneDotnetFiles(r runner.Runner, src *source.Input, excludes []string, writeRuntime bool) error { + adjustedFile := filepath.Join(mustCwd(), fmt.Sprintf("%s-%s-linux-x64.tar.xz", src.Name, src.Version)) + tmpDir := fmt.Sprintf("/tmp/dotnet-prune-%s-%s", src.Name, src.Version) + + if err := r.Run("mkdir", "-p", tmpDir); err != nil { + return err + } + + // Resolve source/*.tar.gz via glob — the runner does not use a shell so + // glob patterns are NOT expanded by the OS. + matches, err := filepath.Glob("source/*.tar.gz") + if err != nil || len(matches) == 0 { + return fmt.Errorf("dotnet: no source tarball found matching source/*.tar.gz") + } + sourceTarball := matches[0] + + // Build exclude args. + extractArgs := []string{"-xf", sourceTarball, "-C", tmpDir} + for _, exc := range excludes { + extractArgs = append(extractArgs, fmt.Sprintf("--exclude=%s", exc)) + } + + if err := r.Run("tar", extractArgs...); err != nil { + return fmt.Errorf("extracting dotnet: %w", err) + } + + if writeRuntime { + // Extract runtime version from the original archive. + // List entries under ./shared/Microsoft.NETCore.App/ and take the last directory, + // mirroring the Ruby recipe's write_runtime_version_file. + runtimeOutput, err := r.Output("tar", "tf", sourceTarball, "./shared/Microsoft.NETCore.App/") + if err != nil { + return fmt.Errorf("listing runtime version: %w", err) + } + + // Parse output: keep only directory entries (ending with '/'), take the last one. + lines := strings.Split(strings.TrimSpace(runtimeOutput), "\n") + var lastDir string + for _, line := range lines { + if strings.HasSuffix(line, "/") { + lastDir = line + } + } + if lastDir == "" { + return fmt.Errorf("dotnet: no directory found under ./shared/Microsoft.NETCore.App/") + } + runtimeVersion := filepath.Base(strings.TrimSuffix(lastDir, "/")) + + runtimeVersionFile := filepath.Join(tmpDir, "RuntimeVersion.txt") + if err := os.MkdirAll(tmpDir, 0755); err != nil { + return fmt.Errorf("mkdir tmpDir for RuntimeVersion.txt: %w", err) + } + if err := os.WriteFile(runtimeVersionFile, []byte(runtimeVersion), 0644); err != nil { + return fmt.Errorf("writing RuntimeVersion.txt: %w", err) + } + } + + // Re-compress with xz. + if err := r.RunInDir(tmpDir, "tar", "-Jcf", adjustedFile, "."); err != nil { + return fmt.Errorf("creating xz archive: %w", err) + } + + return nil +} diff --git a/internal/recipe/go_recipe.go b/internal/recipe/go_recipe.go new file mode 100644 index 00000000..554d5aad --- /dev/null +++ b/internal/recipe/go_recipe.go @@ -0,0 +1,100 @@ +package recipe + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// GoRecipe builds Go from source using a pre-downloaded bootstrap binary and make.bash. +// The source tarball extracts into a `go/` subdirectory; we pack that directory then +// strip the top-level `go/` prefix so the final artifact has `./`-prefixed paths, +// matching what Ruby's builder.rb build_go produces via strip_top_level_directory_from_tar. +type GoRecipe struct { + Fetcher fetch.Fetcher +} + +func (g *GoRecipe) Name() string { return "go" } +func (g *GoRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (g *GoRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, _ *output.OutData) error { + // Strip the `go` prefix from version (e.g. "go1.24.2" → "1.24.2"). + version := strings.TrimPrefix(src.Version, "go") + + srcTarball := fmt.Sprintf("/tmp/go%s.src.tar.gz", version) + bootstrapDir := fmt.Sprintf("/tmp/go-bootstrap-%s", version) + srcDir := fmt.Sprintf("/tmp/go-src-%s", version) + // Use a dash between name and version so findIntermediateArtifact can locate this file. + // Pattern: go-1.22.0.linux-amd64.tar.gz (matches glob "go-1.22.0*.tar.gz") + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("go-%s.linux-amd64.tar.gz", version)) + + // Download Go source tarball. + if err := g.Fetcher.Download(ctx, src.URL, srcTarball, src.PrimaryChecksum()); err != nil { + return fmt.Errorf("go: downloading source: %w", err) + } + + // Extract source. The Go source tarball extracts into a `go/` subdirectory. + // We do NOT use --strip-components so srcDir/go/ contains bin/, src/, etc. + if err := run.Run("mkdir", "-p", srcDir); err != nil { + return err + } + if err := run.Run("tar", "xzf", srcTarball, "-C", srcDir); err != nil { + return fmt.Errorf("go: extracting source: %w", err) + } + + // Download and extract the bootstrap Go binary. + // Match the Ruby go.rb recipe which uses go1.24.2 as bootstrap. + bootstrapTarball := "/tmp/go-bootstrap.tar.gz" + + if err := run.Run("mkdir", "-p", bootstrapDir); err != nil { + return err + } + bootstrapChecksum := source.Checksum{Algorithm: "sha256", Value: s.Bootstrap.Go.SHA256} + if err := g.Fetcher.Download(ctx, s.Bootstrap.Go.URL, bootstrapTarball, bootstrapChecksum); err != nil { + return fmt.Errorf("go: downloading bootstrap: %w", err) + } + if err := run.Run("tar", "xzf", bootstrapTarball, "-C", bootstrapDir, "--strip-components=1"); err != nil { + return fmt.Errorf("go: extracting bootstrap: %w", err) + } + + // Run make.bash to compile Go from source. + // make.bash must be run from within $GOROOT/src (it infers GOROOT from its own location). + srcGoSrc := fmt.Sprintf("%s/go/src", srcDir) + if err := run.RunInDirWithEnv( + srcGoSrc, + map[string]string{ + "GOROOT_BOOTSTRAP": bootstrapDir, + "GOROOT_FINAL": "/usr/local/go", + "GOTOOLCHAIN": "local", + }, + "bash", "./make.bash", + ); err != nil { + return fmt.Errorf("go: make.bash: %w", err) + } + + // Pack the compiled Go distribution. + // srcDir/go/ contains bin/, src/, pkg/, etc. — pack the `go` directory itself + // so the artifact has a top-level `go/` entry, matching the Ruby recipe layout. + if err := run.Run("tar", "czf", artifactPath, "-C", srcDir, "go"); err != nil { + return fmt.Errorf("go: packing artifact: %w", err) + } + + // Strip the top-level `go/` directory from the artifact, matching what Ruby's + // builder.rb build_go does via Archive.strip_top_level_directory_from_tar. + // This produces "./" prefixed paths (./bin/go, ./src/..., etc.) in the tarball. + if err := archive.StripTopLevelDir(artifactPath); err != nil { + return fmt.Errorf("go: stripping top-level dir: %w", err) + } + + return nil +} diff --git a/internal/recipe/helpers.go b/internal/recipe/helpers.go new file mode 100644 index 00000000..69652589 --- /dev/null +++ b/internal/recipe/helpers.go @@ -0,0 +1,61 @@ +package recipe + +import ( + "crypto/sha256" + "fmt" + "io" + "os" +) + +// computeSHA256 returns the hex-encoded SHA256 of the given data. +func computeSHA256(data []byte) string { + h := sha256.Sum256(data) + return fmt.Sprintf("%x", h) +} + +// SourceEntry is one entry in a sources.yml file. +type SourceEntry struct { + URL string + SHA256 string // hex SHA256 of the source tarball +} + +// fileSHA256 returns the hex-encoded SHA256 digest of the file at path. +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// mustCwd returns the current working directory, panicking on error. +func mustCwd() string { + cwd, err := os.Getwd() + if err != nil { + panic(fmt.Sprintf("recipe: getting cwd: %v", err)) + } + return cwd +} + +// buildSourcesYAML returns the content of a sources.yml file matching the +// format produced by Ruby's YAMLPresenter#to_yaml: +// +// --- +// - url: https://... +// sha256: abc123... +// +// The returned bytes are intended to be injected into an artifact tarball +// via archive.InjectFile, mirroring what ArchiveRecipe#compress! does by +// writing sources.yml into the tmpdir alongside the archive_files before tar. +func buildSourcesYAML(entries []SourceEntry) []byte { + content := "---\n" + for _, e := range entries { + content += fmt.Sprintf("- url: %s\n sha256: %s\n", e.URL, e.SHA256) + } + return []byte(content) +} diff --git a/internal/recipe/httpd.go b/internal/recipe/httpd.go new file mode 100644 index 00000000..1fa65995 --- /dev/null +++ b/internal/recipe/httpd.go @@ -0,0 +1,297 @@ +package recipe + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/portile" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// HTTPDRecipe builds Apache HTTPD along with its full dependency chain: +// APR → APR-Iconv → APR-Util → HTTPD → mod_auth_openidc. +// APR/APR-Iconv/APR-Util/mod_auth_openidc versions, URLs and SHA256s are +// pinned in the stack YAML under httpd_sub_deps. +type HTTPDRecipe struct { + Fetcher fetch.Fetcher +} + +func (h *HTTPDRecipe) Name() string { return "httpd" } +func (h *HTTPDRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (h *HTTPDRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, _ *output.OutData) error { + a := apt.New(run) + + // Step 1: apt install httpd build dependencies from stack config. + if err := a.Install(ctx, s.AptPackages["httpd_build"]...); err != nil { + return fmt.Errorf("httpd: apt install httpd_build: %w", err) + } + + // Step 2: Create /app directory. + if err := run.Run("mkdir", "-p", "/app"); err != nil { + return fmt.Errorf("httpd: mkdir /app: %w", err) + } + + // Step 3: Read sub-dep config from stack YAML. + aprDep := s.HTTPDSubDeps.APR + aprIconvDep := s.HTTPDSubDeps.APRIconv + aprUtilDep := s.HTTPDSubDeps.APRUtil + + // Step 4: Build APR. + aprPrefix := fmt.Sprintf("/tmp/apr-%s-prefix", aprDep.Version) + aprPortile := &portile.Portile{ + Name: "apr", + Version: aprDep.Version, + URL: aprDep.URL, + Checksum: source.Checksum{Algorithm: "sha256", Value: aprDep.SHA256}, + Prefix: aprPrefix, + Runner: run, + Fetcher: h.Fetcher, + } + if err := aprPortile.Cook(ctx); err != nil { + return fmt.Errorf("httpd: building APR: %w", err) + } + + // Step 5: Build APR-Iconv (depends on APR). + aprIconvPrefix := fmt.Sprintf("/tmp/apr-iconv-%s-prefix", aprIconvDep.Version) + aprIconvPortile := &portile.Portile{ + Name: "apr-iconv", + Version: aprIconvDep.Version, + URL: aprIconvDep.URL, + Checksum: source.Checksum{Algorithm: "sha256", Value: aprIconvDep.SHA256}, + Prefix: aprIconvPrefix, + Options: []string{ + fmt.Sprintf("--with-apr=%s/bin/apr-1-config", aprPrefix), + }, + Runner: run, + Fetcher: h.Fetcher, + } + if err := aprIconvPortile.Cook(ctx); err != nil { + return fmt.Errorf("httpd: building APR-Iconv: %w", err) + } + + // Step 6: Build APR-Util (depends on APR + APR-Iconv). + aprUtilPrefix := fmt.Sprintf("/tmp/apr-util-%s-prefix", aprUtilDep.Version) + aprUtilPortile := &portile.Portile{ + Name: "apr-util", + Version: aprUtilDep.Version, + URL: aprUtilDep.URL, + Checksum: source.Checksum{Algorithm: "sha256", Value: aprUtilDep.SHA256}, + Prefix: aprUtilPrefix, + Options: []string{ + fmt.Sprintf("--with-apr=%s", aprPrefix), + fmt.Sprintf("--with-iconv=%s", aprIconvPrefix), + "--with-crypto", + "--with-openssl", + "--with-mysql", + "--with-pgsql", + "--with-gdbm", + "--with-ldap", + }, + Runner: run, + Fetcher: h.Fetcher, + } + if err := aprUtilPortile.Cook(ctx); err != nil { + return fmt.Errorf("httpd: building APR-Util: %w", err) + } + + // Step 7: Build HTTPD (depends on APR + APR-Iconv + APR-Util). + httpdPrefix := "/app/httpd" + httpdPortile := &portile.Portile{ + Name: "httpd", + Version: src.Version, + URL: fmt.Sprintf("https://archive.apache.org/dist/httpd/httpd-%s.tar.bz2", src.Version), + Checksum: src.PrimaryChecksum(), + Prefix: httpdPrefix, + Options: []string{ + fmt.Sprintf("--with-apr=%s", aprPrefix), + fmt.Sprintf("--with-apr-util=%s", aprUtilPrefix), + "--with-ssl=/usr/lib/x86_64-linux-gnu", + "--enable-mpms-shared=worker event", + "--enable-mods-shared=reallyall", + "--disable-isapi", + "--disable-dav", + "--disable-dialup", + }, + Runner: run, + Fetcher: h.Fetcher, + } + if err := httpdPortile.Cook(ctx); err != nil { + return fmt.Errorf("httpd: building HTTPD: %w", err) + } + + // Step 7b: apt install mod_auth_openidc dependencies AFTER httpd is built. + // These must be installed after httpd so that jansson/cjose are NOT available + // during the HTTPD configure step — otherwise mod_md.so gets compiled (since + // mod_md depends on jansson), creating a file-list mismatch with the Ruby build. + if err := a.Install(ctx, s.AptPackages["httpd_mod_auth_build"]...); err != nil { + return fmt.Errorf("httpd: apt install httpd_mod_auth_build: %w", err) + } + + // Step 8: Build mod_auth_openidc (depends on HTTPD + APR). + // APR_LIBS and APR_CFLAGS are set in the environment for ./configure. + aprLibs := fmt.Sprintf("`%s/bin/apr-1-config --link-ld --libs`", aprPrefix) + aprCFlags := fmt.Sprintf("`%s/bin/apr-1-config --cflags --includes`", aprPrefix) + modAuthEnv := map[string]string{ + "APR_LIBS": aprLibs, + "APR_CFLAGS": aprCFlags, + } + modAuthDep := s.HTTPDSubDeps.ModAuthOpenidc + modAuthChecksum := source.Checksum{Algorithm: "sha256", Value: modAuthDep.SHA256} + modAuthPrefix := fmt.Sprintf("/tmp/mod_auth_openidc-%s-prefix", modAuthDep.Version) + + // We need to pass env vars to the configure step. The portile package runs + // configure via RunInDir without env, so we handle mod_auth_openidc manually: + // download, extract, configure with env, make, make install. + if err := h.buildModAuthOpenidc(ctx, run, modAuthDep.URL, modAuthDep.Version, modAuthPrefix, httpdPrefix, modAuthEnv, modAuthChecksum); err != nil { + return fmt.Errorf("httpd: building mod_auth_openidc: %w", err) + } + + // Step 9: setup_tar — copy shared libraries into the httpd prefix lib/ dir. + if err := h.setupTar(run, httpdPrefix, aprPrefix, aprUtilPrefix, aprIconvPrefix); err != nil { + return fmt.Errorf("httpd: setup_tar: %w", err) + } + + // Step 10: Pack the httpd prefix into the artifact tarball. + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("httpd-%s-linux-x64.tgz", src.Version)) + if err := run.Run("tar", "czf", artifactPath, "-C", "/app", "httpd"); err != nil { + return fmt.Errorf("httpd: packing artifact: %w", err) + } + + // Step 11: Strip top-level directory from the artifact. + if err := archive.StripTopLevelDir(artifactPath); err != nil { + return fmt.Errorf("httpd: stripping top-level dir: %w", err) + } + + return nil +} + +// buildModAuthOpenidc manually handles the configure/make/install cycle for +// mod_auth_openidc, passing APR_LIBS and APR_CFLAGS via RunWithEnv. +func (h *HTTPDRecipe) buildModAuthOpenidc( + ctx context.Context, + run runner.Runner, + url, version, prefix, httpdPrefix string, + env map[string]string, + checksum source.Checksum, +) error { + // Download tarball. + tarball := fmt.Sprintf("/tmp/mod_auth_openidc-%s.tar.gz", version) + if err := h.Fetcher.Download(ctx, url, tarball, checksum); err != nil { + return fmt.Errorf("download: %w", err) + } + + // Create temp dir and extract. + tmpDir := fmt.Sprintf("/tmp/mod_auth_openidc-%s-build", version) + if err := run.Run("mkdir", "-p", tmpDir); err != nil { + return fmt.Errorf("mkdir: %w", err) + } + if err := run.Run("tar", "xf", tarball, "-C", tmpDir); err != nil { + return fmt.Errorf("extract: %w", err) + } + + srcDir := fmt.Sprintf("%s/mod_auth_openidc-%s", tmpDir, version) + + // Configure with env (APR_LIBS + APR_CFLAGS), passing --with-apxs2. + configureCmd := fmt.Sprintf( + "./configure --prefix=%s --with-apxs2=%s/bin/apxs", + prefix, httpdPrefix, + ) + if err := run.RunWithEnv(env, "sh", "-c", + fmt.Sprintf("cd %s && %s", srcDir, configureCmd)); err != nil { + return fmt.Errorf("configure: %w", err) + } + + if err := run.RunInDir(srcDir, "make"); err != nil { + return fmt.Errorf("make: %w", err) + } + + if err := run.RunInDir(srcDir, "make", "install"); err != nil { + return fmt.Errorf("make install: %w", err) + } + + return nil +} + +// setupTar copies the runtime shared libraries into the httpd prefix lib/ dir, +// mirroring the Ruby httpd_meal.rb setup_tar method. +func (h *HTTPDRecipe) setupTar(run runner.Runner, httpdPrefix, aprPrefix, aprUtilPrefix, aprIconvPrefix string) error { + libDir := fmt.Sprintf("%s/lib", httpdPrefix) + aprUtilLibDir := fmt.Sprintf("%s/lib/apr-util-1", httpdPrefix) + iconvLibDir := fmt.Sprintf("%s/lib/iconv", httpdPrefix) + + // Remove unneeded directories. + for _, dir := range []string{"cgi-bin", "error", "icons", "include", "man", "manual", "htdocs"} { + if err := run.Run("rm", "-rf", fmt.Sprintf("%s/%s", httpdPrefix, dir)); err != nil { + return fmt.Errorf("rm %s: %w", dir, err) + } + } + + // Remove conf files but keep the conf/ directory. + if err := run.Run("sh", "-c", fmt.Sprintf( + "rm -rf %s/conf/extra/* %s/conf/httpd.conf %s/conf/httpd.conf.bak %s/conf/magic %s/conf/original", + httpdPrefix, httpdPrefix, httpdPrefix, httpdPrefix, httpdPrefix, + )); err != nil { + return fmt.Errorf("cleaning conf: %w", err) + } + + // Create lib subdirs. + for _, dir := range []string{libDir, aprUtilLibDir, iconvLibDir} { + if err := run.Run("mkdir", "-p", dir); err != nil { + return fmt.Errorf("mkdir %s: %w", dir, err) + } + } + + // Copy APR runtime library. + if err := run.Run("cp", fmt.Sprintf("%s/lib/libapr-1.so.0", aprPrefix), libDir); err != nil { + return fmt.Errorf("cp libapr: %w", err) + } + + // Copy APR-Util runtime library. + if err := run.Run("cp", fmt.Sprintf("%s/lib/libaprutil-1.so.0", aprUtilPrefix), libDir); err != nil { + return fmt.Errorf("cp libaprutil: %w", err) + } + + // Copy APR-Util plugins (apr-util-1/*.so). + if err := run.Run("sh", "-c", fmt.Sprintf( + "cp %s/lib/apr-util-1/*.so %s/", + aprUtilPrefix, aprUtilLibDir, + )); err != nil { + return fmt.Errorf("cp apr-util-1 plugins: %w", err) + } + + // Copy APR-Iconv library. + if err := run.Run("cp", fmt.Sprintf("%s/lib/libapriconv-1.so.0", aprIconvPrefix), libDir); err != nil { + return fmt.Errorf("cp libapriconv: %w", err) + } + + // Copy APR-Iconv converters (iconv/*.so). + if err := run.Run("sh", "-c", fmt.Sprintf( + "cp %s/lib/iconv/*.so %s/", + aprIconvPrefix, iconvLibDir, + )); err != nil { + return fmt.Errorf("cp iconv plugins: %w", err) + } + + // Copy system shared libraries (cjose, hiredis, jansson). + for _, pattern := range []string{ + "/usr/lib/x86_64-linux-gnu/libcjose.so*", + "/usr/lib/x86_64-linux-gnu/libhiredis.so*", + "/usr/lib/x86_64-linux-gnu/libjansson.so*", + } { + if err := run.Run("sh", "-c", fmt.Sprintf("cp %s %s/", pattern, libDir)); err != nil { + return fmt.Errorf("cp %s: %w", pattern, err) + } + } + + return nil +} diff --git a/internal/recipe/hwc.go b/internal/recipe/hwc.go new file mode 100644 index 00000000..4c9882fd --- /dev/null +++ b/internal/recipe/hwc.go @@ -0,0 +1,130 @@ +package recipe + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// HWCRecipe cross-compiles the Hostable Web Core (HWC) for Windows. +// The cross-compiler apt packages (e.g. mingw-w64) are read from +// s.AptPackages["hwc_build"] so they can be overridden per stack in +// stacks/*.yaml without modifying Go source. +// +// Ruby recipe (hwc.rb) builds two Windows binaries: +// - hwc-windows-amd64 (via release-binaries.bash amd64) → /tmp/hwc.exe +// - hwc-windows-386 (via release-binaries.bash 386) → /tmp/hwc_x86.exe +// +// Both are zipped into the artifact named hwc_{version}_windows_x86-64_any-stack.zip. +// ArchiveRecipe uses zip because archive_filename ends in .zip. +type HWCRecipe struct { + Fetcher fetch.Fetcher +} + +func (h *HWCRecipe) Name() string { return "hwc" } +func (h *HWCRecipe) Artifact() ArtifactMeta { + // Windows binary — arch is x86-64, stack is any-stack, extension is .zip. + return ArtifactMeta{OS: "windows", Arch: "x86-64", Stack: "any-stack"} +} + +func (h *HWCRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, _ *output.OutData) error { + version := src.Version + tmpPath := "/tmp/src/code.cloudfoundry.org" + srcDir := fmt.Sprintf("%s/hwc", tmpPath) + srcTarball := fmt.Sprintf("/tmp/hwc-%s.tar.gz", version) + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("hwc-%s-windows-x86-64.zip", version)) + + // Install cross-compiler packages (e.g. mingw-w64). The package list lives in + // stacks/*.yaml under apt_packages.hwc_build so it can be adjusted per stack + // without touching Go source. + if err := apt.New(run).Install(ctx, s.AptPackages["hwc_build"]...); err != nil { + return fmt.Errorf("hwc: installing hwc_build packages: %w", err) + } + + // Download source. + if err := h.Fetcher.Download(ctx, src.URL, srcTarball, src.PrimaryChecksum()); err != nil { + return fmt.Errorf("hwc: downloading source: %w", err) + } + + // Extract source. + if err := run.Run("mkdir", "-p", tmpPath); err != nil { + return err + } + if err := run.Run("tar", "xzf", srcTarball, "-C", tmpPath); err != nil { + return fmt.Errorf("hwc: extracting source: %w", err) + } + // Rename hwc-VERSION → hwc. + if err := run.Run("sh", "-c", + fmt.Sprintf("mv %s/hwc-* %s", tmpPath, srcDir)); err != nil { + return fmt.Errorf("hwc: renaming source dir: %w", err) + } + + // Cross-compile for Windows amd64 using mingw-w64. + // Must run from inside srcDir (where go.mod lives). + // Matches release-binaries.bash: CGO_ENABLED=1 GO_EXTLINK_ENABLED=1 CC=... GOARCH=... GOOS=windows go build -o OUTPUT -ldflags "-X main.version=VERSION" + hwcExePath := fmt.Sprintf("%s/hwc-windows-amd64", srcDir) + if err := run.RunInDirWithEnv(srcDir, + map[string]string{ + "GOOS": "windows", + "GOARCH": "amd64", + "CGO_ENABLED": "1", + "GO_EXTLINK_ENABLED": "1", + "CC": "x86_64-w64-mingw32-gcc", + "CXX": "x86_64-w64-mingw32-g++", + }, + "go", "build", "-o", hwcExePath, "-ldflags", fmt.Sprintf("-X main.version=%s", version), + ); err != nil { + return fmt.Errorf("hwc: go build amd64: %w", err) + } + + // Cross-compile for Windows 386 using mingw-w64. + hwcX86ExePath := fmt.Sprintf("%s/hwc-windows-386", srcDir) + if err := run.RunInDirWithEnv(srcDir, + map[string]string{ + "GOOS": "windows", + "GOARCH": "386", + "CGO_ENABLED": "1", + "GO_EXTLINK_ENABLED": "1", + "CC": "i686-w64-mingw32-gcc", + "CXX": "i686-w64-mingw32-g++", + }, + "go", "build", "-o", hwcX86ExePath, "-ldflags", fmt.Sprintf("-X main.version=%s", version), + ); err != nil { + return fmt.Errorf("hwc: go build 386: %w", err) + } + + // Move binaries to /tmp, renamed to hwc.exe and hwc_x86.exe. + if err := run.Run("mv", hwcExePath, "/tmp/hwc.exe"); err != nil { + return fmt.Errorf("hwc: moving amd64 binary: %w", err) + } + if err := run.Run("mv", hwcX86ExePath, "/tmp/hwc_x86.exe"); err != nil { + return fmt.Errorf("hwc: moving 386 binary: %w", err) + } + + // Write sources.yml to /tmp — matches Ruby's YAMLPresenter which writes + // sources.yml into the tmpdir before zipping (zip -r . includes it). + srcSHA256, err := fileSHA256(srcTarball) + if err != nil { + return fmt.Errorf("hwc: computing source sha256: %w", err) + } + sourcesContent := buildSourcesYAML([]SourceEntry{{URL: src.URL, SHA256: srcSHA256}}) + if err := os.WriteFile("/tmp/sources.yml", sourcesContent, 0644); err != nil { + return fmt.Errorf("hwc: writing sources.yml: %w", err) + } + + // Pack as zip. + // Ruby: zip archive.zip -r . from inside tmpdir containing hwc.exe, hwc_x86.exe, sources.yml. + if err := run.RunInDir("/tmp", "zip", artifactPath, "hwc.exe", "hwc_x86.exe", "sources.yml"); err != nil { + return fmt.Errorf("hwc: creating zip: %w", err) + } + + return nil +} diff --git a/internal/recipe/jruby.go b/internal/recipe/jruby.go new file mode 100644 index 00000000..44fd81b4 --- /dev/null +++ b/internal/recipe/jruby.go @@ -0,0 +1,220 @@ +package recipe + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// jdkSubdir returns the actual JDK directory that the tarball extracted into. +// The bellsoft/openjdk tarballs extract to a subdirectory like "jdk8u452/" +// inside the install dir, not directly into the install dir itself. +func jdkSubdir(jdkInstallDir string) (string, error) { + matches, err := filepath.Glob(filepath.Join(jdkInstallDir, "jdk*")) + if err != nil { + return "", fmt.Errorf("globbing JDK subdir: %w", err) + } + if len(matches) == 0 { + return "", fmt.Errorf("no jdk* subdirectory found in %s", jdkInstallDir) + } + return matches[0], nil +} + +// JRubyRecipe builds JRuby by: +// 1. Downloading the JDK from the stack-specific URL. +// 2. Building Maven from source (pinned version). +// 3. Compiling JRuby via Maven. +// 4. Stripping incorrect_words.yaml from the resulting jar files. +// +// The input version is the raw JRuby version (e.g. "9.4.5.0"). +// Internally the recipe computes the Ruby compatibility version and +// produces a full version of the form "9.4.5.0-ruby-3.1" which is +// used in the artifact filename. +type JRubyRecipe struct { + Fetcher fetch.Fetcher +} + +func (j *JRubyRecipe) Name() string { return "jruby" } +func (j *JRubyRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +// jrubyToRubyVersion maps a JRuby major.minor prefix to the Ruby compatibility version. +// This mapping is JRuby's own versioning scheme — not stack-specific. +var jrubyToRubyVersion = map[string]string{ + "9.3": "2.6", + "9.4": "3.1", +} + +// mavenVersion is the pinned Maven version used for the JRuby build. +// SHA512 matches the official Apache Maven 3.6.3 binary distribution. +const ( + mavenVersion = "3.6.3" + mavenSHA512 = "c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0" +) + +func (j *JRubyRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, outData *output.OutData) error { + jrubyVersion := src.Version + + // Step 1: Determine Ruby compatibility version from JRuby version prefix. + rubyVersion, err := jrubyRubyVersion(jrubyVersion) + if err != nil { + return fmt.Errorf("jruby: %w", err) + } + + // Full version used in artifact filename: e.g. "9.4.5.0-ruby-3.1" + fullVersion := fmt.Sprintf("%s-ruby-%s", jrubyVersion, rubyVersion) + + // Set ArtifactVersion so the artifact filename uses the full version + // (e.g. "9.4.14.0-ruby-3.1") rather than just the raw JRuby version. + // outData.Version (the raw "9.4.14.0") is preserved for dep-metadata and + // builds JSON, matching Ruby builder behaviour where out_data[:version] = + // source_input.version (never overwritten by build_jruby). + outData.ArtifactVersion = fullVersion + + // Step 2: Download and install JDK from stack-specific URL. + jdkDir := s.Bootstrap.JRuby.InstallDir + jdkTar := fmt.Sprintf("%s/openjdk-8-jdk.tar.gz", jdkDir) + + if err := run.Run("mkdir", "-p", jdkDir); err != nil { + return fmt.Errorf("jruby: creating JDK dir: %w", err) + } + + jdkChecksum := source.Checksum{Algorithm: "sha256", Value: s.Bootstrap.JRuby.SHA256} + if err := j.Fetcher.Download(ctx, s.Bootstrap.JRuby.URL, jdkTar, jdkChecksum); err != nil { + return fmt.Errorf("jruby: downloading JDK: %w", err) + } + + if err := run.Run("tar", "xvf", jdkTar, "-C", jdkDir); err != nil { + return fmt.Errorf("jruby: extracting JDK: %w", err) + } + + // The JDK tarball extracts into a subdir like "jdk8u452/" inside jdkDir. + // JAVA_HOME must point to that subdir, not to jdkDir itself. + javaHome, err := jdkSubdir(jdkDir) + if err != nil { + return fmt.Errorf("jruby: locating JDK subdir: %w", err) + } + + // Step 3: Download and set up Maven. + mavenURL := fmt.Sprintf("https://archive.apache.org/dist/maven/maven-3/%s/binaries/apache-maven-%s-bin.tar.gz", mavenVersion, mavenVersion) + mavenTar := fmt.Sprintf("/tmp/apache-maven-%s-bin.tar.gz", mavenVersion) + mavenChecksum := source.Checksum{Algorithm: "sha512", Value: mavenSHA512} + + if err := j.Fetcher.Download(ctx, mavenURL, mavenTar, mavenChecksum); err != nil { + return fmt.Errorf("jruby: downloading Maven: %w", err) + } + + mavenInstallDir := fmt.Sprintf("/tmp/apache-maven-%s", mavenVersion) + if err := run.Run("tar", "xf", mavenTar, "-C", "/tmp"); err != nil { + return fmt.Errorf("jruby: extracting Maven: %w", err) + } + + // Step 4: Download JRuby source zip. + // + // NOTE: The Ruby builder (builder.rb build_jruby) historically had a bug where it + // passed @source_input.sha256 into the sha512 slot of the inner SourceInput, and + // @source_input.git_commit_sha (always empty) into the sha256 slot. This caused the + // inner binary-builder.rb to be invoked with --sha256= (empty), failing verification. + // Fixed in builder.rb by passing nil for sha512 and @source_input.sha256 for sha256. + // + // The Go builder uses PrimaryChecksum() which correctly prefers sha512 > sha256, + // so data.json should supply both sha256 and sha512 fields. + jrubyURL := fmt.Sprintf("https://repo1.maven.org/maven2/org/jruby/jruby-dist/%s/jruby-dist-%s-src.zip", jrubyVersion, jrubyVersion) + jrubySrcZip := fmt.Sprintf("/tmp/jruby-dist-%s-src.zip", jrubyVersion) + jrubyChecksum := src.PrimaryChecksum() + + if err := j.Fetcher.Download(ctx, jrubyURL, jrubySrcZip, jrubyChecksum); err != nil { + return fmt.Errorf("jruby: downloading source: %w", err) + } + + // Extract source zip. + if err := run.Run("unzip", "-o", jrubySrcZip, "-d", "/tmp"); err != nil { + return fmt.Errorf("jruby: extracting source: %w", err) + } + + srcDir := fmt.Sprintf("/tmp/jruby-%s", jrubyVersion) + + // Step 5: Compile JRuby via Maven inside srcDir. + // We embed export statements directly in the shell command so that all + // child processes (including Maven's polyglot plugin which invokes bin/jruby + // as a subprocess) inherit JAVA_HOME, JAVACMD, and PATH. + // JAVACMD is set explicitly so bin/jruby does not need to discover java via + // PATH lookup — without it the subprocess fails with "Permission denied" + // (exit 126) because JAVACMD is empty and `exec ""` fails. + javacmd := fmt.Sprintf("%s/bin/java", javaHome) + mvnCmd := fmt.Sprintf( + "export JAVA_HOME=%s && export JAVACMD=%s && export PATH=%s/bin:%s/bin:/usr/bin:/bin && cd %s && mvn clean package -P '!truffle' -Djruby.default.ruby.version=%s", + javaHome, javacmd, javaHome, mavenInstallDir, srcDir, rubyVersion, + ) + if err := run.Run("sh", "-c", mvnCmd); err != nil { + return fmt.Errorf("jruby: mvn build: %w", err) + } + + // Step 6: Pack artifact — mirror Ruby's compress! exactly. + // compress! creates a tmpdir, cp -r's bin/ and lib/ into it, writes sources.yml + // into the tmpdir root, then runs: ls -A tmpdir | xargs tar czf archive -C tmpdir + // which produces ./bin/... ./lib/... ./sources.yml paths with the ./ prefix. + // We reproduce this by creating a tmpdir, copying bin/ and lib/, writing + // sources.yml, and tarring from there. + jrubySrcSHA256, err := fileSHA256(jrubySrcZip) + if err != nil { + return fmt.Errorf("jruby: computing source SHA256: %w", err) + } + sourcesContent := buildSourcesYAML([]SourceEntry{{URL: jrubyURL, SHA256: jrubySrcSHA256}}) + + packDir, err := os.MkdirTemp("", "jruby-pack-*") + if err != nil { + return fmt.Errorf("jruby: creating pack tmpdir: %w", err) + } + defer os.RemoveAll(packDir) + + if err := run.Run("cp", "-r", filepath.Join(srcDir, "bin"), filepath.Join(srcDir, "lib"), packDir); err != nil { + return fmt.Errorf("jruby: copying bin/lib to pack dir: %w", err) + } + if err := os.WriteFile(filepath.Join(packDir, "sources.yml"), sourcesContent, 0644); err != nil { + return fmt.Errorf("jruby: writing sources.yml: %w", err) + } + + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("jruby-%s-linux-x64.tgz", fullVersion)) + + // Use `tar czf ... -C packDir .` — the trailing `.` argument makes tar emit + // a root `./` entry and `./`-prefixed paths for all members, exactly matching + // what Ruby's Archive.strip_incorrect_words_yaml_from_tar re-archives with + // `tar -C dir -czf filename .`. + if err := run.Run("tar", "czf", artifactPath, "-C", packDir, "."); err != nil { + return fmt.Errorf("jruby: packing artifact: %w", err) + } + + // Step 7: Strip incorrect_words.yaml from jars in the artifact. + if err := archive.StripIncorrectWordsYAML(artifactPath); err != nil { + return fmt.Errorf("jruby: stripping incorrect_words.yaml: %w", err) + } + + return nil +} + +// jrubyRubyVersion maps a JRuby version string to the Ruby compatibility version. +func jrubyRubyVersion(jrubyVersion string) (string, error) { + // Extract major.minor prefix (first two parts). + parts := strings.Split(jrubyVersion, ".") + if len(parts) < 2 { + return "", fmt.Errorf("unexpected jruby version format %q", jrubyVersion) + } + prefix := strings.Join(parts[:2], ".") + + rubyVer, ok := jrubyToRubyVersion[prefix] + if !ok { + return "", fmt.Errorf("unknown JRuby version %q — cannot determine Ruby compatibility version (supported prefixes: 9.3, 9.4)", jrubyVersion) + } + return rubyVer, nil +} diff --git a/internal/recipe/libgdiplus.go b/internal/recipe/libgdiplus.go new file mode 100644 index 00000000..25bdf6c4 --- /dev/null +++ b/internal/recipe/libgdiplus.go @@ -0,0 +1,74 @@ +package recipe + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/autoconf" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// LibgdiplusRecipe builds libgdiplus from source via git clone + autogen + make. +type LibgdiplusRecipe struct{} + +func (l *LibgdiplusRecipe) Name() string { return "libgdiplus" } +func (l *LibgdiplusRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: ""} +} + +func (l *LibgdiplusRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, out *output.OutData) error { + return newLibgdiplusAutoconf().Build(ctx, s, src, run, out) +} + +// newLibgdiplusAutoconf constructs the AutoconfRecipe for libgdiplus. +func newLibgdiplusAutoconf() *autoconf.Recipe { + return &autoconf.Recipe{ + DepName: "libgdiplus", + // No Fetcher needed: SourceProvider uses git clone. + Hooks: autoconf.Hooks{ + AptPackages: func(s *stack.Stack) []string { + return s.AptPackages["libgdiplus_build"] + }, + + // SourceProvider clones the repository at the given version tag. + SourceProvider: func(ctx context.Context, src *source.Input, _ fetch.Fetcher, r runner.Runner) (string, error) { + version := src.Version + cloneDir := fmt.Sprintf("libgdiplus-%s", version) + repoURL := fmt.Sprintf("https://github.com/%s", src.Repo) + if err := r.Run("git", "clone", "--single-branch", "--branch", version, repoURL, cloneDir); err != nil { + return "", fmt.Errorf("git clone: %w", err) + } + // Return an absolute path so RunInDir works correctly from any CWD. + absCloneDir := filepath.Join(mustCwd(), cloneDir) + return absCloneDir, nil + }, + + // AfterExtract runs autogen.sh (which generates and runs ./configure for libgdiplus). + // AutoconfRecipe will also call ./configure after this hook; that second call is + // a harmless reconfigure since autogen.sh already generated the script. + AfterExtract: func(ctx context.Context, srcDir, prefix string, r runner.Runner) error { + buildEnv := map[string]string{ + "CFLAGS": "-g -Wno-maybe-uninitialized", + "CXXFLAGS": "-g -Wno-maybe-uninitialized", + } + return r.RunWithEnv(buildEnv, "sh", "-c", + fmt.Sprintf("cd %s && ./autogen.sh --prefix=%s", srcDir, prefix)) + }, + + // ConfigureEnv sets warning-suppression flags for configure, make, and make install. + ConfigureEnv: func() map[string]string { + return map[string]string{ + "CFLAGS": "-g -Wno-maybe-uninitialized", + "CXXFLAGS": "-g -Wno-maybe-uninitialized", + } + }, + + PackDirs: func() []string { return []string{"lib"} }, + }, + } +} diff --git a/internal/recipe/libunwind.go b/internal/recipe/libunwind.go new file mode 100644 index 00000000..71dde79f --- /dev/null +++ b/internal/recipe/libunwind.go @@ -0,0 +1,72 @@ +package recipe + +import ( + "context" + "fmt" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/autoconf" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// LibunwindRecipe builds libunwind from a pre-downloaded source tarball. +// The Concourse github-releases depwatcher has already placed the tarball in source/. +// Only the include/ and lib/ directories are packed into the artifact. +type LibunwindRecipe struct{} + +func (l *LibunwindRecipe) Name() string { return "libunwind" } +func (l *LibunwindRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: ""} +} + +func (l *LibunwindRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, out *output.OutData) error { + return newLibunwindAutoconf().Build(ctx, s, src, run, out) +} + +// newLibunwindAutoconf constructs the AutoconfRecipe for libunwind. +func newLibunwindAutoconf() *autoconf.Recipe { + return &autoconf.Recipe{ + DepName: "libunwind", + // No Fetcher: SourceProvider reads from source/ instead. + Hooks: autoconf.Hooks{ + AptPackages: func(s *stack.Stack) []string { + return s.AptPackages["libunwind_build"] + }, + + // SourceProvider reads the pre-downloaded tarball from source/ and + // extracts it to /tmp, returning the extracted directory path. + SourceProvider: func(ctx context.Context, src *source.Input, _ fetch.Fetcher, r runner.Runner) (string, error) { + parts := strings.Split(src.URL, "/") + filename := parts[len(parts)-1] + tag := strings.TrimSuffix(strings.TrimSuffix(filename, ".tar.gz"), ".tgz") + // Two URL styles: + // refs/tags/v1.6.2.tar.gz → tag="v1.6.2" → extracts to libunwind-1.6.2/ + // libunwind-1.6.2.tar.gz → tag="libunwind-1.6.2" → extracts to libunwind-1.6.2/ + var dirName string + if strings.HasPrefix(tag, "libunwind-") { + dirName = tag + } else { + dirName = "libunwind-" + strings.TrimPrefix(tag, "v") + } + + srcTarball := fmt.Sprintf("source/%s", filename) + if err := r.Run("tar", "xzf", srcTarball, "-C", "/tmp"); err != nil { + return "", fmt.Errorf("extracting source: %w", err) + } + return fmt.Sprintf("/tmp/%s", dirName), nil + }, + + // AfterExtract regenerates ./configure from configure.ac. + // GitHub source archives only contain autotools sources, not the generated script. + AfterExtract: func(ctx context.Context, srcDir, _ string, r runner.Runner) error { + return r.RunInDir(srcDir, "autoreconf", "-i") + }, + + PackDirs: func() []string { return []string{"include", "lib"} }, + }, + } +} diff --git a/internal/recipe/nginx.go b/internal/recipe/nginx.go new file mode 100644 index 00000000..2f4b59db --- /dev/null +++ b/internal/recipe/nginx.go @@ -0,0 +1,180 @@ +package recipe + +import ( + "context" + "fmt" + "slices" + + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/autoconf" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/gpg" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// nginxGPGKeys are the 6 nginx signing keys used for tarball verification. +var nginxGPGKeys = []string{ + "http://nginx.org/keys/maxim.key", + "http://nginx.org/keys/arut.key", + "https://nginx.org/keys/pluknet.key", + "http://nginx.org/keys/sb.key", + "http://nginx.org/keys/thresh.key", + "https://nginx.org/keys/nginx_signing.key", +} + +// nginxBaseConfigureArgs are the configure options shared between nginx and nginx-static. +// These match the Ruby build_nginx_helper base_nginx_options exactly. +var nginxBaseConfigureArgs = []string{ + "--prefix=/", + "--error-log-path=stderr", + "--with-http_ssl_module", + "--with-http_v2_module", + "--with-http_realip_module", + "--with-http_gunzip_module", + "--with-http_gzip_static_module", + "--with-http_auth_request_module", + "--with-http_random_index_module", + "--with-http_secure_link_module", + "--with-http_stub_status_module", + "--without-http_uwsgi_module", + "--without-http_scgi_module", + "--with-pcre", + "--with-pcre-jit", + "--with-debug", +} + +// NginxRecipe builds nginx with PIC flags and dynamic modules. +type NginxRecipe struct { + Fetcher fetch.Fetcher +} + +func (n *NginxRecipe) Name() string { return "nginx" } +func (n *NginxRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (n *NginxRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, out *output.OutData) error { + return newNginxAutoconf(n.Fetcher, false).Build(ctx, s, src, run, out) +} + +// NginxStaticRecipe builds nginx with PIE flags and a minimal module set. +type NginxStaticRecipe struct { + Fetcher fetch.Fetcher +} + +func (n *NginxStaticRecipe) Name() string { return "nginx-static" } +func (n *NginxStaticRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (n *NginxStaticRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, out *output.OutData) error { + return newNginxAutoconf(n.Fetcher, true).Build(ctx, s, src, run, out) +} + +// newNginxAutoconf constructs the AutoconfRecipe for nginx or nginx-static. +// +// isStatic=true → PIE flags, minimal modules (nginx-static) +// isStatic=false → PIC flags + dynamic modules (nginx) +// +// Install layout: --prefix=/ with DESTDIR=/nginx causes the install +// tree to land at /nginx/{sbin,modules,...}. +// AfterPack strips the top-level dir so the final artifact's root is nginx/. +func newNginxAutoconf(fetcher fetch.Fetcher, isStatic bool) *autoconf.Recipe { + depName := "nginx" + if isStatic { + depName = "nginx-static" + } + + return &autoconf.Recipe{ + DepName: depName, + Fetcher: fetcher, + Hooks: autoconf.Hooks{ + // No apt packages needed for nginx. + AptPackages: func(_ *stack.Stack) []string { return nil }, + + // BeforeDownload verifies the GPG signature of the source tarball. + BeforeDownload: func(ctx context.Context, src *source.Input, r runner.Runner) error { + sigURL := src.URL + ".asc" + return gpg.VerifySignature(ctx, src.URL, sigURL, nginxGPGKeys, r) + }, + + // SourceProvider downloads and extracts the nginx source tarball. + SourceProvider: func(ctx context.Context, src *source.Input, f fetch.Fetcher, r runner.Runner) (string, error) { + version := src.Version + srcTarball := fmt.Sprintf("/tmp/nginx-%s.tar.gz", version) + if err := f.Download(ctx, src.URL, srcTarball, src.PrimaryChecksum()); err != nil { + return "", fmt.Errorf("downloading source: %w", err) + } + if err := r.Run("tar", "xzf", srcTarball, "-C", "/tmp"); err != nil { + return "", fmt.Errorf("extracting source: %w", err) + } + return fmt.Sprintf("/tmp/nginx-%s", src.Version), nil + }, + + // AfterExtract creates the DESTDIR install directory before make install. + AfterExtract: func(_ context.Context, _, prefix string, r runner.Runner) error { + nginxInstallDir := fmt.Sprintf("%s/nginx", prefix) + return r.Run("mkdir", "-p", nginxInstallDir) + }, + + // ConfigureArgs returns the full nginx configure flags for this variant. + // Base args include --prefix=/ (Ruby parity); variant args follow. + ConfigureArgs: func(_, _ string) []string { + var variantArgs []string + if isStatic { + variantArgs = []string{ + "--with-cc-opt=-fPIE -pie", + "--with-ld-opt=-fPIE -pie -z now", + } + } else { + variantArgs = []string{ + "--with-cc-opt=-fPIC -pie", + "--with-ld-opt=-fPIC -pie -z now", + "--with-compat", + "--with-mail=dynamic", + "--with-mail_ssl_module", + "--with-stream=dynamic", + "--with-http_sub_module", + } + } + // Use slices.Concat to avoid appending to the package-level slice's backing array. + return slices.Concat(nginxBaseConfigureArgs, variantArgs) + }, + + // InstallEnv sets DESTDIR so --prefix=/ resolves relative to /nginx. + InstallEnv: func(prefix string) map[string]string { + return map[string]string{"DESTDIR": fmt.Sprintf("%s/nginx", prefix)} + }, + + // AfterInstall removes html/ and conf/ from the nginx install tree. + AfterInstall: func(_ context.Context, prefix string, r runner.Runner) error { + nginxInstallDir := fmt.Sprintf("%s/nginx", prefix) + return removeNginxRuntimeDirs(r, nginxInstallDir) + }, + + // AfterPack strips the top-level directory from the artifact so the + // archive root is nginx/ (matching the Ruby builder output). + AfterPack: func(artifactPath string) error { + return archive.StripTopLevelDir(artifactPath) + }, + }, + } +} + +// removeNginxRuntimeDirs removes html/ and conf/ from nginxDir, then recreates +// an empty conf/. Shared between nginx and openresty post-install cleanup. +func removeNginxRuntimeDirs(run runner.Runner, nginxDir string) error { + if err := run.Run("rm", "-rf", + fmt.Sprintf("%s/html", nginxDir), + fmt.Sprintf("%s/conf", nginxDir), + ); err != nil { + return fmt.Errorf("removing html/conf: %w", err) + } + if err := run.Run("mkdir", "-p", fmt.Sprintf("%s/conf", nginxDir)); err != nil { + return fmt.Errorf("recreating conf: %w", err) + } + return nil +} diff --git a/internal/recipe/nginx_static.go b/internal/recipe/nginx_static.go new file mode 100644 index 00000000..e0744f41 --- /dev/null +++ b/internal/recipe/nginx_static.go @@ -0,0 +1,5 @@ +package recipe + +// NginxStaticRecipe is declared in nginx.go alongside NginxRecipe and newNginxAutoconf. +// This file is intentionally empty; it exists only to preserve the filename in case +// tooling or git history references it. diff --git a/internal/recipe/node.go b/internal/recipe/node.go new file mode 100644 index 00000000..a8b0c464 --- /dev/null +++ b/internal/recipe/node.go @@ -0,0 +1,102 @@ +package recipe + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/compiler" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/portile" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// NodeRecipe builds Node.js, matching the Ruby NodeRecipe exactly: +// - configure with --prefix=/ --openssl-use-def-ca-store +// - install with DESTDIR=/tmp/node-v{version}-linux-x64 PORTABLE=1 +// - copy LICENSE from work_path into dest_dir +// - pack dest_dir as top-level dir, then strip top-level from the artifact +// +// The tarball node-v{version}.tar.gz extracts to node-v{version}/ (not node-{version}/), +// so ExtractedDirName is set to "node-v{version}" to match. +type NodeRecipe struct { + Fetcher fetch.Fetcher +} + +func (n *NodeRecipe) Name() string { return "node" } +func (n *NodeRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (n *NodeRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, _ *output.OutData) error { + // Step 1: Install python3/pip3 build tools (node configure requires python3). + if err := setupPythonAndPip(ctx, s, run); err != nil { + return fmt.Errorf("node: setup python/pip: %w", err) + } + + // Step 2: Set up GCC (stack-driven; cflinuxfs5 skips the PPA). + a := apt.New(run) + gcc := compiler.NewGCC(s.Compilers.GCC, a, run) + if err := gcc.Setup(ctx); err != nil { + return fmt.Errorf("node: GCC setup: %w", err) + } + + // Step 3: Strip `v` prefix from version (e.g. "v22.14.0" → "22.14.0"). + version := strings.TrimPrefix(src.Version, "v") + + // Ruby recipe's dest_dir: /tmp/node-v{version}-linux-x64 + destDir := fmt.Sprintf("/tmp/node-v%s-linux-x64", version) + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("node-%s-linux-x64.tgz", version)) + + // Step 4: Install optional node build packages from stack config. + if pkgs, ok := s.AptPackages["node_build"]; ok && len(pkgs) > 0 { + if err := a.Install(ctx, pkgs...); err != nil { + return fmt.Errorf("node: apt install node_build: %w", err) + } + } + + // Step 5: Build via portile. + // Ruby recipe: --prefix=/ --openssl-use-def-ca-store + // Install: make install DESTDIR={destDir} PORTABLE=1 + // Tarball node-v{version}.tar.gz extracts to node-v{version}/ directory. + p := &portile.Portile{ + Name: "node", + Version: version, + URL: src.URL, + Checksum: src.PrimaryChecksum(), + Prefix: "/", + Options: []string{"--openssl-use-def-ca-store"}, + ExtractedDirName: fmt.Sprintf("node-v%s", version), + InstallArgs: []string{fmt.Sprintf("DESTDIR=%s", destDir), "PORTABLE=1"}, + Runner: run, + Fetcher: n.Fetcher, + } + + if err := p.Cook(ctx); err != nil { + return fmt.Errorf("node: portile cook: %w", err) + } + + // Step 6: Copy LICENSE into dest_dir (mirrors Ruby recipe's setup_tar). + // The portile work_path is TmpPath()/port. + licenseSource := fmt.Sprintf("%s/port/LICENSE", p.TmpPath()) + if err := run.Run("cp", licenseSource, destDir); err != nil { + return fmt.Errorf("node: copying LICENSE: %w", err) + } + + // Step 7: Pack dest_dir as the top-level directory, then strip it. + // Ruby: cp -r destDir tmpdir/ → tar czf artifact -C tmpdir → strip_top_level_directory_from_tar + // We recreate this by packing destDir's parent with the dirname, then stripping. + destParent := filepath.Dir(destDir) + destBase := filepath.Base(destDir) + if err := run.Run("tar", "czf", artifactPath, "-C", destParent, destBase); err != nil { + return fmt.Errorf("node: packing artifact: %w", err) + } + + return archive.StripTopLevelDir(artifactPath) +} diff --git a/internal/recipe/openresty.go b/internal/recipe/openresty.go new file mode 100644 index 00000000..acb4b0d9 --- /dev/null +++ b/internal/recipe/openresty.go @@ -0,0 +1,93 @@ +package recipe + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/autoconf" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// OpenrestyRecipe builds OpenResty (nginx + Lua) via configure/make. +// No GPG verification (known gap, carried forward from Ruby code as a TODO). +type OpenrestyRecipe struct { + Fetcher fetch.Fetcher +} + +func (o *OpenrestyRecipe) Name() string { return "openresty" } +func (o *OpenrestyRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (o *OpenrestyRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, out *output.OutData) error { + return o.newAutoconf().Build(ctx, s, src, run, out) +} + +// newAutoconf constructs the AutoconfRecipe for openresty. +func (o *OpenrestyRecipe) newAutoconf() *autoconf.Recipe { + return &autoconf.Recipe{ + DepName: "openresty", + Fetcher: o.Fetcher, + Hooks: autoconf.Hooks{ + // No apt packages needed for openresty. + AptPackages: func(_ *stack.Stack) []string { return nil }, + + // SourceProvider downloads and extracts the openresty source tarball. + SourceProvider: func(ctx context.Context, src *source.Input, f fetch.Fetcher, r runner.Runner) (string, error) { + version := src.Version + srcTarball := fmt.Sprintf("/tmp/openresty-%s.tar.gz", version) + if err := f.Download(ctx, src.URL, srcTarball, src.PrimaryChecksum()); err != nil { + return "", fmt.Errorf("downloading source: %w", err) + } + if err := r.Run("tar", "xzf", srcTarball, "-C", "/tmp"); err != nil { + return "", fmt.Errorf("extracting source: %w", err) + } + return fmt.Sprintf("/tmp/openresty-%s", version), nil + }, + + // ConfigureArgs returns the full openresty configure flags. + // Flags match Ruby builder.rb build_openresty exactly. + ConfigureArgs: func(_, prefix string) []string { + return []string{ + fmt.Sprintf("--prefix=%s", prefix), + "-j2", + "--error-log-path=stderr", + "--with-http_ssl_module", + "--with-http_v2_module", + "--with-http_realip_module", + "--with-http_gunzip_module", + "--with-http_gzip_static_module", + "--with-http_auth_request_module", + "--with-http_random_index_module", + "--with-http_secure_link_module", + "--with-http_stub_status_module", + "--without-http_uwsgi_module", + "--without-http_scgi_module", + "--with-pcre", + "--with-pcre-jit", + "--with-cc-opt=-fPIC -pie", + "--with-ld-opt=-fPIC -pie -z now", + "--with-compat", + "--with-mail=dynamic", + "--with-mail_ssl_module", + "--with-stream=dynamic", + } + }, + + MakeArgs: func() []string { return []string{"-j2"} }, + + // AfterInstall removes the bin/openresty symlink and cleans nginx runtime dirs. + AfterInstall: func(ctx context.Context, prefix string, r runner.Runner) error { + if err := r.Run("rm", "-rf", fmt.Sprintf("%s/bin/openresty", prefix)); err != nil { + return fmt.Errorf("removing bin/openresty: %w", err) + } + nginxDir := fmt.Sprintf("%s/nginx", prefix) + return removeNginxRuntimeDirs(r, nginxDir) + }, + }, + } +} diff --git a/internal/recipe/passthrough.go b/internal/recipe/passthrough.go new file mode 100644 index 00000000..6459803d --- /dev/null +++ b/internal/recipe/passthrough.go @@ -0,0 +1,162 @@ +package recipe + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/fileutil" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// SourceFilenameFunc returns the expected source filename for a given version. +type SourceFilenameFunc func(version string) string + +// PassthroughRecipe handles dependencies that are downloaded and passed through +// without compilation. Covers: tomcat, composer, appdynamics, appdynamics-java, +// skywalking-agent, openjdk, zulu, sapmachine, jprofiler-profiler, your-kit-profiler. +type PassthroughRecipe struct { + DepName string + SourceFilenameFunc SourceFilenameFunc + Meta ArtifactMeta + Fetcher fetch.Fetcher +} + +func (p *PassthroughRecipe) Name() string { return p.DepName } + +func (p *PassthroughRecipe) Artifact() ArtifactMeta { return p.Meta } + +func (p *PassthroughRecipe) Build(ctx context.Context, _ *stack.Stack, src *source.Input, r runner.Runner, _ *output.OutData) error { + filename := p.SourceFilenameFunc(src.Version) + localPath := filepath.Join("source", filename) + + if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil { + return fmt.Errorf("creating source directory: %w", err) + } + + if _, err := os.Stat(localPath); os.IsNotExist(err) { + if err := p.Fetcher.Download(ctx, src.URL, localPath, src.PrimaryChecksum()); err != nil { + return fmt.Errorf("downloading %s: %w", p.DepName, err) + } + } + + // Move the downloaded file to the working directory with a version-named + // intermediate filename (e.g. composer-2.7.1.phar, tomcat-9.0.85.tar.gz) + // so that findIntermediateArtifact in main.go can locate it via glob + // patterns like "tomcat-9.0.85*.tar.gz". + ext := archiveExt(filename) + intermediateName := fmt.Sprintf("%s-%s%s", p.DepName, src.Version, ext) + if err := fileutil.MoveFile(localPath, intermediateName); err != nil { + return fmt.Errorf("staging artifact %s: %w", p.DepName, err) + } + + return nil +} + +// NewPassthroughRecipes creates all passthrough and repack-only recipe instances. +// This includes JVM passthrough deps (openjdk, zulu, …) as well as PyPI sdist +// deps (setuptools, flit-core, …) — anything that needs no compilation step. +// Add a new entry here whenever buildpacks-ci adds a dep with source_type: pypi +// or a similar "download-only" source type. +func NewPassthroughRecipes(f fetch.Fetcher) []Recipe { + return []Recipe{ + &PassthroughRecipe{ + DepName: "tomcat", + SourceFilenameFunc: func(v string) string { return fmt.Sprintf("apache-tomcat-%s.tar.gz", v) }, + Meta: ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"}, + Fetcher: f, + }, + &PassthroughRecipe{ + DepName: "composer", + SourceFilenameFunc: func(_ string) string { return "composer.phar" }, + Meta: ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"}, + Fetcher: f, + }, + &PassthroughRecipe{ + DepName: "appdynamics", + SourceFilenameFunc: func(v string) string { return fmt.Sprintf("appdynamics-php-agent-linux_x64-%s.tar.bz2", v) }, + Meta: ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"}, + Fetcher: f, + }, + &PassthroughRecipe{ + DepName: "appdynamics-java", + SourceFilenameFunc: func(v string) string { return fmt.Sprintf("appdynamics-java-agent-%s.zip", v) }, + Meta: ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"}, + Fetcher: f, + }, + &PassthroughRecipe{ + DepName: "skywalking-agent", + SourceFilenameFunc: func(v string) string { + return fmt.Sprintf("apache-skywalking-java-agent-%s.tgz", v) + }, + Meta: ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"}, + Fetcher: f, + }, + &PassthroughRecipe{ + DepName: "openjdk", + SourceFilenameFunc: func(v string) string { return fmt.Sprintf("bellsoft-jre%s-linux-amd64.tar.gz", v) }, + Meta: ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""}, + Fetcher: f, + }, + &PassthroughRecipe{ + DepName: "zulu", + SourceFilenameFunc: func(v string) string { return fmt.Sprintf("zulu%s-jre-linux_x64.tar.gz", v) }, + Meta: ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""}, + Fetcher: f, + }, + &PassthroughRecipe{ + DepName: "sapmachine", + SourceFilenameFunc: func(v string) string { + return fmt.Sprintf("sapmachine-jre-%s_linux-x64_bin.tar.gz", v) + }, + Meta: ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""}, + Fetcher: f, + }, + &PassthroughRecipe{ + DepName: "jprofiler-profiler", + SourceFilenameFunc: func(v string) string { + return fmt.Sprintf("jprofiler_linux_%s.tar.gz", underscoreVersion(v)) + }, + Meta: ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""}, + Fetcher: f, + }, + &PassthroughRecipe{ + DepName: "your-kit-profiler", + SourceFilenameFunc: func(v string) string { return fmt.Sprintf("YourKit-JavaProfiler-%s.zip", v) }, + Meta: ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""}, + Fetcher: f, + }, + // PyPI sdist deps — download and strip top-level dir, no compilation. + &PyPISourceRecipe{DepName: "setuptools", Fetcher: f}, + &PyPISourceRecipe{DepName: "flit-core", Fetcher: f}, + } +} + +// underscoreVersion replaces dots with underscores: "13.0.14" → "13_0_14". +func underscoreVersion(v string) string { + result := make([]byte, len(v)) + for i := range v { + if v[i] == '.' { + result[i] = '_' + } else { + result[i] = v[i] + } + } + return string(result) +} + +// archiveExt returns the full file extension, handling compound extensions +// like ".tar.gz", ".tar.bz2", ".tar.xz" that filepath.Ext misses. +func archiveExt(filename string) string { + for _, compound := range []string{".tar.gz", ".tar.bz2", ".tar.xz"} { + if len(filename) > len(compound) && filename[len(filename)-len(compound):] == compound { + return compound + } + } + return filepath.Ext(filename) +} diff --git a/internal/recipe/php.go b/internal/recipe/php.go new file mode 100644 index 00000000..93bf10d9 --- /dev/null +++ b/internal/recipe/php.go @@ -0,0 +1,276 @@ +package recipe + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/php" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// phpConfigureFlags are the PHP core ./configure flags (not stack-specific). +var phpConfigureFlags = []string{ + "--disable-static", + "--enable-shared", + "--enable-ftp=shared", + "--enable-sockets=shared", + "--enable-soap=shared", + "--enable-fileinfo=shared", + "--enable-bcmath", + "--enable-calendar", + "--enable-intl", + "--with-kerberos", + "--with-bz2=shared", + "--with-curl=shared", + "--enable-dba=shared", + "--with-password-argon2=/usr/lib/x86_64-linux-gnu", + "--with-cdb", + "--with-gdbm", + "--with-mysqli=shared", + "--enable-pdo=shared", + "--with-pdo-sqlite=shared,/usr", + "--with-pdo-mysql=shared,mysqlnd", + "--with-pdo-pgsql=shared", + "--with-pgsql=shared", + "--with-pspell=shared", + "--with-gettext=shared", + "--with-gmp=shared", + "--with-imap=shared", + "--with-imap-ssl=shared", + "--with-ldap=shared", + "--with-ldap-sasl", + "--with-zlib=shared", + "--with-xsl=shared", + "--with-snmp=shared", + "--enable-mbstring=shared", + "--enable-mbregex", + "--enable-exif=shared", + "--with-openssl=shared", + "--enable-fpm", + "--enable-pcntl=shared", + "--enable-sysvsem=shared", + "--enable-sysvshm=shared", + "--enable-sysvmsg=shared", + "--enable-shmop=shared", +} + +// PHPRecipe builds PHP and all of its extensions. +// Extension data (which PECL extensions and native modules to build) is embedded +// directly in the internal/php package — see internal/php/extensions.go and the +// YAML files alongside it. +type PHPRecipe struct { + Fetcher fetch.Fetcher +} + +func (p *PHPRecipe) Name() string { return "php" } +func (p *PHPRecipe) Artifact() ArtifactMeta { + // Stack is appended at runtime from src.Stack. + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (p *PHPRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, outData *output.OutData) error { + // Parse version: "8.3.2" → major="8", minor="3" + parts := strings.SplitN(src.Version, ".", 3) + if len(parts) < 2 { + return fmt.Errorf("php: invalid version %q", src.Version) + } + phpMajor := parts[0] + phpMinor := parts[1] + + // Load extension set (base + patch) from embedded YAML data in internal/php/. + extSet, err := php.Load(phpMajor, phpMinor) + if err != nil { + return fmt.Errorf("php: loading extensions: %w", err) + } + + // Step 1: apt install php_build packages. + a := apt.New(run) + if err := a.Install(ctx, s.AptPackages["php_build"]...); err != nil { + return fmt.Errorf("php: apt install php_build: %w", err) + } + + // Step 2: create symlinks from stack config. + for _, sym := range s.PHPSymlinks { + if err := run.Run("ln", "-sf", sym.Src, sym.Dst); err != nil { + return fmt.Errorf("php: symlink %s → %s: %w", sym.Src, sym.Dst, err) + } + } + + // Step 3: download and extract PHP source. + // Use the URL and checksum provided by go-depwatcher (src.URL / src.PrimaryChecksum). + phpTarball := fmt.Sprintf("/tmp/php-%s.tar.gz", src.Version) + if err := p.Fetcher.Download(ctx, src.URL, phpTarball, src.PrimaryChecksum()); err != nil { + return fmt.Errorf("php: download source: %w", err) + } + if err := run.Run("tar", "xzf", phpTarball, "-C", "/tmp/"); err != nil { + return fmt.Errorf("php: extract source: %w", err) + } + + phpSrcDir := fmt.Sprintf("/tmp/php-%s", src.Version) + phpInstallPath := fmt.Sprintf("/app/vendor/php-%s", src.Version) + + // Step 4: configure + make + make install PHP core. + configureArgs := append([]string{fmt.Sprintf("--prefix=%s", phpInstallPath)}, phpConfigureFlags...) + configureCmd := "LIBS=-lz ./configure " + strings.Join(configureArgs, " ") + if err := run.RunInDir(phpSrcDir, "bash", "-c", configureCmd); err != nil { + return fmt.Errorf("php: configure: %w", err) + } + if err := run.RunInDir(phpSrcDir, "make"); err != nil { + return fmt.Errorf("php: make: %w", err) + } + if err := run.RunInDir(phpSrcDir, "make", "install"); err != nil { + return fmt.Errorf("php: make install: %w", err) + } + + ec := php.ExtensionContext{ + PHPPath: phpInstallPath, + PHPSourceDir: phpSrcDir, + PHPMajor: phpMajor, + PHPMinor: phpMinor, + Fetcher: p.Fetcher, + } + + // Step 5: build native modules (in order — some later extensions depend on them). + for _, mod := range extSet.NativeModules { + recipe, err := php.RecipeFor(mod.Klass) + if err != nil { + return fmt.Errorf("php: native module %s: %w", mod.Name, err) + } + if err := recipe.Build(ctx, mod, ec, run); err != nil { + return fmt.Errorf("php: native module %s: %w", mod.Name, err) + } + // Wire up paths that later extensions depend on. + switch mod.Name { + case "hiredis": + ec.HiredisPath = fmt.Sprintf("/tmp/hiredis-install-%s", mod.Version) + case "libsodium": + ec.LibSodiumPath = fmt.Sprintf("/tmp/libsodium-install-%s", mod.Version) + case "lua": + ec.LuaPath = fmt.Sprintf("/tmp/lua-install-%s", mod.Version) + case "rabbitmq": + // cmake installs headers to /usr/local/include and libs to + // /usr/local/lib/x86_64-linux-gnu; pass /usr/local so that + // --with-librabbitmq-dir finds amqp.h and librabbitmq.so. + ec.RabbitMQPath = "/usr/local" + } + } + + // Step 6: build PHP extensions. + // Skip oracle extensions (oci8, pdo_oci) when /oracle is not mounted — + // matches Ruby's should_cook? check (OraclePeclRecipe.oracle_sdk?). + _, oracleErr := os.Stat("/oracle") + oraclePresent := oracleErr == nil + for _, ext := range extSet.Extensions { + if (ext.Name == "oci8" || ext.Name == "pdo_oci") && !oraclePresent { + continue + } + recipe, err := php.RecipeFor(ext.Klass) + if err != nil { + return fmt.Errorf("php: extension %s: %w", ext.Name, err) + } + if err := recipe.Build(ctx, ext, ec, run); err != nil { + return fmt.Errorf("php: extension %s: %w", ext.Name, err) + } + // Wire up ioncube path after it's downloaded (ioncube is an extension, not a native module). + if ext.Name == "ioncube" { + ec.IonCubePath = fmt.Sprintf("/tmp/ioncube-%s", ext.Version) + } + } + + // Step 7: setup_tar — copy bundled shared libs into PHP prefix. + if err := p.setupTar(ec, run); err != nil { + return fmt.Errorf("php: setup_tar: %w", err) + } + + // Step 8: populate sub-dependencies. + outData.SubDependencies = make(map[string]output.SubDependency) + for _, ext := range append(extSet.NativeModules, extSet.Extensions...) { + outData.SubDependencies[ext.Name] = output.SubDependency{Version: ext.Version} + } + + // Step 9: pack artifact. + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("php-%s-linux-x64-%s.tgz", src.Version, s.Name)) + if err := run.Run("tar", "czf", artifactPath, "-C", "/app/vendor", fmt.Sprintf("php-%s", src.Version)); err != nil { + return fmt.Errorf("php: packing artifact: %w", err) + } + // Strip the top-level directory (php-{version}/) from the artifact so paths + // start with "./" — matching Ruby's Archive.strip_top_level_directory_from_tar. + if err := archive.StripTopLevelDir(artifactPath); err != nil { + return fmt.Errorf("php: strip top-level dir: %w", err) + } + + return nil +} + +// setupTar copies bundled shared libraries into the PHP install prefix. +func (p *PHPRecipe) setupTar(ec php.ExtensionContext, run runner.Runner) error { + phpPath := ec.PHPPath + libDir := "/usr/lib/x86_64-linux-gnu" + + copies := []string{ + fmt.Sprintf("cp -a /usr/local/lib/x86_64-linux-gnu/librabbitmq.so* %s/lib/", phpPath), + fmt.Sprintf("cp -a /usr/lib/libc-client.so* %s/lib/", phpPath), + fmt.Sprintf("cp -a %s/libmcrypt.so* %s/lib", libDir, phpPath), + fmt.Sprintf("cp -a %s/libaspell.so* %s/lib", libDir, phpPath), + fmt.Sprintf("cp -a %s/libpspell.so* %s/lib", libDir, phpPath), + fmt.Sprintf("cp -a %s/libmemcached.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libuv.so* %s/lib", libDir, phpPath), + fmt.Sprintf("cp -a %s/libargon2.so* %s/lib", libDir, phpPath), + fmt.Sprintf("cp -a /usr/lib/librdkafka.so* %s/lib/", phpPath), + fmt.Sprintf("cp -a %s/libzip.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libGeoIP.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libgpgme.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libassuan.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libgpg-error.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libtidy*.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libenchant*.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libfbclient.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/librecode.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libtommath.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libmaxminddb.so* %s/lib/", libDir, phpPath), + fmt.Sprintf("cp -a %s/libssh2.so* %s/lib/", libDir, phpPath), + } + + // Hiredis libs (only if hiredis was built). + if ec.HiredisPath != "" { + copies = append(copies, fmt.Sprintf("cp -a %s/lib/libhiredis.so* %s/lib/", ec.HiredisPath, phpPath)) + } + + // IonCube loader (copy to PHP extensions zts dir). + // Ruby uses major_version = version.match(/^(\d+\.\d+)/)[1], e.g. "8.1" for PHP 8.1.32. + // The ioncube archive contains ioncube_loader_lin_8.1.so (major.minor), not ioncube_loader_lin_8.so. + if ec.IonCubePath != "" { + phpMajorMinor := ec.PHPMajor + "." + ec.PHPMinor + copies = append(copies, fmt.Sprintf( + "cp %s/ioncube/ioncube_loader_lin_%s.so $(find %s/lib/php/extensions -name 'no-debug-non-zts-*' -type d | head -1)/ioncube.so", + ec.IonCubePath, phpMajorMinor, phpPath, + )) + } + + // Run all copy commands. + for _, cmd := range copies { + if err := run.Run("sh", "-c", cmd); err != nil { + return fmt.Errorf("setup_tar: %s: %w", cmd, err) + } + } + + // Cleanup. + cleanup := fmt.Sprintf( + `rm -f "%s/etc/php-fpm.conf.default" && rm -f "%s/bin/php-cgi" && find "%s/lib/php/extensions" -name "*.a" -type f -delete`, + phpPath, phpPath, phpPath, + ) + if err := run.Run("sh", "-c", cleanup); err != nil { + return fmt.Errorf("setup_tar: cleanup: %w", err) + } + + return nil +} diff --git a/internal/recipe/php_test.go b/internal/recipe/php_test.go new file mode 100644 index 00000000..5acff60a --- /dev/null +++ b/internal/recipe/php_test.go @@ -0,0 +1,166 @@ +package recipe_test + +import ( + "context" + "strings" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/recipe" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPHPRecipeName(t *testing.T) { + r := &recipe.PHPRecipe{} + assert.Equal(t, "php", r.Name()) +} + +func TestPHPRecipeArtifact(t *testing.T) { + r := &recipe.PHPRecipe{} + meta := r.Artifact() + assert.Equal(t, "linux", meta.OS) + assert.Equal(t, "x64", meta.Arch) +} + +func TestPHPRecipeBuildInstallsAptPackages(t *testing.T) { + useTempWorkDir(t) + fakeRun := runner.NewFakeRunner() + + s := &stack.Stack{ + Name: "cflinuxfs4", + AptPackages: map[string][]string{ + "php_build": {"libssl-dev", "libxml2-dev", "libbz2-dev"}, + }, + PHPSymlinks: []stack.Symlink{}, + } + src := &source.Input{Version: "8.3.2"} + outData := &output.OutData{} + + r := &recipe.PHPRecipe{Fetcher: newFakeFetcher()} + _ = r.Build(context.Background(), s, src, fakeRun, outData) + + assert.True(t, hasCallMatching(fakeRun.Calls, "apt-get", "libssl-dev"), "should apt-get install libssl-dev") + assert.True(t, hasCallMatching(fakeRun.Calls, "apt-get", "libxml2-dev"), "should apt-get install libxml2-dev") +} + +func TestPHPRecipeBuildCreatesSymlinks(t *testing.T) { + useTempWorkDir(t) + fakeRun := runner.NewFakeRunner() + + s := &stack.Stack{ + Name: "cflinuxfs4", + AptPackages: map[string][]string{ + "php_build": {}, + }, + PHPSymlinks: []stack.Symlink{ + {Src: "/usr/include/x86_64-linux-gnu/curl", Dst: "/usr/local/include/curl"}, + {Src: "/usr/include/x86_64-linux-gnu/gmp.h", Dst: "/usr/include/gmp.h"}, + }, + } + src := &source.Input{Version: "8.3.2"} + outData := &output.OutData{} + + r := &recipe.PHPRecipe{Fetcher: newFakeFetcher()} + _ = r.Build(context.Background(), s, src, fakeRun, outData) + + assert.True(t, hasCallMatching(fakeRun.Calls, "ln", "/usr/local/include/curl"), "should create curl symlink") + assert.True(t, hasCallMatching(fakeRun.Calls, "ln", "/usr/include/gmp.h"), "should create gmp.h symlink") +} + +func TestPHPRecipeBuildConfigureFlags(t *testing.T) { + useTempWorkDir(t) + fakeRun := runner.NewFakeRunner() + + s := &stack.Stack{ + Name: "cflinuxfs4", + AptPackages: map[string][]string{"php_build": {}}, + PHPSymlinks: []stack.Symlink{}, + } + src := &source.Input{Version: "8.3.2"} + outData := &output.OutData{} + + r := &recipe.PHPRecipe{Fetcher: newFakeFetcher()} + _ = r.Build(context.Background(), s, src, fakeRun, outData) + + // The configure command is run via bash -c with LIBS=-lz prefix. + found := false + for _, c := range fakeRun.Calls { + if c.Name == "bash" { + for _, arg := range c.Args { + if strings.Contains(arg, "LIBS=-lz") && strings.Contains(arg, "--disable-static") { + found = true + } + } + } + } + assert.True(t, found, "should run configure with LIBS=-lz and --disable-static") +} + +func TestPHPRecipeBuildPopulatesSubDependencies(t *testing.T) { + useTempWorkDir(t) + fakeRun := runner.NewFakeRunner() + + s := &stack.Stack{ + Name: "cflinuxfs4", + AptPackages: map[string][]string{"php_build": {}}, + PHPSymlinks: []stack.Symlink{}, + } + src := &source.Input{Version: "8.3.2"} + outData := &output.OutData{} + + r := &recipe.PHPRecipe{Fetcher: newFakeFetcher()} + _ = r.Build(context.Background(), s, src, fakeRun, outData) + + // SubDependencies should be populated from the embedded extension YAML. + // Check a representative sample of well-known extensions. + require.NotNil(t, outData.SubDependencies) + assert.Contains(t, outData.SubDependencies, "apcu", "apcu should be in sub-dependencies") + assert.Contains(t, outData.SubDependencies, "rabbitmq", "rabbitmq should be in sub-dependencies") + assert.Greater(t, len(outData.SubDependencies), 20, "should have many sub-dependencies from the embedded YAML") +} + +func TestPHPRecipeBuildDownloadsSource(t *testing.T) { + useTempWorkDir(t) + fakeRun := runner.NewFakeRunner() + + s := &stack.Stack{ + Name: "cflinuxfs4", + AptPackages: map[string][]string{"php_build": {}}, + PHPSymlinks: []stack.Symlink{}, + } + src := &source.Input{ + Version: "8.3.2", + URL: "https://www.php.net/distributions/php-8.3.2.tar.gz", + SHA256: "abc123", + } + outData := &output.OutData{} + + f := newFakeFetcher() + r := &recipe.PHPRecipe{Fetcher: f} + _ = r.Build(context.Background(), s, src, fakeRun, outData) + + // Should download PHP source via Fetcher (not wget). + assert.True(t, hasDownload(f, src.URL), "should download PHP source via Fetcher") +} + +func TestPHPRecipeBuildInvalidVersion(t *testing.T) { + useTempWorkDir(t) + fakeRun := runner.NewFakeRunner() + + s := &stack.Stack{ + Name: "cflinuxfs4", + AptPackages: map[string][]string{"php_build": {}}, + PHPSymlinks: []stack.Symlink{}, + } + src := &source.Input{Version: "invalid"} + outData := &output.OutData{} + + r := &recipe.PHPRecipe{Fetcher: newFakeFetcher()} + err := r.Build(context.Background(), s, src, fakeRun, outData) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid version") +} diff --git a/internal/recipe/pip.go b/internal/recipe/pip.go new file mode 100644 index 00000000..ec08c193 --- /dev/null +++ b/internal/recipe/pip.go @@ -0,0 +1,114 @@ +package recipe + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// PipRecipe builds pip: pip3 download + bundle setuptools + wheel. +type PipRecipe struct { + Fetcher fetch.Fetcher +} + +func (p *PipRecipe) Name() string { return "pip" } +func (p *PipRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: ""} +} + +func (p *PipRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, r runner.Runner, out *output.OutData) error { + return (&BundleRecipe{ + DepName: "pip", + Meta: ArtifactMeta{OS: "linux", Arch: "noarch"}, + Fetcher: p.Fetcher, + MainPackage: func(version string) string { + return fmt.Sprintf("pip==%s", version) + }, + DownloadArgs: []string{"--no-binary", ":all:"}, + // pip's extra steps: download source tarball, strip top-level dir, then extract. + ExtraSteps: func(ctx context.Context, tmpDir string, src *source.Input, f fetch.Fetcher, r runner.Runner) error { + pipSrcTar := fmt.Sprintf("%s/pip-%s.tar.gz", tmpDir, src.Version) + if err := f.Download(ctx, src.URL, pipSrcTar, src.PrimaryChecksum()); err != nil { + return fmt.Errorf("downloading pip source: %w", err) + } + if err := archive.StripTopLevelDir(pipSrcTar); err != nil { + return fmt.Errorf("stripping top-level dir from pip source: %w", err) + } + if err := r.RunInDir(tmpDir, "tar", "zxf", fmt.Sprintf("pip-%s.tar.gz", src.Version)); err != nil { + return fmt.Errorf("extracting pip source: %w", err) + } + return nil + }, + ExtraDeps: []string{ + "setuptools", + "wheel>=0.46.2", // CVE-2026-24049 + }, + }).Build(ctx, s, src, r, out) +} + +// PipenvRecipe builds pipenv: pip3 download + bundle 7 dependencies. +type PipenvRecipe struct { + Fetcher fetch.Fetcher +} + +func (p *PipenvRecipe) Name() string { return "pipenv" } +func (p *PipenvRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: ""} +} + +func (p *PipenvRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, r runner.Runner, out *output.OutData) error { + return (&BundleRecipe{ + DepName: "pipenv", + Meta: ArtifactMeta{OS: "linux", Arch: "noarch"}, + Fetcher: p.Fetcher, + MainPackage: func(version string) string { + return fmt.Sprintf("pipenv==%s", version) + }, + DownloadArgs: []string{"--no-cache-dir", "--no-binary", ":all:"}, + // pipenv: also download the source tarball into tmpDir for bundling. + ExtraSteps: func(ctx context.Context, tmpDir string, src *source.Input, f fetch.Fetcher, r runner.Runner) error { + pipenvTar := fmt.Sprintf("pipenv-%s.tar.gz", src.Version) + return f.Download(ctx, src.URL, fmt.Sprintf("%s/%s", tmpDir, pipenvTar), src.PrimaryChecksum()) + }, + ExtraDeps: []string{ + "pytest-runner", + "setuptools_scm", + "parver", + "wheel>=0.46.2", // CVE-2026-24049 + "invoke", + "flit_core", + "hatch-vcs", + }, + // pipenv output path has a 'v' prefix: /tmp/pipenv-v{version}.tgz + OutputPath: func(version string) string { + return fmt.Sprintf("/tmp/pipenv-v%s.tgz", version) + }, + }).Build(ctx, s, src, r, out) +} + +// setupPythonAndPip installs the Python interpreter and pip via apt, then +// upgrades pip and setuptools. The packages to install are read from +// s.AptPackages["pip_build"] (stacks/*.yaml) so they can be adjusted per +// stack without modifying Go source. +func setupPythonAndPip(ctx context.Context, s *stack.Stack, r runner.Runner) error { + if err := r.RunWithEnv( + map[string]string{"DEBIAN_FRONTEND": "noninteractive"}, + "apt-get", "update", + ); err != nil { + return err + } + installArgs := append([]string{"install", "-y"}, s.AptPackages["pip_build"]...) + if err := r.RunWithEnv( + map[string]string{"DEBIAN_FRONTEND": "noninteractive"}, + "apt-get", installArgs..., + ); err != nil { + return err + } + return r.Run("pip3", "install", "--upgrade", "pip", "setuptools") +} diff --git a/internal/recipe/python.go b/internal/recipe/python.go new file mode 100644 index 00000000..4e49e236 --- /dev/null +++ b/internal/recipe/python.go @@ -0,0 +1,138 @@ +package recipe + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// PythonRecipe builds Python, matching the Ruby builder's build_python method +// in builder.rb exactly. +// +// Critical ordering: configure runs BEFORE libdb-dev/libgdbm-dev are installed, +// so _dbm and _gdbm modules are NOT detected/built (matching Ruby output). +// Then packages are installed, tcl/tk debs are downloaded and extracted, and +// finally make && make install are run. +type PythonRecipe struct { + Fetcher fetch.Fetcher +} + +func (p *PythonRecipe) Name() string { return "python" } +func (p *PythonRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (p *PythonRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, _ *output.OutData) error { + a := apt.New(run) + + tclVersion := s.Python.TCLVersion // e.g. "8.6" + aptFlag := "-y" + if s.Python.UseForceYes { + aptFlag = "--force-yes" + } + // debPkgs are .deb packages that are downloaded and extracted (not installed) + // into the build prefix to bundle tcl/tk and its X11 dependencies. + // libtcl/libtk versions come from s.Python.TCLVersion (stack config). + // Additional packages (e.g. libxss1) live in s.AptPackages["python_deb_extras"] + // so they can be adjusted per stack without modifying Go source. + debPkgs := append( + []string{ + fmt.Sprintf("libtcl%s", tclVersion), + fmt.Sprintf("libtk%s", tclVersion), + }, + s.AptPackages["python_deb_extras"]..., + ) + + builtPath := fmt.Sprintf("/app/vendor/python-%s", src.Version) + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("python-%s-linux-x64.tgz", src.Version)) + + // Download Python source tarball. + srcTarball := fmt.Sprintf("/tmp/Python-%s.tgz", src.Version) + if err := p.Fetcher.Download(ctx, src.URL, srcTarball, src.PrimaryChecksum()); err != nil { + return fmt.Errorf("python: downloading source: %w", err) + } + + // Extract source. + srcDir := fmt.Sprintf("/tmp/Python-%s", src.Version) + if err := run.Run("mkdir", "-p", srcDir); err != nil { + return err + } + if err := run.Run("tar", "xf", srcTarball, "-C", "/tmp"); err != nil { + return fmt.Errorf("python: extracting source: %w", err) + } + + // Create install prefix dir. + if err := run.Run("mkdir", "-p", builtPath); err != nil { + return err + } + + // Configure flags reference the tcl/tk version from stack config. + tclInclude := fmt.Sprintf("-I/usr/include/tcl%s", tclVersion) + tclLib := fmt.Sprintf("-L/usr/lib/x86_64-linux-gnu -ltcl%s -ltk%s", tclVersion, tclVersion) + + // Step 1: Run ./configure BEFORE installing libdb-dev/libgdbm-dev. + // This matches the Ruby builder which also runs configure first, then + // installs packages. Because db/gdbm headers are absent at configure time, + // Python will NOT build _dbm or _gdbm extension modules. + configureArgs := []string{ + fmt.Sprintf("--prefix=%s", builtPath), + "--enable-shared", + "--with-ensurepip=yes", + "--with-dbmliborder=bdb:gdbm", + fmt.Sprintf("--with-tcltk-includes=%s", tclInclude), + fmt.Sprintf("--with-tcltk-libs=%s", tclLib), + "--enable-unicode=ucs4", + } + if err := run.RunInDir(srcDir, "./configure", configureArgs...); err != nil { + return fmt.Errorf("python: configure: %w", err) + } + + // Step 2: Install build packages AFTER configure (matching Ruby builder order). + if err := a.Install(ctx, s.AptPackages["python_build"]...); err != nil { + return fmt.Errorf("python: apt install python_build: %w", err) + } + + // Step 3: Download tcl/tk .deb packages (without installing). + aptArgs := append([]string{aptFlag, "-y", "-d", "install", "--reinstall"}, debPkgs...) + if err := run.RunWithEnv(map[string]string{"DEBIAN_FRONTEND": "noninteractive"}, "apt-get", aptArgs...); err != nil { + return fmt.Errorf("python: downloading tcl/tk debs: %w", err) + } + + // Step 4: Extract each tcl/tk .deb into the install prefix to bundle them. + aptCacheDir := "/var/cache/apt/archives" + for _, pkg := range debPkgs { + if err := run.Run("sh", "-c", fmt.Sprintf("dpkg -x %s/%s_*.deb %s", aptCacheDir, pkg, builtPath)); err != nil { + return fmt.Errorf("python: dpkg -x %s: %w", pkg, err) + } + } + + // Step 5: Compile and install. + if err := run.RunInDir(srcDir, "make"); err != nil { + return fmt.Errorf("python: make: %w", err) + } + if err := run.RunInDir(srcDir, "make", "install"); err != nil { + return fmt.Errorf("python: make install: %w", err) + } + + // Step 6: Create bin/python symlink → ./python3 (relative, matching Ruby recipe's + // File.symlink('./python3', "#{destdir}/bin/python")). + pythonLink := fmt.Sprintf("%s/bin/python", builtPath) + if err := run.Run("ln", "-sf", "./python3", pythonLink); err != nil { + return fmt.Errorf("python: creating bin/python symlink: %w", err) + } + + // Step 7: Pack with --hard-dereference to resolve all symlinks into the artifact. + if err := archive.PackWithDereference(run, artifactPath, builtPath); err != nil { + return fmt.Errorf("python: packing artifact: %w", err) + } + + return nil +} diff --git a/internal/recipe/r.go b/internal/recipe/r.go new file mode 100644 index 00000000..93909867 --- /dev/null +++ b/internal/recipe/r.go @@ -0,0 +1,191 @@ +package recipe + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/compiler" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/portile" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// RRecipe builds R from source and installs remotes + 4 R packages +// (forecast, plumber, Rserve, shiny). Sub-dependency inputs are read +// from well-known Concourse resource directories alongside the working +// directory. +type RRecipe struct { + Fetcher fetch.Fetcher +} + +func (r *RRecipe) Name() string { return "r" } +func (r *RRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: ""} +} + +// rPackage holds a sub-dependency name alongside the dir that contains its +// source/data.json Concourse resource. +type rPackage struct { + name string // R package name (passed to devtools::install_version) + srcDir string // e.g. "source-forecast-latest" +} + +// rSubDeps lists the R packages to install, in the same order as the Ruby builder: +// Rserve, forecast, shiny, plumber. +var rSubDeps = []rPackage{ + {name: "Rserve", srcDir: "source-rserve-latest"}, + {name: "forecast", srcDir: "source-forecast-latest"}, + {name: "shiny", srcDir: "source-shiny-latest"}, + {name: "plumber", srcDir: "source-plumber-latest"}, +} + +func (r *RRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, outData *output.OutData) error { + a := apt.New(run) + + // Step 1: Install R build dependencies from stack config (gfortran, libpcre, etc.) + if err := a.Install(ctx, s.AptPackages["r_build"]...); err != nil { + return fmt.Errorf("r: apt install r_build: %w", err) + } + + // Step 2: Set up gfortran (stack-driven version). + gf := compiler.NewGfortran(s.Compilers.Gfortran, a, run) + if err := gf.Setup(ctx); err != nil { + return fmt.Errorf("r: gfortran setup: %w", err) + } + + // Step 3: Download R source. + srcTarball := fmt.Sprintf("/tmp/R-%s.tar.gz", src.Version) + if err := r.Fetcher.Download(ctx, src.URL, srcTarball, src.PrimaryChecksum()); err != nil { + return fmt.Errorf("r: downloading source: %w", err) + } + + // Compute SHA256 of the downloaded tarball (matches Ruby's git_commit_sha field). + sourceSHA, err := fileSHA256(srcTarball) + if err != nil { + return fmt.Errorf("r: computing source sha256: %w", err) + } + + // Step 4: Build via portile (configure + make + make install). + installPrefix := fmt.Sprintf("/usr/local") + pt := &portile.Portile{ + Name: "R", + Version: src.Version, + URL: src.URL, + // We already downloaded to srcTarball — portile fetches independently, so + // we pass checksum for portile's own download/verify path. + Checksum: src.PrimaryChecksum(), + Prefix: installPrefix, + Options: []string{ + "--with-readline=no", + "--with-x=no", + "--enable-R-shlib", + }, + Runner: run, + Fetcher: r.Fetcher, + } + if err := pt.Cook(ctx); err != nil { + return fmt.Errorf("r: portile cook: %w", err) + } + + // Step 5: Install remotes (required for install_version below). + // devtools::install_version was deprecated in devtools 2.5.0 and now requires + // the remotes package separately. Use remotes directly instead. + // We also explicitly install packages that devtools used to bundle as freebies + // (stringr, cli, etc.) to avoid breaking R apps that depend on them implicitly. + remotesCmd := `/usr/local/lib/R/bin/R --vanilla -e 'install.packages("remotes", repos="https://cran.r-project.org")'` + if err := run.Run("sh", "-c", remotesCmd); err != nil { + return fmt.Errorf("r: installing remotes: %w", err) + } + + depsCmd := `/usr/local/lib/R/bin/R --vanilla -e 'install.packages(c("cli", "curl", "lifecycle", "R6", "rlang", "withr", "processx", "callr", "stringr"), repos="https://cran.r-project.org")'` + if err := run.Run("sh", "-c", depsCmd); err != nil { + return fmt.Errorf("r: installing explicit r dependencies: %w", err) + } + + // Step 6: Read sub-dependency source inputs and install R packages. + // Install order matches Ruby: Rserve, forecast, shiny, plumber. + subDeps := make(map[string]output.SubDependency) + for _, pkg := range rSubDeps { + subInput, err := source.FromFile(fmt.Sprintf("%s/data.json", pkg.srcDir)) + if err != nil { + return fmt.Errorf("r: reading sub-dep source for %s: %w", pkg.name, err) + } + + // Format version for Rserve: "1.8.15" → "1.8-15" + pkgVersion := subInput.Version + if pkg.name == "Rserve" { + pkgVersion = formatRserveVersion(subInput.Version) + } + + // Install via devtools::install_version with dependencies=TRUE and type='source', + // matching the ruby-builder-final tag behaviour. + rCmd := fmt.Sprintf( + `/usr/local/lib/R/bin/R --vanilla -e "require('remotes'); remotes::install_version('%s', '%s', repos='https://cran.r-project.org', type='source', dependencies=TRUE)"`, + pkg.name, pkgVersion, + ) + if err := run.Run("sh", "-c", rCmd); err != nil { + return fmt.Errorf("r: installing R package %s: %w", pkg.name, err) + } + + subDeps[pkg.name] = output.SubDependency{ + Source: &output.SubDepSource{ + URL: subInput.URL, + SHA256: subInput.SHA256, + }, + Version: subInput.Version, + } + } + + // Step 7: Remove remotes after use. The explicitly pre-installed packages + // (cli, curl, stringr, etc.) are intentionally left in the artifact — they + // are R app runtime dependencies that devtools used to bundle as freebies. + removeRemotesCmd := `/usr/local/lib/R/bin/R --vanilla -e 'remove.packages("remotes")'` + if err := run.Run("sh", "-c", removeRemotesCmd); err != nil { + return fmt.Errorf("r: removing remotes: %w", err) + } + + // Step 8: Copy gfortran libs into R install. + // Ruby copies gfortran/f951 into /usr/local/lib/R/bin (R's own bin dir) + // and gfortran libs into /usr/local/lib/R/lib, so they end up inside the + // packed artifact (which is tarred from /usr/local/lib/R). + rLibDir := "/usr/local/lib/R/lib" + rBinDir := "/usr/local/lib/R/bin" + if err := gf.CopyLibs(ctx, rLibDir, rBinDir); err != nil { + return fmt.Errorf("r: copying gfortran libs: %w", err) + } + + // Step 9: Create f95 symlink → gfortran (relative, matching Ruby's ln -s ./gfortran ./bin/f95). + if err := run.RunInDir(rBinDir, "ln", "-s", "./gfortran", "./f95"); err != nil { + return fmt.Errorf("r: creating f95 symlink: %w", err) + } + + // Step 10: Pack the R install from /usr/local/lib/R. + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("r-%s.tgz", src.Version)) + if err := run.RunInDir("/usr/local/lib/R", "tar", "zcvf", artifactPath, "."); err != nil { + return fmt.Errorf("r: packing artifact: %w", err) + } + + // Populate sub-dependencies and git_commit_sha in outData. + outData.SubDependencies = subDeps + outData.GitCommitSHA = sourceSHA + + return nil +} + +// formatRserveVersion converts "1.8.14" → "1.8-14" +// (first two dot-separated parts joined by '.', remainder joined by '-'). +func formatRserveVersion(v string) string { + parts := strings.Split(v, ".") + if len(parts) <= 2 { + return v + } + prefix := strings.Join(parts[:2], ".") + suffix := strings.Join(parts[2:], ".") + return fmt.Sprintf("%s-%s", prefix, suffix) +} diff --git a/internal/recipe/recipe.go b/internal/recipe/recipe.go new file mode 100644 index 00000000..62229446 --- /dev/null +++ b/internal/recipe/recipe.go @@ -0,0 +1,66 @@ +// Package recipe defines the Recipe interface and the global recipe registry. +// Each dependency type implements Recipe. The registry maps dep names to builders. +package recipe + +import ( + "context" + "fmt" + + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// Recipe is the interface every dependency builder must implement. +type Recipe interface { + // Name returns the dependency name (e.g. "ruby", "php"). + Name() string + + // Build performs the full build: download, configure, compile, install, archive. + // It populates outData with artifact URL, SHA256, and any sub-dependencies. + Build(ctx context.Context, s *stack.Stack, src *source.Input, r runner.Runner, outData *output.OutData) error + + // Artifact returns the artifact metadata for this recipe. + Artifact() ArtifactMeta +} + +// ArtifactMeta describes the artifact naming for a recipe. +type ArtifactMeta struct { + OS string // "linux" or "windows" + Arch string // "x64", "noarch", "x86-64" + Stack string // stack name, "any-stack", or "" (use build stack) +} + +// Registry maps dependency names to recipe constructors. +type Registry struct { + recipes map[string]Recipe +} + +// NewRegistry creates an empty recipe registry. +func NewRegistry() *Registry { + return &Registry{recipes: make(map[string]Recipe)} +} + +// Register adds a recipe to the registry. +func (r *Registry) Register(recipe Recipe) { + r.recipes[recipe.Name()] = recipe +} + +// Get returns the recipe for the given dependency name. +func (r *Registry) Get(name string) (Recipe, error) { + recipe, ok := r.recipes[name] + if !ok { + return nil, fmt.Errorf("no recipe registered for %q", name) + } + return recipe, nil +} + +// Names returns all registered recipe names. +func (r *Registry) Names() []string { + names := make([]string, 0, len(r.recipes)) + for name := range r.recipes { + names = append(names, name) + } + return names +} diff --git a/internal/recipe/recipe_compiled_test.go b/internal/recipe/recipe_compiled_test.go new file mode 100644 index 00000000..32b6d16d --- /dev/null +++ b/internal/recipe/recipe_compiled_test.go @@ -0,0 +1,1100 @@ +package recipe_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/recipe" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ── helpers ────────────────────────────────────────────────────────────────── + +func newCompiledStack(t *testing.T) *stack.Stack { + t.Helper() + return &stack.Stack{ + Name: "cflinuxfs4", + Bootstrap: stack.BootstrapConfig{ + Ruby: stack.BootstrapBinary{ + URL: "https://example.com/ruby-bootstrap.tgz", + SHA256: "deadbeef", + InstallDir: "/opt/ruby", + }, + JRuby: stack.BootstrapBinary{ + URL: "https://example.com/openjdk.tar.gz", + SHA256: "cafebabe", + InstallDir: t.TempDir() + "/java", + }, + Go: stack.BootstrapBinary{ + URL: "https://example.com/go-bootstrap.tar.gz", + SHA256: "deadbeef", + }, + }, + Compilers: stack.CompilerConfig{ + GCC: stack.GCCConfig{ + Version: 12, + Packages: []string{"gcc-12", "g++-12"}, + PPA: "ppa:ubuntu-toolchain-r/test", + ToolPackages: []string{"software-properties-common"}, + }, + Gfortran: stack.GfortranConfig{ + Version: 11, + Bin: "/usr/bin/gfortran-11", + LibPath: "/usr/lib/gcc/x86_64-linux-gnu/11", + Packages: []string{"gfortran"}, + }, + }, + AptPackages: map[string][]string{ + "ruby_build": {"libffi-dev"}, + "python_build": {"libdb-dev", "libgdbm-dev", "tk8.6-dev"}, + "python_deb_extras": {"libxss1"}, + "node_build": {}, + "hwc_build": {"mingw-w64"}, + "pip_build": {"python3", "python3-pip"}, + "libgdiplus_build": {"automake", "libtool", "libglib2.0-dev", "libcairo2-dev"}, + }, + Python: stack.PythonConfig{ + TCLVersion: "8.6", + UseForceYes: true, + }, + HTTPDSubDeps: stack.HTTPDSubDepsConfig{ + APR: stack.HTTPDSubDep{ + Version: "1.7.4", + URL: "https://example.com/apr-1.7.4.tar.gz", + SHA256: "aprsha256", + }, + APRIconv: stack.HTTPDSubDep{ + Version: "1.2.2", + URL: "https://example.com/apr-iconv-1.2.2.tar.gz", + SHA256: "apriconvsha256", + }, + APRUtil: stack.HTTPDSubDep{ + Version: "1.6.3", + URL: "https://example.com/apr-util-1.6.3.tar.gz", + SHA256: "aprutilsha256", + }, + ModAuthOpenidc: stack.HTTPDSubDep{ + Version: "2.3.8", + URL: "https://example.com/mod_auth_openidc-2.3.8.tar.gz", + SHA256: "modauthsha256", + }, + }, + } +} + +// helpers are in recipe_helpers_test.go + +// ── RubyRecipe ─────────────────────────────────────────────────────────────── + +func TestRubyRecipeName(t *testing.T) { + r := &recipe.RubyRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "ruby", r.Name()) +} + +func TestRubyRecipeArtifact(t *testing.T) { + r := &recipe.RubyRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "linux", r.Artifact().OS) + assert.Equal(t, "x64", r.Artifact().Arch) + assert.Equal(t, "", r.Artifact().Stack) // stack-specific → set at build time +} + +func TestRubyRecipeBuild(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "ruby-3.3.1-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.RubyRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("ruby", "3.3.1", "https://cache.ruby-lang.org/pub/ruby/ruby-3.3.1.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should install apt packages. + assert.True(t, hasCallMatching(run.Calls, "apt-get", "libffi-dev"), "should apt-install ruby_build packages") + + // Should have downloaded via portile fetcher. + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, src.URL, f.DownloadedURLs[0].URL) + + // Should invoke tar + configure chain (mkdir, tar xf, mv, ./configure, make, make install, tar czf). + names := callNames(run.Calls) + assert.Contains(t, names, "mkdir") + assert.Contains(t, names, "tar") + + // Should configure with the correct portile flags. + assert.True(t, hasCallMatching(run.Calls, "./configure", "--enable-load-relative"), "missing --enable-load-relative") + assert.True(t, hasCallMatching(run.Calls, "./configure", "--disable-install-doc"), "missing --disable-install-doc") + assert.True(t, hasCallMatching(run.Calls, "./configure", "--without-gmp"), "missing --without-gmp") + + // Prefix should include the version. + assert.True(t, hasCallMatching(run.Calls, "./configure", "ruby-3.3.1"), "prefix should reference version") +} + +func TestRubyRecipeFetchError(t *testing.T) { + f := newFakeFetcher() + f.ErrMap[newInput("ruby", "3.3.1", "https://example.com/ruby.tgz").URL] = assert.AnError + r := &recipe.RubyRecipe{Fetcher: f} + run := runner.NewFakeRunner() + + src := newInput("ruby", "3.3.1", "https://example.com/ruby.tgz") + err := r.Build(context.Background(), newCompiledStack(t), src, run, &output.OutData{}) + require.Error(t, err) +} + +// ── BundlerRecipe ───────────────────────────────────────────────────────────── + +func TestBundlerRecipeName(t *testing.T) { + r := &recipe.BundlerRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "bundler", r.Name()) +} + +func TestBundlerRecipeArtifact(t *testing.T) { + r := &recipe.BundlerRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "noarch", r.Artifact().Arch) +} + +func TestBundlerRecipeBuild(t *testing.T) { + f := newFakeFetcher() + r := &recipe.BundlerRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("bundler", "2.5.6", "https://rubygems.org/gems/bundler-2.5.6.gem") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should have downloaded the Ruby bootstrap binary. + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, s.Bootstrap.Ruby.URL, f.DownloadedURLs[0].URL) + + // Should create the install dir and extract the bootstrap. + assert.True(t, hasCallMatching(run.Calls, "mkdir", "/opt/ruby"), "should mkdir install dir") + assert.True(t, hasCallMatching(run.Calls, "tar", "/opt/ruby"), "should extract bootstrap to install dir") + + // Should call gem install with version. + // The gem binary is invoked by full path (e.g. /opt/ruby/bin/gem) to avoid + // PATH resolution issues, so we match on the bootstrap install dir. + gemBin := filepath.Join(s.Bootstrap.Ruby.InstallDir, "bin", "gem") + assert.True(t, hasCallMatching(run.Calls, gemBin, "bundler"), "should call gem install bundler") + assert.True(t, hasCallMatching(run.Calls, gemBin, "2.5.6"), "should install specific version") + assert.True(t, hasCallMatching(run.Calls, gemBin, "--no-document"), "should skip documentation") + + // gem install should run with GEM_HOME set so gems land in an isolated tmpdir. + assert.True(t, hasCallWithEnv(run.Calls, gemBin, "GEM_HOME"), "gem install should have GEM_HOME env set") +} + +// ── PythonRecipe ────────────────────────────────────────────────────────────── + +func TestPythonRecipeName(t *testing.T) { + r := &recipe.PythonRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "python", r.Name()) +} + +func TestPythonRecipeArtifact(t *testing.T) { + r := &recipe.PythonRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "x64", r.Artifact().Arch) +} + +func TestPythonRecipeBuildForceYes(t *testing.T) { + f := newFakeFetcher() + r := &recipe.PythonRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + s.Python.UseForceYes = true + src := newInput("python", "3.12.0", "https://python.org/Python-3.12.0.tgz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // cflinuxfs4: should use --force-yes when downloading deb packages. + assert.True(t, hasCallMatching(run.Calls, "apt-get", "--force-yes"), "cflinuxfs4 should use --force-yes") + + // Should invoke portile configure with tcl/tk flags. + assert.True(t, hasCallMatching(run.Calls, "./configure", "--enable-shared"), "missing --enable-shared") + assert.True(t, hasCallMatching(run.Calls, "./configure", "tcl8.6"), "configure should reference tcl version") + + // Should extract each deb via dpkg -x. + assert.True(t, hasCallMatching(run.Calls, "sh", "dpkg -x"), "should run dpkg -x for each deb") + + // Should create bin/python symlink. + assert.True(t, hasCallMatching(run.Calls, "ln", "python"), "should create bin/python symlink") +} + +func TestPythonRecipeBuildNoForceYes(t *testing.T) { + f := newFakeFetcher() + r := &recipe.PythonRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + s.Python.UseForceYes = false // cflinuxfs5 + src := newInput("python", "3.12.0", "https://python.org/Python-3.12.0.tgz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // cflinuxfs5: should NOT use --force-yes. + assert.False(t, hasCallMatching(run.Calls, "apt-get", "--force-yes"), "cflinuxfs5 must not use --force-yes") +} + +// ── NodeRecipe ──────────────────────────────────────────────────────────────── + +func TestNodeRecipeName(t *testing.T) { + r := &recipe.NodeRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "node", r.Name()) +} + +func TestNodeRecipeArtifact(t *testing.T) { + r := &recipe.NodeRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "x64", r.Artifact().Arch) +} + +func TestNodeRecipeStripsVPrefix(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "node-22.14.0-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.NodeRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("node", "v22.14.0", "https://nodejs.org/v22.14.0.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // configure uses --prefix=/ (DESTDIR-based install) + --openssl-use-def-ca-store. + // The version (without `v`) appears in the DESTDIR passed to make install, not in configure. + assert.True(t, hasCallMatching(run.Calls, "./configure", "--prefix=/"), "configure must use --prefix=/") + assert.True(t, hasCallMatching(run.Calls, "./configure", "--openssl-use-def-ca-store"), "configure must pass --openssl-use-def-ca-store") + assert.False(t, hasCallMatching(run.Calls, "./configure", "v22.14.0"), "configure must not reference v-prefixed version") + // DESTDIR uses stripped version; "node-v22.14.0" is acceptable since DESTDIR path includes the `v`. + assert.True(t, hasCallMatching(run.Calls, "make", "DESTDIR="), "make install must pass DESTDIR") + assert.True(t, hasCallMatching(run.Calls, "make", "22.14.0"), "make install DESTDIR must contain the version") +} + +func TestNodeRecipeSetsUpGCC(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "node-22.14.0-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.NodeRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("node", "v22.14.0", "https://nodejs.org/v22.14.0.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should install software-properties-common. + assert.True(t, hasCallMatching(run.Calls, "apt-get", "software-properties-common"), "should install software-properties-common") + + // cflinuxfs4 has a PPA — should add it. + assert.True(t, hasCallMatching(run.Calls, "add-apt-repository", "ppa:ubuntu-toolchain-r/test"), "should add GCC PPA on cflinuxfs4") + + // Should set up update-alternatives. + assert.True(t, hasCallMatching(run.Calls, "update-alternatives", "gcc"), "should set up update-alternatives for gcc") +} + +func TestNodeRecipeSkipsPPAWhenEmpty(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "node-22.14.0-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.NodeRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + s.Compilers.GCC.PPA = "" // cflinuxfs5 — no PPA + src := newInput("node", "v22.14.0", "https://nodejs.org/v22.14.0.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should NOT call add-apt-repository when PPA is empty. + assert.False(t, hasCallMatching(run.Calls, "add-apt-repository", ""), "must not add PPA when PPA is empty") +} + +// ── GoRecipe ────────────────────────────────────────────────────────────────── + +func TestGoRecipeName(t *testing.T) { + r := &recipe.GoRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "go", r.Name()) +} + +func TestGoRecipeArtifact(t *testing.T) { + r := &recipe.GoRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "x64", r.Artifact().Arch) +} + +func TestGoRecipeStripsGoPrefix(t *testing.T) { + useTempWorkDir(t) + // Artifact filename uses a dash between name and version so findIntermediateArtifact + // can locate it via the "go-1.24.2*.tar.gz" glob pattern. + writeFakeArtifact(t, "go-1.24.2.linux-amd64.tar.gz") + + f := newFakeFetcher() + r := &recipe.GoRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("go", "go1.24.2", "https://go.dev/dl/go1.24.2.src.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should download bootstrap AND source — 2 downloads total. + require.Len(t, f.DownloadedURLs, 2) + assert.True(t, hasDownload(f, s.Bootstrap.Go.URL), "should download bootstrap go binary") + assert.True(t, hasDownload(f, src.URL), "should download go source") + + // Artifact path should use stripped version. + assert.True(t, hasCallMatching(run.Calls, "tar", "1.24.2"), "artifact should use version without go prefix") + + // Should run make.bash. + assert.True(t, hasCallMatching(run.Calls, "bash", "make.bash"), "should run make.bash") + + // make.bash should have GOROOT_BOOTSTRAP env. + assert.True(t, hasCallWithEnv(run.Calls, "bash", "GOROOT_BOOTSTRAP"), "make.bash should set GOROOT_BOOTSTRAP") +} + +// ── NginxRecipe ─────────────────────────────────────────────────────────────── + +func TestNginxRecipeName(t *testing.T) { + r := &recipe.NginxRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "nginx", r.Name()) +} + +func TestNginxRecipeArtifact(t *testing.T) { + r := &recipe.NginxRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "x64", r.Artifact().Arch) +} + +func TestNginxRecipeBuildRunsGPGVerify(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "nginx-1.25.3-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.NginxRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("nginx", "1.25.3", "https://nginx.org/download/nginx-1.25.3.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should download GPG keys and verify signature. + assert.True(t, hasCallMatching(run.Calls, "wget", "nginx.org/keys"), "should download nginx GPG keys") + assert.True(t, hasCallMatching(run.Calls, "gpg", "--import"), "should import GPG keys") + assert.True(t, hasCallMatching(run.Calls, "gpg", "--verify"), "should verify GPG signature") +} + +func TestNginxRecipeUsesPICFlags(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "nginx-1.25.3-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.NginxRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("nginx", "1.25.3", "https://nginx.org/download/nginx-1.25.3.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // nginx uses PIC flags. + assert.True(t, hasCallMatching(run.Calls, "./configure", "-fPIC"), "nginx configure should use -fPIC") + + // nginx includes dynamic modules. + assert.True(t, hasCallMatching(run.Calls, "./configure", "--with-compat"), "nginx should have --with-compat") + assert.True(t, hasCallMatching(run.Calls, "./configure", "--with-mail=dynamic"), "nginx should have --with-mail=dynamic") +} + +// ── NginxStaticRecipe ───────────────────────────────────────────────────────── + +func TestNginxStaticRecipeName(t *testing.T) { + r := &recipe.NginxStaticRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "nginx-static", r.Name()) +} + +func TestNginxStaticRecipeUsesPIEFlags(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "nginx-static-1.25.3-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.NginxStaticRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("nginx-static", "1.25.3", "https://nginx.org/download/nginx-1.25.3.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // nginx-static uses PIE flags. + assert.True(t, hasCallMatching(run.Calls, "./configure", "-fPIE"), "nginx-static configure should use -fPIE") + + // nginx-static does NOT include the extra dynamic modules. + assert.False(t, hasCallMatching(run.Calls, "./configure", "--with-compat"), "nginx-static must not have --with-compat") + assert.False(t, hasCallMatching(run.Calls, "./configure", "--with-mail=dynamic"), "nginx-static must not have --with-mail=dynamic") +} + +func TestNginxStaticRecipeAlsoRunsGPGVerify(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "nginx-static-1.25.3-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.NginxStaticRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("nginx-static", "1.25.3", "https://nginx.org/download/nginx-1.25.3.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hasCallMatching(run.Calls, "gpg", "--verify"), "nginx-static should also verify GPG signature") +} + +// ── OpenrestyRecipe ─────────────────────────────────────────────────────────── + +func TestOpenrestyRecipeName(t *testing.T) { + r := &recipe.OpenrestyRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "openresty", r.Name()) +} + +func TestOpenrestyRecipeArtifact(t *testing.T) { + r := &recipe.OpenrestyRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "x64", r.Artifact().Arch) +} + +func TestOpenrestyRecipeBuild(t *testing.T) { + f := newFakeFetcher() + r := &recipe.OpenrestyRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("openresty", "1.21.4.3", "https://openresty.org/download/openresty-1.21.4.3.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should download source via Fetcher (not wget). + assert.True(t, hasDownload(f, src.URL), "should download source via Fetcher") + assert.False(t, hasCallMatching(run.Calls, "wget", src.URL), "must not use wget") + assert.False(t, hasCallMatching(run.Calls, "gpg", ""), "openresty must not run gpg verify") + + // Should configure with PIC flags. + assert.True(t, hasCallMatching(run.Calls, "./configure", "-fPIC"), "openresty configure should use -fPIC") + + // Should use -j2 for make. + assert.True(t, hasCallMatching(run.Calls, "make", "-j2"), "openresty should make -j2") +} + +// ── LibunwindRecipe ─────────────────────────────────────────────────────────── + +func TestLibunwindRecipeName(t *testing.T) { + r := &recipe.LibunwindRecipe{} + assert.Equal(t, "libunwind", r.Name()) +} + +func TestLibunwindRecipeArtifact(t *testing.T) { + r := &recipe.LibunwindRecipe{} + assert.Equal(t, "noarch", r.Artifact().Arch) +} + +func TestLibunwindRecipeBuild(t *testing.T) { + r := &recipe.LibunwindRecipe{} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := &source.Input{ + Name: "libunwind", + Version: "1.6.2", + URL: "https://github.com/libunwind/libunwind/releases/download/v1.6.2/libunwind-1.6.2.tar.gz", + SHA256: "abc", + } + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should extract from source/ (pre-downloaded by Concourse). + assert.True(t, hasCallMatching(run.Calls, "tar", "source/libunwind-1.6.2.tar.gz"), "should extract pre-downloaded source tarball") + + // Should run configure, make, make install. + assert.True(t, hasCallMatching(run.Calls, "./configure", "--prefix="), "should configure with prefix") + assert.True(t, hasCallMatching(run.Calls, "make", ""), "should run make") + + // Should pack only include/ and lib/. + assert.True(t, hasCallMatching(run.Calls, "tar", "include"), "artifact should contain include/") + assert.True(t, hasCallMatching(run.Calls, "tar", "lib"), "artifact should contain lib/") +} + +// ── LibgdiplusRecipe ────────────────────────────────────────────────────────── + +func TestLibgdiplusRecipeName(t *testing.T) { + r := &recipe.LibgdiplusRecipe{} + assert.Equal(t, "libgdiplus", r.Name()) +} + +func TestLibgdiplusRecipeArtifact(t *testing.T) { + r := &recipe.LibgdiplusRecipe{} + assert.Equal(t, "noarch", r.Artifact().Arch) +} + +func TestLibgdiplusRecipeBuild(t *testing.T) { + r := &recipe.LibgdiplusRecipe{} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := &source.Input{ + Name: "libgdiplus", + Version: "6.1", + URL: "https://github.com/mono/libgdiplus/releases/tag/6.1", + SHA256: "abc", + Repo: "mono/libgdiplus", + } + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should apt install libgdiplus_build packages. + assert.True(t, hasCallMatching(run.Calls, "apt-get", "automake"), "should install libgdiplus_build packages") + + // Should git clone the repo at the version tag. + assert.True(t, hasCallMatching(run.Calls, "git", "mono/libgdiplus"), "should clone mono/libgdiplus") + assert.True(t, hasCallMatching(run.Calls, "git", "6.1"), "should clone at version tag") + + // Should run autogen with warning suppression flags. + assert.True(t, hasCallWithEnv(run.Calls, "sh", "CFLAGS"), "autogen.sh should have CFLAGS env") + + // Should pack only lib/. + assert.True(t, hasCallMatching(run.Calls, "tar", "lib"), "artifact should contain only lib/") +} + +// ── DepRecipe ───────────────────────────────────────────────────────────────── + +func TestDepRecipeName(t *testing.T) { + r := &recipe.DepRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "dep", r.Name()) +} + +func TestDepRecipeArtifact(t *testing.T) { + r := &recipe.DepRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "x64", r.Artifact().Arch) +} + +func TestDepRecipeBuild(t *testing.T) { + useTempWorkDir(t) + + f := newFakeFetcher() + r := &recipe.DepRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("dep", "0.5.4", "https://github.com/golang/dep/archive/v0.5.4.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, src.URL, f.DownloadedURLs[0].URL) + + // Ruby runs `go get -asmflags -trimpath ./...` via `sh -c "cd {srcDir} && GOPATH=... go get ..."`. + assert.True(t, hasCallMatching(run.Calls, "sh", "go get"), "should run go get via sh -c") + assert.True(t, hasCallMatching(run.Calls, "sh", "-asmflags"), "go get should use -asmflags flag") + assert.True(t, hasCallMatching(run.Calls, "sh", "GOPATH="), "should set GOPATH") + + // Should pack bin/dep + bin/LICENSE. + assert.True(t, hasCallMatching(run.Calls, "tar", "bin/dep"), "artifact should contain bin/dep") + assert.True(t, hasCallMatching(run.Calls, "tar", "bin/LICENSE"), "artifact should contain bin/LICENSE") +} + +// ── GlideRecipe ─────────────────────────────────────────────────────────────── + +func TestGlideRecipeName(t *testing.T) { + r := &recipe.GlideRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "glide", r.Name()) +} + +func TestGlideRecipeBuild(t *testing.T) { + useTempWorkDir(t) + + f := newFakeFetcher() + r := &recipe.GlideRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("glide", "0.13.3", "https://github.com/Masterminds/glide/archive/v0.13.3.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Ruby runs `go build` via `sh -c "cd {srcDir} && GOPATH=/tmp go build"`. + assert.True(t, hasCallMatching(run.Calls, "sh", "go build"), "should run go build via sh -c") + assert.True(t, hasCallMatching(run.Calls, "sh", "GOPATH="), "should set GOPATH") + + // Should pack bin/glide + bin/LICENSE. + assert.True(t, hasCallMatching(run.Calls, "tar", "bin/glide"), "artifact should contain bin/glide") + assert.True(t, hasCallMatching(run.Calls, "tar", "bin/LICENSE"), "artifact should contain bin/LICENSE") +} + +// ── GodepRecipe ─────────────────────────────────────────────────────────────── + +func TestGodepRecipeName(t *testing.T) { + r := &recipe.GodepRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "godep", r.Name()) +} + +func TestGodepRecipeBuild(t *testing.T) { + useTempWorkDir(t) + + f := newFakeFetcher() + r := &recipe.GodepRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("godep", "80", "https://github.com/tools/godep/archive/v80.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Ruby runs `go get ./...` via `sh -c "cd {srcDir} && GOPATH=... go get ./..."`. + assert.True(t, hasCallMatching(run.Calls, "sh", "go get"), "should run go get via sh -c") + assert.True(t, hasCallMatching(run.Calls, "sh", "GOPATH="), "should set GOPATH") + + // Should pack bin/godep + bin/License (capital L, no E — matches Ruby). + assert.True(t, hasCallMatching(run.Calls, "tar", "bin/godep"), "artifact should contain bin/godep") + assert.True(t, hasCallMatching(run.Calls, "tar", "bin/License"), "artifact should contain bin/License (capital L)") +} + +// ── HWCRecipe ───────────────────────────────────────────────────────────────── + +func TestHWCRecipeName(t *testing.T) { + r := &recipe.HWCRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "hwc", r.Name()) +} + +func TestHWCRecipeArtifact(t *testing.T) { + r := &recipe.HWCRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "windows", r.Artifact().OS) + assert.Equal(t, "x86-64", r.Artifact().Arch) + assert.Equal(t, "any-stack", r.Artifact().Stack) +} + +func TestHWCRecipeBuild(t *testing.T) { + f := newFakeFetcher() + r := &recipe.HWCRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("hwc", "2.0.10", "https://github.com/cloudfoundry/hwc/archive/v2.0.10.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should install mingw-w64. + assert.True(t, hasCallMatching(run.Calls, "apt-get", "mingw-w64"), "should install mingw-w64") + + // Should download source. + require.Len(t, f.DownloadedURLs, 1) + + // Should cross-compile with GOOS=windows. + assert.True(t, hasCallWithEnv(run.Calls, "go", "GOOS"), "go build should have GOOS env set") + assert.True(t, hasCallWithEnv(run.Calls, "go", "GOARCH"), "go build should have GOARCH env set") + + // Verify GOOS=windows value. + for _, c := range run.Calls { + if c.Name == "go" && c.Env != nil { + if goos, ok := c.Env["GOOS"]; ok { + assert.Equal(t, "windows", goos) + } + } + } + + // Should produce a .zip (not .tgz) containing BOTH hwc.exe and hwc_x86.exe. + assert.True(t, hasCallMatching(run.Calls, "zip", "hwc.exe"), "hwc artifact should be a zip containing hwc.exe") + assert.True(t, hasCallMatching(run.Calls, "zip", "hwc_x86.exe"), "hwc artifact should be a zip containing hwc_x86.exe (386)") + + // Should build BOTH amd64 and 386. + amd64Found := false + x86Found := false + for _, c := range run.Calls { + if c.Name == "go" && c.Env != nil { + if arch, ok := c.Env["GOARCH"]; ok { + if arch == "amd64" { + amd64Found = true + } + if arch == "386" { + x86Found = true + } + } + } + } + assert.True(t, amd64Found, "should build amd64 binary") + assert.True(t, x86Found, "should build 386 binary") +} + +// ── RRecipe ─────────────────────────────────────────────────────────────────── + +// writeRSubDepFiles creates the stub Concourse source data.json files that +// r.go reads via source.FromFile for each sub-dependency. +// Must be called after useTempWorkDir(t) so the files land in the temp dir. +func writeRSubDepFiles(t *testing.T) { + t.Helper() + subDeps := []struct { + dir string + version string + }{ + {"source-forecast-latest", "8.21.0"}, + {"source-plumber-latest", "1.2.1"}, + {"source-rserve-latest", "1.8.14"}, // will be formatted as 1.8-14 + {"source-shiny-latest", "1.8.0"}, + } + for _, sd := range subDeps { + if err := os.MkdirAll(sd.dir, 0755); err != nil { + t.Fatalf("writeRSubDepFiles: mkdir %s: %v", sd.dir, err) + } + // Use the real depwatcher modern format: + // { "source": {"name": "...", "type": "..."}, "version": {"url": "...", "ref": "...", "sha256": "..."} } + jsonContent := fmt.Sprintf( + `{"source":{"name":%q,"type":"cran"},"version":{"url":"https://cran.r-project.org/fake/%s.tar.gz","ref":%q,"sha256":"abc"}}`, + sd.dir, sd.dir, sd.version, + ) + if err := os.WriteFile(sd.dir+"/data.json", []byte(jsonContent), 0644); err != nil { + t.Fatalf("writeRSubDepFiles: write %s/data.json: %v", sd.dir, err) + } + } +} + +func TestRRecipeName(t *testing.T) { + r := &recipe.RRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "r", r.Name()) +} + +func TestRRecipeArtifact(t *testing.T) { + r := &recipe.RRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "linux", r.Artifact().OS) + assert.Equal(t, "noarch", r.Artifact().Arch) +} + +func TestRRecipeInstallsRemotesBeforePackages(t *testing.T) { + useTempWorkDir(t) + writeRSubDepFiles(t) + + f := newFakeFetcher() + r := &recipe.RRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + s.AptPackages["r_build"] = []string{"r-base"} + src := newInput("r", "4.3.1", "https://cran.r-project.org/src/base/R-4.3.1.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Find indices of remotes install, explicit deps install, and first install_version call. + remotesIdx := -1 + explicitDepsIdx := -1 + installVersionIdx := -1 + for i, c := range run.Calls { + if c.Name == "sh" { + joined := strings.Join(c.Args, " ") + if strings.Contains(joined, `install.packages("remotes"`) && remotesIdx < 0 { + remotesIdx = i + } + if strings.Contains(joined, `"stringr"`) && explicitDepsIdx < 0 { + explicitDepsIdx = i + } + if strings.Contains(joined, "install_version") && installVersionIdx < 0 { + installVersionIdx = i + } + } + } + + require.True(t, remotesIdx >= 0, "remotes install call not found") + require.True(t, explicitDepsIdx >= 0, "explicit deps install call not found") + require.True(t, installVersionIdx >= 0, "install_version call not found") + assert.Less(t, remotesIdx, installVersionIdx, + "remotes must be installed BEFORE any install_version call (remotes at %d, install_version at %d)", + remotesIdx, installVersionIdx) + assert.Less(t, explicitDepsIdx, installVersionIdx, + "explicit deps must be installed BEFORE any install_version call (deps at %d, install_version at %d)", + explicitDepsIdx, installVersionIdx) +} + +func TestRRecipeRemovesRemotes(t *testing.T) { + useTempWorkDir(t) + writeRSubDepFiles(t) + + f := newFakeFetcher() + r := &recipe.RRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + s.AptPackages["r_build"] = []string{"r-base"} + src := newInput("r", "4.3.1", "https://cran.r-project.org/src/base/R-4.3.1.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, hasCallMatching(run.Calls, "sh", `remove.packages("remotes")`), + "should call remove.packages(\"remotes\") after package installs") +} + +func TestRserveVersionFormatting(t *testing.T) { + useTempWorkDir(t) + writeRSubDepFiles(t) + + f := newFakeFetcher() + r := &recipe.RRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + s.AptPackages["r_build"] = []string{"r-base"} + src := newInput("r", "4.3.1", "https://cran.r-project.org/src/base/R-4.3.1.tar.gz") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Rserve version "1.8.14" must be formatted as "1.8-14" in the install_version call. + // The R command uses single quotes (matching Ruby builder style). + assert.True(t, hasCallMatching(run.Calls, "sh", "install_version('Rserve'"), + "should call install_version for Rserve") + assert.True(t, hasCallMatching(run.Calls, "sh", "1.8-14"), + "Rserve version should be formatted as '1.8-14'") + assert.False(t, hasCallMatching(run.Calls, "sh", "1.8.14"), + "Rserve version must not be passed as '1.8.14' (unformatted)") +} + +// ── JRubyRecipe ─────────────────────────────────────────────────────────────── + +func TestJRubyRecipeName(t *testing.T) { + r := &recipe.JRubyRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "jruby", r.Name()) +} + +func TestJRubyRecipeArtifact(t *testing.T) { + r := &recipe.JRubyRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "linux", r.Artifact().OS) + assert.Equal(t, "x64", r.Artifact().Arch) +} + +func TestJRubyRecipeBuild(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "jruby-9.4.5.0-ruby-3.1-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.JRubyRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + + // The recipe globs for jdk*/ inside JDKInstallDir after extracting the JDK + // tarball. FakeRunner doesn't execute commands, so create the subdir manually. + require.NoError(t, os.MkdirAll(filepath.Join(s.Bootstrap.JRuby.InstallDir, "jdk8u452"), 0755)) + + src := newInput("jruby", "9.4.5.0", "https://repo1.maven.org/maven2/org/jruby/jruby-dist/9.4.5.0/jruby-dist-9.4.5.0-src.zip") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // JDK must be downloaded from the stack-configured URL. + assert.True(t, hasDownload(f, s.Bootstrap.JRuby.URL), "should download JDK from stack JRuby.JDKURL") + + // Maven must be downloaded. + assert.True(t, hasDownloadContaining(f, "apache-maven"), "should download Maven") + + // mvn must be invoked inside the correct source directory. + assert.True(t, hasCallMatching(run.Calls, "sh", "cd /tmp/jruby-9.4.5.0"), + "mvn must run inside srcDir") + assert.True(t, hasCallMatching(run.Calls, "sh", "mvn"), + "should invoke mvn") + + // Artifact must encode the full version including Ruby compatibility suffix. + assert.True(t, hasCallMatching(run.Calls, "tar", "9.4.5.0-ruby-3.1"), + "artifact should use full version 9.4.5.0-ruby-3.1") +} + +func TestJRubyRecipeVersion93(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "jruby-9.3.14.0-ruby-2.6-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.JRubyRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + + require.NoError(t, os.MkdirAll(filepath.Join(s.Bootstrap.JRuby.InstallDir, "jdk8u452"), 0755)) + + src := newInput("jruby", "9.3.14.0", "https://repo1.maven.org/maven2/org/jruby/jruby-dist/9.3.14.0/jruby-dist-9.3.14.0-src.zip") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // 9.3.x maps to Ruby 2.6. + assert.True(t, hasCallMatching(run.Calls, "tar", "9.3.14.0-ruby-2.6"), + "JRuby 9.3.x should produce artifact with ruby-2.6") +} + +func TestJRubyRecipeUnknownVersion(t *testing.T) { + f := newFakeFetcher() + r := &recipe.JRubyRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + src := newInput("jruby", "9.9.0.0", "https://repo1.maven.org/maven2/org/jruby/jruby-dist/9.9.0.0/jruby-dist-9.9.0.0-src.zip") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "9.9") +} + +// ── HTTPDRecipe ─────────────────────────────────────────────────────────────── + +func TestHTTPDRecipeName(t *testing.T) { + r := &recipe.HTTPDRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "httpd", r.Name()) +} + +func TestHTTPDRecipeArtifact(t *testing.T) { + r := &recipe.HTTPDRecipe{Fetcher: newFakeFetcher()} + assert.Equal(t, "linux", r.Artifact().OS) + assert.Equal(t, "x64", r.Artifact().Arch) +} + +func TestHTTPDRecipeBuild(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "httpd-2.4.58-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.HTTPDRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + s.AptPackages["httpd_build"] = []string{"libssl-dev", "libpcre3-dev", "libcjose-dev"} + src := newInput("httpd", "2.4.58", "https://archive.apache.org/dist/httpd/httpd-2.4.58.tar.bz2") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // Should apt install httpd_build packages. + assert.True(t, hasCallMatching(run.Calls, "apt-get", "libssl-dev"), + "should apt install httpd_build packages") + + // Should read GitHub release API 3 times (APR, APR-Iconv, APR-Util). + // We verify via BodyMap keys — if the fetcher called ReadBody for those URLs + // the recipe would have gotten valid JSON and proceeded without error. + // Indirectly verified by no error and correct portile configure flags below. + + // Should configure HTTPD with --enable-mods-shared=reallyall. + assert.True(t, hasCallMatching(run.Calls, "./configure", "--enable-mods-shared=reallyall"), + "httpd configure should include --enable-mods-shared=reallyall") + + // Should configure HTTPD with --with-apr= pointing to the APR prefix. + assert.True(t, hasCallMatching(run.Calls, "./configure", "--with-apr="), + "httpd configure should include --with-apr=") + + // mod_auth_openidc configure should set APR_LIBS/APR_CFLAGS via env. + assert.True(t, hasCallWithEnv(run.Calls, "sh", "APR_LIBS"), + "mod_auth_openidc configure should have APR_LIBS env") + assert.True(t, hasCallWithEnv(run.Calls, "sh", "APR_CFLAGS"), + "mod_auth_openidc configure should have APR_CFLAGS env") + + // Artifact should be packed. + assert.True(t, hasCallMatching(run.Calls, "tar", "httpd"), + "should pack httpd artifact") +} + +func TestHTTPDRecipeSetupTar(t *testing.T) { + useTempWorkDir(t) + writeFakeArtifact(t, "httpd-2.4.58-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.HTTPDRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + s.AptPackages["httpd_build"] = []string{"libssl-dev"} + src := newInput("httpd", "2.4.58", "https://archive.apache.org/dist/httpd/httpd-2.4.58.tar.bz2") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // setup_tar should copy APR library. + assert.True(t, hasCallMatching(run.Calls, "cp", "libapr-1.so.0"), + "setup_tar should copy libapr-1.so.0") + + // setup_tar should copy APR-Util library. + assert.True(t, hasCallMatching(run.Calls, "cp", "libaprutil-1.so.0"), + "setup_tar should copy libaprutil-1.so.0") + + // setup_tar should copy APR-Iconv library. + assert.True(t, hasCallMatching(run.Calls, "cp", "libapriconv-1.so.0"), + "setup_tar should copy libapriconv-1.so.0") + + // setup_tar should copy system libs (cjose, hiredis, jansson). + assert.True(t, hasCallMatching(run.Calls, "sh", "libcjose.so"), + "setup_tar should copy libcjose.so*") + assert.True(t, hasCallMatching(run.Calls, "sh", "libhiredis.so"), + "setup_tar should copy libhiredis.so*") + assert.True(t, hasCallMatching(run.Calls, "sh", "libjansson.so"), + "setup_tar should copy libjansson.so*") + + // setup_tar should remove unneeded directories. + assert.True(t, hasCallMatching(run.Calls, "rm", "cgi-bin"), + "setup_tar should remove cgi-bin") + assert.True(t, hasCallMatching(run.Calls, "rm", "manual"), + "setup_tar should remove manual") +} + +func TestHTTPDRecipeVersionsFromStackConfig(t *testing.T) { + // Verify that APR sub-dep versions are read from the stack YAML config and + // that configure calls use the plain version string (no leading 'v' prefix). + useTempWorkDir(t) + writeFakeArtifact(t, "httpd-2.4.58-linux-x64.tgz") + + f := newFakeFetcher() + r := &recipe.HTTPDRecipe{Fetcher: f} + run := runner.NewFakeRunner() + s := newCompiledStack(t) + s.AptPackages["httpd_build"] = []string{"libssl-dev"} + src := newInput("httpd", "2.4.58", "https://archive.apache.org/dist/httpd/httpd-2.4.58.tar.bz2") + + err := r.Build(context.Background(), s, src, run, &output.OutData{}) + require.NoError(t, err) + + // The APR version from stack config must not include a leading 'v' prefix. + assert.False(t, hasCallMatching(run.Calls, "./configure", "apr-v"), + "configure prefix must not include a 'v' prefix") + // A configure call for APR-Util or HTTPD must reference --with-apr. + assert.True(t, hasCallMatching(run.Calls, "./configure", "--with-apr"), + "configure for APR-Util or HTTPD should reference --with-apr") +} + +// ── Artifact naming sanity checks ───────────────────────────────────────────── + +func TestCompiledRecipeArtifactMetaSanity(t *testing.T) { + f := newFakeFetcher() + cases := []struct { + recipe recipe.Recipe + wantOS string + wantArch string + }{ + {&recipe.RubyRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.BundlerRecipe{Fetcher: f}, "linux", "noarch"}, + {&recipe.PythonRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.NodeRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.GoRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.NginxRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.NginxStaticRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.OpenrestyRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.LibunwindRecipe{}, "linux", "noarch"}, + {&recipe.LibgdiplusRecipe{}, "linux", "noarch"}, + {&recipe.DepRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.GlideRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.GodepRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.HWCRecipe{Fetcher: f}, "windows", "x86-64"}, + {&recipe.RRecipe{Fetcher: f}, "linux", "noarch"}, + {&recipe.JRubyRecipe{Fetcher: f}, "linux", "x64"}, + {&recipe.HTTPDRecipe{Fetcher: f}, "linux", "x64"}, + } + + for _, tc := range cases { + t.Run(tc.recipe.Name(), func(t *testing.T) { + meta := tc.recipe.Artifact() + assert.Equal(t, tc.wantOS, meta.OS, "wrong OS for %s", tc.recipe.Name()) + assert.Equal(t, tc.wantArch, meta.Arch, "wrong Arch for %s", tc.recipe.Name()) + }) + } +} diff --git a/internal/recipe/recipe_helpers_test.go b/internal/recipe/recipe_helpers_test.go new file mode 100644 index 00000000..25a4abbd --- /dev/null +++ b/internal/recipe/recipe_helpers_test.go @@ -0,0 +1,158 @@ +package recipe_test + +import ( + "archive/tar" + "compress/gzip" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/runner" +) + +// callNames returns the command name for every recorded call. +func callNames(calls []runner.Call) []string { + names := make([]string, len(calls)) + for i, c := range calls { + names[i] = c.Name + } + return names +} + +// anyCallContains returns true if any call's Name equals name. +func anyCallContains(calls []runner.Call, name string) bool { + for _, c := range calls { + if c.Name == name { + return true + } + } + return false +} + +// anyArgsContain returns true if any call has an argument that contains target. +func anyArgsContain(calls []runner.Call, target string) bool { + for _, c := range calls { + for _, arg := range c.Args { + if strings.Contains(arg, target) { + return true + } + } + } + return false +} + +// hasCallMatching returns true if any call matches name and (optionally) has argSubstr in its joined args. +func hasCallMatching(calls []runner.Call, name string, argSubstr string) bool { + for _, c := range calls { + if c.Name == name { + joined := strings.Join(c.Args, " ") + if argSubstr == "" || strings.Contains(joined, argSubstr) { + return true + } + } + } + return false +} + +// hasDownload returns true if the fetcher recorded a download with the exact URL. +func hasDownload(f *FakeFetcher, url string) bool { + for _, dl := range f.DownloadedURLs { + if dl.URL == url { + return true + } + } + return false +} + +// hasDownloadContaining returns true if any downloaded URL contains substr. +func hasDownloadContaining(f *FakeFetcher, substr string) bool { + for _, dl := range f.DownloadedURLs { + if strings.Contains(dl.URL, substr) { + return true + } + } + return false +} + +// hasCallWithEnv returns true if any call matches name and has envKey in its Env map. +func hasCallWithEnv(calls []runner.Call, name string, envKey string) bool { + for _, c := range calls { + if c.Name == name && c.Env != nil { + if _, ok := c.Env[envKey]; ok { + return true + } + } + } + return false +} + +// useTempWorkDir switches the process working directory to a fresh temp dir for +// the duration of the test and restores it afterwards. It also creates the +// artifacts/ sub-directory so recipes that write relative artifact paths don't +// fail on directory-not-found before they even try to read the file. +// +// NOT safe to use in parallel sub-tests (os.Chdir is process-global). +func useTempWorkDir(t *testing.T) string { + t.Helper() + + orig, err := os.Getwd() + if err != nil { + t.Fatalf("useTempWorkDir: getwd: %v", err) + } + + tmp := t.TempDir() + + if err := os.MkdirAll(filepath.Join(tmp, "artifacts"), 0755); err != nil { + t.Fatalf("useTempWorkDir: mkdir artifacts: %v", err) + } + + if err := os.Chdir(tmp); err != nil { + t.Fatalf("useTempWorkDir: chdir to %s: %v", tmp, err) + } + + t.Cleanup(func() { + _ = os.Chdir(orig) + }) + + return tmp +} + +// writeFakeArtifact creates a minimal valid .tgz at in the current +// working directory. The tarball contains a single dummy file so that +// archive.StripTopLevelDir / StripIncorrectWordsYAML don't fail. +// Recipes write artifacts to mustCwd()/ (CWD root, not artifacts/). +func writeFakeArtifact(t *testing.T, name string) { + t.Helper() + + path := name // write directly into CWD, matching mustCwd() usage in recipes + f, err := os.Create(path) + if err != nil { + t.Fatalf("writeFakeArtifact: create %s: %v", path, err) + } + defer f.Close() + + gw := gzip.NewWriter(f) + tw := tar.NewWriter(gw) + + // Write a single dummy entry so the tarball is valid. + dummy := []byte("dummy") + hdr := &tar.Header{ + Name: "dummy-dir/dummy-file", + Mode: 0644, + Size: int64(len(dummy)), + Typeflag: tar.TypeReg, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("writeFakeArtifact: write header: %v", err) + } + if _, err := tw.Write(dummy); err != nil { + t.Fatalf("writeFakeArtifact: write data: %v", err) + } + if err := tw.Close(); err != nil { + t.Fatalf("writeFakeArtifact: close tar: %v", err) + } + if err := gw.Close(); err != nil { + t.Fatalf("writeFakeArtifact: close gzip: %v", err) + } +} diff --git a/internal/recipe/recipe_test.go b/internal/recipe/recipe_test.go new file mode 100644 index 00000000..6c0307b5 --- /dev/null +++ b/internal/recipe/recipe_test.go @@ -0,0 +1,891 @@ +package recipe_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/recipe" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ── FakeFetcher ────────────────────────────────────────────────────────────── + +// FakeFetcher satisfies fetch.Fetcher without making any network calls. +type FakeFetcher struct { + // DownloadedURLs records every (url, dest) pair passed to Download. + DownloadedURLs []fetchCall + // BodyMap maps URL → body bytes for ReadBody. + BodyMap map[string][]byte + // ErrMap maps URL → error for Download or ReadBody. + ErrMap map[string]error +} + +type fetchCall struct { + URL string + Dest string +} + +func newFakeFetcher() *FakeFetcher { + return &FakeFetcher{ + BodyMap: make(map[string][]byte), + ErrMap: make(map[string]error), + } +} + +func (f *FakeFetcher) Download(_ context.Context, url, dest string, _ source.Checksum) error { + f.DownloadedURLs = append(f.DownloadedURLs, fetchCall{URL: url, Dest: dest}) + if err, ok := f.ErrMap[url]; ok { + return err + } + // Create the destination directory. + if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { + return err + } + // For .tar.gz / .tgz destinations write a minimal valid gzip tarball so + // callers that decompress the file (e.g. archive.StripTopLevelDir) don't + // fail with "invalid gzip header". + if strings.HasSuffix(dest, ".tar.gz") || strings.HasSuffix(dest, ".tgz") { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + // Write one top-level directory entry so StripTopLevelDir has something to strip. + _ = tw.WriteHeader(&tar.Header{Typeflag: tar.TypeDir, Name: "fake-top/", Mode: 0755}) + tw.Close() //nolint:errcheck + gw.Close() //nolint:errcheck + return os.WriteFile(dest, buf.Bytes(), 0644) + } + return os.WriteFile(dest, []byte("fake-content"), 0644) +} + +func (f *FakeFetcher) ReadBody(_ context.Context, url string) ([]byte, error) { + if err, ok := f.ErrMap[url]; ok { + return nil, err + } + if body, ok := f.BodyMap[url]; ok { + return body, nil + } + return []byte("fake-body"), nil +} + +// ── helpers ─────────────────────────────────────────────────────────────────── + +func newStack(t *testing.T) *stack.Stack { + t.Helper() + return &stack.Stack{ + Name: "cflinuxfs4", + AptPackages: map[string][]string{ + "hwc_build": {"mingw-w64"}, + "pip_build": {"python3", "python3-pip"}, + "python_deb_extras": {"libxss1"}, + "python_build": {"libdb-dev", "libgdbm-dev", "tk8.6-dev"}, + "node_build": {}, + }, + Python: stack.PythonConfig{TCLVersion: "8.6"}, + } +} + +func newInput(name, version, url string) *source.Input { + return &source.Input{ + Name: name, + Version: version, + URL: url, + SHA256: "abc123", + } +} + +// ── Registry ───────────────────────────────────────────────────────────────── + +func TestRegistryRegisterAndGet(t *testing.T) { + reg := recipe.NewRegistry() + f := newFakeFetcher() + r := &recipe.PassthroughRecipe{ + DepName: "tomcat", + SourceFilenameFunc: func(v string) string { return fmt.Sprintf("apache-tomcat-%s.tar.gz", v) }, + Meta: recipe.ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"}, + Fetcher: f, + } + reg.Register(r) + + got, err := reg.Get("tomcat") + require.NoError(t, err) + assert.Equal(t, "tomcat", got.Name()) +} + +func TestRegistryGetUnknown(t *testing.T) { + reg := recipe.NewRegistry() + _, err := reg.Get("does-not-exist") + require.Error(t, err) + assert.Contains(t, err.Error(), "does-not-exist") +} + +func TestRegistryNames(t *testing.T) { + reg := recipe.NewRegistry() + f := newFakeFetcher() + for _, name := range []string{"tomcat", "composer", "yarn"} { + reg.Register(&recipe.PassthroughRecipe{ + DepName: name, + SourceFilenameFunc: func(v string) string { return name + ".tgz" }, + Meta: recipe.ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"}, + Fetcher: f, + }) + } + + names := reg.Names() + assert.Len(t, names, 3) + assert.ElementsMatch(t, []string{"tomcat", "composer", "yarn"}, names) +} + +// ── PassthroughRecipe ──────────────────────────────────────────────────────── + +func TestPassthroughRecipeDownloadsWhenMissing(t *testing.T) { + tmpDir := t.TempDir() + // Ensure the "source" dir exists inside our temp dir so the recipe can write to it. + sourceDir := filepath.Join(tmpDir, "source") + require.NoError(t, os.MkdirAll(sourceDir, 0755)) + + // Change to the temp dir so relative paths in the recipe resolve correctly. + origDir, err := os.Getwd() + require.NoError(t, err) + require.NoError(t, os.Chdir(tmpDir)) + defer os.Chdir(origDir) + + f := newFakeFetcher() + r := &recipe.PassthroughRecipe{ + DepName: "tomcat", + SourceFilenameFunc: func(v string) string { return fmt.Sprintf("apache-tomcat-%s.tar.gz", v) }, + Meta: recipe.ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"}, + Fetcher: f, + } + + src := newInput("tomcat", "9.0.85", "https://example.com/tomcat.tar.gz") + err = r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + require.NoError(t, err) + + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, "https://example.com/tomcat.tar.gz", f.DownloadedURLs[0].URL) + assert.Equal(t, filepath.Join("source", "apache-tomcat-9.0.85.tar.gz"), f.DownloadedURLs[0].Dest) +} + +func TestPassthroughRecipeSkipsDownloadIfFileExists(t *testing.T) { + tmpDir := t.TempDir() + sourceDir := filepath.Join(tmpDir, "source") + require.NoError(t, os.MkdirAll(sourceDir, 0755)) + + origDir, err := os.Getwd() + require.NoError(t, err) + require.NoError(t, os.Chdir(tmpDir)) + defer os.Chdir(origDir) + + // Pre-create the file so it already exists. + require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "apache-tomcat-9.0.85.tar.gz"), []byte("data"), 0644)) + + f := newFakeFetcher() + r := &recipe.PassthroughRecipe{ + DepName: "tomcat", + SourceFilenameFunc: func(v string) string { return fmt.Sprintf("apache-tomcat-%s.tar.gz", v) }, + Meta: recipe.ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"}, + Fetcher: f, + } + + src := newInput("tomcat", "9.0.85", "https://example.com/tomcat.tar.gz") + err = r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + require.NoError(t, err) + + assert.Empty(t, f.DownloadedURLs, "should not re-download existing file") +} + +func TestPassthroughRecipeFetchError(t *testing.T) { + tmpDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, "source"), 0755)) + origDir, _ := os.Getwd() + require.NoError(t, os.Chdir(tmpDir)) + defer os.Chdir(origDir) + + f := newFakeFetcher() + f.ErrMap["https://example.com/tomcat.tar.gz"] = errors.New("network failure") + + r := &recipe.PassthroughRecipe{ + DepName: "tomcat", + SourceFilenameFunc: func(v string) string { return fmt.Sprintf("apache-tomcat-%s.tar.gz", v) }, + Meta: recipe.ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"}, + Fetcher: f, + } + + src := newInput("tomcat", "9.0.85", "https://example.com/tomcat.tar.gz") + err := r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "network failure") +} + +// Test the source filename functions for every passthrough recipe. +func TestPassthroughSourceFilenames(t *testing.T) { + f := newFakeFetcher() + recipes := recipe.NewPassthroughRecipes(f) + + cases := []struct { + name string + version string + wantFile string + }{ + {"tomcat", "9.0.85", "apache-tomcat-9.0.85.tar.gz"}, + {"composer", "2.7.1", "composer.phar"}, + {"appdynamics", "23.11.0.35198", "appdynamics-php-agent-linux_x64-23.11.0.35198.tar.bz2"}, + {"appdynamics-java", "23.11.0.35198", "appdynamics-java-agent-23.11.0.35198.zip"}, + {"skywalking-agent", "9.2.0", "apache-skywalking-java-agent-9.2.0.tgz"}, + {"openjdk", "21.0.2+13", "bellsoft-jre21.0.2+13-linux-amd64.tar.gz"}, + {"zulu", "21.0.2", "zulu21.0.2-jre-linux_x64.tar.gz"}, + {"sapmachine", "21.0.2", "sapmachine-jre-21.0.2_linux-x64_bin.tar.gz"}, + {"jprofiler-profiler", "13.0.14", "jprofiler_linux_13_0_14.tar.gz"}, + {"your-kit-profiler", "2023.11.462", "YourKit-JavaProfiler-2023.11.462.zip"}, + } + + recipeMap := make(map[string]recipe.Recipe) + for _, rec := range recipes { + recipeMap[rec.Name()] = rec + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec, ok := recipeMap[tc.name] + require.True(t, ok, "recipe %q not found", tc.name) + + // We need to check the source filename function indirectly by calling Build + // and inspecting what was passed to the fetcher. Use a fresh fetcher. + ff := newFakeFetcher() + pr := rec.(*recipe.PassthroughRecipe) + pr.Fetcher = ff + + tmpDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, "source"), 0755)) + origDir, _ := os.Getwd() + require.NoError(t, os.Chdir(tmpDir)) + defer os.Chdir(origDir) + + src := newInput(tc.name, tc.version, "https://example.com/file") + err := pr.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + require.NoError(t, err) + + require.Len(t, ff.DownloadedURLs, 1) + assert.Equal(t, filepath.Join("source", tc.wantFile), ff.DownloadedURLs[0].Dest) + }) + } +} + +func TestPassthroughArtifactMeta(t *testing.T) { + f := newFakeFetcher() + recipes := recipe.NewPassthroughRecipes(f) + recipeMap := make(map[string]recipe.Recipe) + for _, rec := range recipes { + recipeMap[rec.Name()] = rec + } + + anyStack := []string{"tomcat", "composer", "appdynamics", "appdynamics-java", "skywalking-agent"} + for _, name := range anyStack { + t.Run(name+"_any-stack", func(t *testing.T) { + rec := recipeMap[name] + assert.Equal(t, "any-stack", rec.Artifact().Stack) + assert.Equal(t, "noarch", rec.Artifact().Arch) + }) + } + + stackSpecific := []string{"openjdk", "zulu", "sapmachine", "jprofiler-profiler", "your-kit-profiler"} + for _, name := range stackSpecific { + t.Run(name+"_x64", func(t *testing.T) { + rec := recipeMap[name] + assert.Equal(t, "x64", rec.Artifact().Arch) + }) + } +} + +// ── NewPassthroughRecipes ───────────────────────────────────────────────────── + +// TestNewPassthroughRecipesContents verifies that every expected dep name is +// present. Prefer extending this list over bumping a raw count. +func TestNewPassthroughRecipesContents(t *testing.T) { + f := newFakeFetcher() + recipes := recipe.NewPassthroughRecipes(f) + names := make([]string, len(recipes)) + for i, r := range recipes { + names[i] = r.Name() + } + assert.Subset(t, names, []string{ + "tomcat", "composer", "appdynamics", "appdynamics-java", + "skywalking-agent", "openjdk", "zulu", "sapmachine", + "jprofiler-profiler", "your-kit-profiler", + "setuptools", "flit-core", + }) +} + +// ── BowerRecipe ─────────────────────────────────────────────────────────────── + +func TestBowerRecipeDownloads(t *testing.T) { + f := newFakeFetcher() + r := &recipe.BowerRecipe{Fetcher: f} + + src := newInput("bower", "1.8.14", "https://example.com/bower.tgz") + err := r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + require.NoError(t, err) + + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, "https://example.com/bower.tgz", f.DownloadedURLs[0].URL) + assert.Equal(t, filepath.Join(os.TempDir(), "bower-1.8.14.tgz"), f.DownloadedURLs[0].Dest) +} + +func TestBowerRecipeArtifact(t *testing.T) { + r := &recipe.BowerRecipe{} + assert.Equal(t, "bower", r.Name()) + assert.Equal(t, "noarch", r.Artifact().Arch) + assert.Equal(t, "linux", r.Artifact().OS) +} + +// ── YarnRecipe ──────────────────────────────────────────────────────────────── + +func TestYarnRecipeStripsVPrefix(t *testing.T) { + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + src := newInput("yarn", "v1.22.22", "https://example.com/yarn.tgz") + r := &recipe.YarnRecipe{Fetcher: f} + outData := &output.OutData{} + _ = r.Build(context.Background(), newStack(t), src, fakeRunner, outData) + + require.Len(t, f.DownloadedURLs, 1) + // File on disk uses the stripped version. + assert.Equal(t, filepath.Join(os.TempDir(), "yarn-1.22.22.tgz"), f.DownloadedURLs[0].Dest) + // outData.Version must be the stripped version so findIntermediateArtifact matches. + assert.Equal(t, "1.22.22", outData.Version) + // src.Version must NOT be mutated — callers after Build rely on the original value. + assert.Equal(t, "v1.22.22", src.Version) +} + +func TestYarnRecipeNameAndArtifact(t *testing.T) { + r := &recipe.YarnRecipe{} + assert.Equal(t, "yarn", r.Name()) + assert.Equal(t, "noarch", r.Artifact().Arch) +} + +// ── PyPISourceRecipe ────────────────────────────────────────────────────────── + +func TestPyPISourceRecipeFilenameFromURL(t *testing.T) { + cases := []struct { + depName string + version string + url string + wantDst string + }{ + { + depName: "setuptools", + version: "69.0.3", + url: "https://example.com/setuptools-69.0.3.tar.gz", + wantDst: filepath.Join(os.TempDir(), "setuptools-69.0.3.tar.gz"), + }, + { + depName: "flit-core", + version: "3.9.0", + url: "https://example.com/flit_core-3.9.0.tar.gz", + wantDst: filepath.Join(os.TempDir(), "flit_core-3.9.0.tar.gz"), + }, + } + for _, tc := range cases { + t.Run(tc.depName, func(t *testing.T) { + f := newFakeFetcher() + r := &recipe.PyPISourceRecipe{DepName: tc.depName, Fetcher: f} + src := newInput(tc.depName, tc.version, tc.url) + _ = r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, tc.wantDst, f.DownloadedURLs[0].Dest) + }) + } +} + +func TestPyPISourceRecipeZipURL(t *testing.T) { + f := newFakeFetcher() + + src := newInput("setuptools", "69.0.3", "https://example.com/setuptools-69.0.3.zip") + r := &recipe.PyPISourceRecipe{DepName: "setuptools", Fetcher: f} + _ = r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, filepath.Join(os.TempDir(), "setuptools-69.0.3.zip"), f.DownloadedURLs[0].Dest) +} + +func TestPyPISourceRecipeNameAndArtifact(t *testing.T) { + cases := []struct{ depName string }{ + {"setuptools"}, + {"flit-core"}, + } + for _, tc := range cases { + t.Run(tc.depName, func(t *testing.T) { + r := &recipe.PyPISourceRecipe{DepName: tc.depName} + assert.Equal(t, tc.depName, r.Name()) + assert.Equal(t, "noarch", r.Artifact().Arch) + assert.Equal(t, "linux", r.Artifact().OS) + }) + } +} + +func TestPyPISourceRecipeStripsURLFragment(t *testing.T) { + // PyPI JSON API URLs sometimes include a #sha256=… fragment; the local + // filename must not contain the fragment. + f := newFakeFetcher() + r := &recipe.PyPISourceRecipe{DepName: "flit-core", Fetcher: f} + src := newInput("flit-core", "3.9.0", "https://files.pythonhosted.org/packages/flit_core-3.9.0.tar.gz#sha256=abc123") + _ = r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, filepath.Join(os.TempDir(), "flit_core-3.9.0.tar.gz"), f.DownloadedURLs[0].Dest, + "fragment must be stripped from destination filename") +} + +// ── RubygemsRecipe ──────────────────────────────────────────────────────────── + +func TestRubygemsRecipeDownloads(t *testing.T) { + f := newFakeFetcher() + + src := newInput("rubygems", "3.5.6", "https://example.com/rubygems.tgz") + r := &recipe.RubygemsRecipe{Fetcher: f} + _ = r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, "https://example.com/rubygems.tgz", f.DownloadedURLs[0].URL) + assert.Equal(t, filepath.Join(os.TempDir(), "rubygems-3.5.6.tgz"), f.DownloadedURLs[0].Dest) +} + +func TestRubygemsRecipeNameAndArtifact(t *testing.T) { + r := &recipe.RubygemsRecipe{} + assert.Equal(t, "rubygems", r.Name()) + assert.Equal(t, "noarch", r.Artifact().Arch) +} + +// ── MinicondaRecipe ─────────────────────────────────────────────────────────── + +func TestMinicondaRecipeSetsOutData(t *testing.T) { + body := []byte("#!/bin/bash\necho miniconda installer") + expectedSHA := fmt.Sprintf("%x", sha256.Sum256(body)) + + f := newFakeFetcher() + f.BodyMap["https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh"] = body + + r := &recipe.MinicondaRecipe{Fetcher: f} + src := newInput("miniconda3-py39", "py39_4.12.0", "https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-Linux-x86_64.sh") + outData := &output.OutData{} + + err := r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), outData) + require.NoError(t, err) + + assert.Equal(t, src.URL, outData.URL) + assert.Equal(t, expectedSHA, outData.SHA256) +} + +func TestMinicondaRecipeNoFileDownloaded(t *testing.T) { + f := newFakeFetcher() + r := &recipe.MinicondaRecipe{Fetcher: f} + + src := newInput("miniconda3-py39", "py39_4.12.0", "https://repo.anaconda.com/miniconda/installer.sh") + err := r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + require.NoError(t, err) + + // No Download calls — miniconda uses ReadBody, not Download. + assert.Empty(t, f.DownloadedURLs) +} + +func TestMinicondaRecipeFetchError(t *testing.T) { + f := newFakeFetcher() + f.ErrMap["https://repo.anaconda.com/miniconda/installer.sh"] = errors.New("timeout") + + r := &recipe.MinicondaRecipe{Fetcher: f} + src := newInput("miniconda3-py39", "py39_4.12.0", "https://repo.anaconda.com/miniconda/installer.sh") + err := r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), &output.OutData{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "timeout") +} + +func TestMinicondaRecipeArtifact(t *testing.T) { + r := &recipe.MinicondaRecipe{} + assert.Equal(t, "miniconda3-py39", r.Name()) + assert.Equal(t, "any-stack", r.Artifact().Stack) + assert.Equal(t, "noarch", r.Artifact().Arch) +} + +// ── PipRecipe ───────────────────────────────────────────────────────────────── + +func TestPipRecipeCallSequence(t *testing.T) { + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + r := &recipe.PipRecipe{Fetcher: f} + src := newInput("pip", "24.0", "https://example.com/pip.tgz") + err := r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + require.NoError(t, err) + + // Verify apt-get update and install were called. + names := callNames(fakeRunner.Calls) + assert.Contains(t, names, "apt-get") + + // Verify pip3 was invoked. + assert.True(t, anyCallContains(fakeRunner.Calls, "pip3"), "pip3 should be called") + // Verify tar was called to bundle. + assert.True(t, anyCallContains(fakeRunner.Calls, "tar"), "tar should be called") +} + +func TestPipRecipeCVEWheelPin(t *testing.T) { + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + r := &recipe.PipRecipe{Fetcher: f} + src := newInput("pip", "24.0", "https://example.com/pip.tgz") + _ = r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + + // Verify wheel>=0.46.2 pin is present somewhere in the call args. + assert.True(t, anyArgsContain(fakeRunner.Calls, "wheel>=0.46.2"), + "CVE-2026-24049 wheel pin must be present") +} + +func TestPipRecipeOutputPath(t *testing.T) { + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + r := &recipe.PipRecipe{Fetcher: f} + src := newInput("pip", "24.0", "https://example.com/pip.tgz") + _ = r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + + // The fetcher downloads pip source into the build tmpDir, not the final output path. + require.Len(t, f.DownloadedURLs, 1) + assert.Equal(t, "/tmp/pip-build-24.0/pip-24.0.tar.gz", f.DownloadedURLs[0].Dest) +} + +func TestPipRecipeNameAndArtifact(t *testing.T) { + r := &recipe.PipRecipe{} + assert.Equal(t, "pip", r.Name()) + assert.Equal(t, "noarch", r.Artifact().Arch) +} + +// ── PipenvRecipe ────────────────────────────────────────────────────────────── + +func TestPipenvRecipeCallSequence(t *testing.T) { + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + r := &recipe.PipenvRecipe{Fetcher: f} + src := newInput("pipenv", "2023.12.1", "https://example.com/pipenv.tgz") + err := r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, anyCallContains(fakeRunner.Calls, "pip3"), "pip3 should be called") + assert.True(t, anyCallContains(fakeRunner.Calls, "tar"), "tar should be called") +} + +func TestPipenvRecipeOutputPathHasVPrefix(t *testing.T) { + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + r := &recipe.PipenvRecipe{Fetcher: f} + src := newInput("pipenv", "2023.12.1", "https://example.com/pipenv.tgz") + _ = r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + + // Output tarball must have 'v' prefix: /tmp/pipenv-v{version}.tgz + assert.True(t, anyArgsContain(fakeRunner.Calls, "/tmp/pipenv-v2023.12.1.tgz"), + "pipenv output path must have v prefix") +} + +func TestPipenvRecipeBundledDeps(t *testing.T) { + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + r := &recipe.PipenvRecipe{Fetcher: f} + src := newInput("pipenv", "2023.12.1", "https://example.com/pipenv.tgz") + _ = r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + + // All 7 bundled packages must be downloaded. + expectedDeps := []string{ + "pytest-runner", "setuptools_scm", "parver", "wheel>=0.46.2", + "invoke", "flit_core", "hatch-vcs", + } + for _, dep := range expectedDeps { + assert.True(t, anyArgsContain(fakeRunner.Calls, dep), + "expected bundled dep %q to appear in runner calls", dep) + } +} + +func TestPipenvRecipeCVEWheelPin(t *testing.T) { + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + r := &recipe.PipenvRecipe{Fetcher: f} + src := newInput("pipenv", "2023.12.1", "https://example.com/pipenv.tgz") + _ = r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + + assert.True(t, anyArgsContain(fakeRunner.Calls, "wheel>=0.46.2"), + "CVE-2026-24049 wheel pin must be present in pipenv build") +} + +func TestPipenvRecipeNameAndArtifact(t *testing.T) { + r := &recipe.PipenvRecipe{} + assert.Equal(t, "pipenv", r.Name()) + assert.Equal(t, "noarch", r.Artifact().Arch) +} + +// ── HWCRecipe ───────────────────────────────────────────────────────────────── + +func TestHWCRecipeInstallsFromStackConfig(t *testing.T) { + useTempWorkDir(t) + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + s := &stack.Stack{ + Name: "cflinuxfs4", + AptPackages: map[string][]string{"hwc_build": {"mingw-w64"}}, + } + src := newInput("hwc", "2.9.0", "https://example.com/hwc.tgz") + r := &recipe.HWCRecipe{Fetcher: f} + _ = r.Build(context.Background(), s, src, fakeRunner, &output.OutData{}) + + // mingw-w64 must be installed via apt-get from the stack config, not hardcoded. + assert.True(t, hasCallMatching(fakeRunner.Calls, "apt-get", "mingw-w64"), + "hwc_build apt package 'mingw-w64' must be installed from stack config") +} + +func TestHWCRecipeUsesStackAptPackages(t *testing.T) { + // Verify that a custom hwc_build list is honoured — the recipe must not + // hardcode the package name. + useTempWorkDir(t) + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + s := &stack.Stack{ + Name: "future-stack", + AptPackages: map[string][]string{"hwc_build": {"mingw-w64-custom"}}, + } + src := newInput("hwc", "2.9.0", "https://example.com/hwc.tgz") + r := &recipe.HWCRecipe{Fetcher: f} + _ = r.Build(context.Background(), s, src, fakeRunner, &output.OutData{}) + + assert.True(t, hasCallMatching(fakeRunner.Calls, "apt-get", "mingw-w64-custom"), + "recipe must use hwc_build packages from stack config, not a hardcoded value") + assert.False(t, hasCallMatching(fakeRunner.Calls, "apt-get", "mingw-w64\x00"), + "hardcoded 'mingw-w64' (without suffix) must not appear when stack config overrides it") +} + +// ── PipRecipe / pip_build config ────────────────────────────────────────────── + +func TestPipRecipeInstallsFromStackConfig(t *testing.T) { + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + s := &stack.Stack{ + Name: "cflinuxfs4", + AptPackages: map[string][]string{"pip_build": {"python3", "python3-pip"}}, + } + src := newInput("pip", "24.0", "https://example.com/pip.tgz") + r := &recipe.PipRecipe{Fetcher: f} + _ = r.Build(context.Background(), s, src, fakeRunner, &output.OutData{}) + + // python3 and python3-pip must come from stack config. + assert.True(t, hasCallMatching(fakeRunner.Calls, "apt-get", "python3"), + "pip_build package 'python3' must be installed from stack config") + assert.True(t, hasCallMatching(fakeRunner.Calls, "apt-get", "python3-pip"), + "pip_build package 'python3-pip' must be installed from stack config") +} + +func TestPipRecipeUsesStackPipBuildPackages(t *testing.T) { + // Verify a custom pip_build list is honoured. + f := newFakeFetcher() + fakeRunner := runner.NewFakeRunner() + + s := &stack.Stack{ + Name: "future-stack", + AptPackages: map[string][]string{"pip_build": {"python3.12", "python3.12-pip"}}, + } + src := newInput("pip", "24.0", "https://example.com/pip.tgz") + r := &recipe.PipRecipe{Fetcher: f} + _ = r.Build(context.Background(), s, src, fakeRunner, &output.OutData{}) + + assert.True(t, hasCallMatching(fakeRunner.Calls, "apt-get", "python3.12"), + "recipe must honour custom pip_build packages from stack config") +} + +// ── DotnetSDKRecipe ─────────────────────────────────────────────────────────── + +// writeFakeDotnetSource creates a source/ directory with a minimal .tar.gz file +// so that filepath.Glob("source/*.tar.gz") in pruneDotnetFiles resolves correctly. +// Returns the resolved path (e.g. "source/dotnet-sdk-8.0.101.tar.gz"). +func writeFakeDotnetSource(t *testing.T, filename string) string { + t.Helper() + if err := os.MkdirAll("source", 0755); err != nil { + t.Fatalf("writeFakeDotnetSource: mkdir source: %v", err) + } + srcPath := filepath.Join("source", filename) + if err := os.WriteFile(srcPath, []byte("fake-dotnet-tarball"), 0644); err != nil { + t.Fatalf("writeFakeDotnetSource: write %s: %v", srcPath, err) + } + return srcPath +} + +func TestDotnetSDKRecipeCallSequence(t *testing.T) { + useTempWorkDir(t) + srcPath := writeFakeDotnetSource(t, "dotnet-sdk-8.0.101.tar.gz") + + fakeRunner := runner.NewFakeRunner() + // Provide output for the tar tf command using the resolved path. + fakeRunner.OutputMap["tar tf "+srcPath+" ./shared/Microsoft.NETCore.App/"] = + "./shared/Microsoft.NETCore.App/\n./shared/Microsoft.NETCore.App/8.0.1/\n" + + r := &recipe.DotnetSDKRecipe{} + src := newInput("dotnet-sdk", "8.0.101", "https://example.com/dotnet-sdk.tar.gz") + err := r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + require.NoError(t, err) + + // mkdir, tar extract, tar tf (runtime version), tar compress. + // RuntimeVersion.txt is written via os.WriteFile (no "sh" call). + names := callNames(fakeRunner.Calls) + assert.Contains(t, names, "mkdir") + assert.Contains(t, names, "tar") + assert.NotContains(t, names, "sh", "RuntimeVersion.txt must not use sh; written via os.WriteFile") +} + +func TestDotnetSDKRecipeExcludesSharedDir(t *testing.T) { + useTempWorkDir(t) + srcPath := writeFakeDotnetSource(t, "dotnet-sdk-8.0.101.tar.gz") + + fakeRunner := runner.NewFakeRunner() + fakeRunner.OutputMap["tar tf "+srcPath+" ./shared/Microsoft.NETCore.App/"] = "" + + r := &recipe.DotnetSDKRecipe{} + src := newInput("dotnet-sdk", "8.0.101", "https://example.com/dotnet-sdk.tar.gz") + _ = r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + + // Verify --exclude=./shared/* appears in the tar extract call. + assert.True(t, anyArgsContain(fakeRunner.Calls, "--exclude=./shared/*"), + "dotnet-sdk must exclude ./shared/*") +} + +func TestDotnetSDKRecipeUsesXZCompression(t *testing.T) { + useTempWorkDir(t) + srcPath := writeFakeDotnetSource(t, "dotnet-sdk-8.0.101.tar.gz") + + fakeRunner := runner.NewFakeRunner() + fakeRunner.OutputMap["tar tf "+srcPath+" ./shared/Microsoft.NETCore.App/"] = + "./shared/Microsoft.NETCore.App/\n./shared/Microsoft.NETCore.App/8.0.1/\n" + + r := &recipe.DotnetSDKRecipe{} + src := newInput("dotnet-sdk", "8.0.101", "https://example.com/dotnet-sdk.tar.gz") + _ = r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + + // Re-compression must use -Jcf (xz), not -czf (gzip). + assert.True(t, anyArgsContain(fakeRunner.Calls, "-Jcf"), + "dotnet-sdk must use xz compression (-Jcf)") +} + +func TestDotnetSDKRecipeNameAndArtifact(t *testing.T) { + r := &recipe.DotnetSDKRecipe{} + assert.Equal(t, "dotnet-sdk", r.Name()) + assert.Equal(t, "x64", r.Artifact().Arch) +} + +// ── DotnetRuntimeRecipe ─────────────────────────────────────────────────────── + +func TestDotnetRuntimeRecipeExcludesDotnet(t *testing.T) { + useTempWorkDir(t) + writeFakeDotnetSource(t, "dotnet-runtime-8.0.1.tar.gz") + fakeRunner := runner.NewFakeRunner() + + r := &recipe.DotnetRuntimeRecipe{} + src := newInput("dotnet-runtime", "8.0.1", "https://example.com/dotnet-runtime.tar.gz") + err := r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, anyArgsContain(fakeRunner.Calls, "--exclude=./dotnet"), + "dotnet-runtime must exclude ./dotnet") +} + +func TestDotnetRuntimeRecipeNoRuntimeVersionTxt(t *testing.T) { + fakeRunner := runner.NewFakeRunner() + + r := &recipe.DotnetRuntimeRecipe{} + src := newInput("dotnet-runtime", "8.0.1", "https://example.com/dotnet-runtime.tar.gz") + _ = r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + + // No "sh" call: RuntimeVersion.txt must NOT be written for dotnet-runtime. + assert.False(t, anyCallContains(fakeRunner.Calls, "sh"), + "dotnet-runtime must NOT write RuntimeVersion.txt") +} + +func TestDotnetRuntimeRecipeNameAndArtifact(t *testing.T) { + r := &recipe.DotnetRuntimeRecipe{} + assert.Equal(t, "dotnet-runtime", r.Name()) + assert.Equal(t, "x64", r.Artifact().Arch) +} + +// ── DotnetAspnetcoreRecipe ──────────────────────────────────────────────────── + +func TestDotnetAspnetcoreRecipeExcludesBoth(t *testing.T) { + useTempWorkDir(t) + writeFakeDotnetSource(t, "dotnet-aspnetcore-8.0.1.tar.gz") + fakeRunner := runner.NewFakeRunner() + + r := &recipe.DotnetAspnetcoreRecipe{} + src := newInput("dotnet-aspnetcore", "8.0.1", "https://example.com/dotnet-aspnetcore.tar.gz") + err := r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + require.NoError(t, err) + + assert.True(t, anyArgsContain(fakeRunner.Calls, "--exclude=./dotnet"), + "dotnet-aspnetcore must exclude ./dotnet") + assert.True(t, anyArgsContain(fakeRunner.Calls, "--exclude=./shared/Microsoft.NETCore.App"), + "dotnet-aspnetcore must exclude ./shared/Microsoft.NETCore.App") +} + +func TestDotnetAspnetcoreRecipeNoRuntimeVersionTxt(t *testing.T) { + fakeRunner := runner.NewFakeRunner() + + r := &recipe.DotnetAspnetcoreRecipe{} + src := newInput("dotnet-aspnetcore", "8.0.1", "https://example.com/dotnet-aspnetcore.tar.gz") + _ = r.Build(context.Background(), newStack(t), src, fakeRunner, &output.OutData{}) + + assert.False(t, anyCallContains(fakeRunner.Calls, "sh"), + "dotnet-aspnetcore must NOT write RuntimeVersion.txt") +} + +func TestDotnetAspnetcoreRecipeNameAndArtifact(t *testing.T) { + r := &recipe.DotnetAspnetcoreRecipe{} + assert.Equal(t, "dotnet-aspnetcore", r.Name()) + assert.Equal(t, "x64", r.Artifact().Arch) +} + +// ── computeSHA256 helper (via MinicondaRecipe) ──────────────────────────────── + +func TestComputeSHA256Determinism(t *testing.T) { + // We test computeSHA256 indirectly via MinicondaRecipe which is the only + // consumer. Same body → same SHA256 on every call. + body := []byte("deterministic content") + expected := fmt.Sprintf("%x", sha256.Sum256(body)) + + f := newFakeFetcher() + f.BodyMap["https://example.com/installer.sh"] = body + + r := &recipe.MinicondaRecipe{Fetcher: f} + src := newInput("miniconda3-py39", "py39_4.12.0", "https://example.com/installer.sh") + + for i := 0; i < 3; i++ { + outData := &output.OutData{} + err := r.Build(context.Background(), newStack(t), src, runner.NewFakeRunner(), outData) + require.NoError(t, err) + assert.Equal(t, expected, outData.SHA256, "SHA256 must be deterministic (run %d)", i+1) + } +} + +// ── test helpers are in recipe_helpers_test.go ──────────────────────────────── diff --git a/internal/recipe/repack.go b/internal/recipe/repack.go new file mode 100644 index 00000000..b198971e --- /dev/null +++ b/internal/recipe/repack.go @@ -0,0 +1,89 @@ +package recipe + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// RepackRecipe downloads an upstream archive and optionally transforms it. +// It covers recipes that follow the pattern: +// +// 1. Infer or compute a local destination filename +// 2. Fetcher.Download(ctx, src.URL, dest, checksum) +// 3. [optional] archive.StripTopLevelDir(dest) — for .tar.gz / .tgz +// or archive.StripTopLevelDirFromZip(dest) — for .zip +// 4. [optional] update outData.Version with the stripped version +type RepackRecipe struct { + DepName string + Meta ArtifactMeta + Fetcher fetch.Fetcher + // StripTopLevelDir strips the top-level directory from the archive after download. + // For .zip files the zip-specific stripper is used automatically. + StripTopLevelDir bool + // StripVersionPrefix strips this prefix from src.Version before building the + // destination filename and writing outData.Version (e.g. "v" for yarn). + StripVersionPrefix string + // DestFilename derives the local destination filename from version and URL. + // If nil, the default is "-.". + // PyPI sdist recipes use this to infer the filename from the URL's last path segment. + DestFilename func(version, url string) string +} + +func (r *RepackRecipe) Name() string { return r.DepName } +func (r *RepackRecipe) Artifact() ArtifactMeta { return r.Meta } + +func (r *RepackRecipe) Build(ctx context.Context, _ *stack.Stack, src *source.Input, _ runner.Runner, outData *output.OutData) error { + version := strings.TrimPrefix(src.Version, r.StripVersionPrefix) + if r.StripVersionPrefix != "" { + outData.Version = version + } + + var dest string + if r.DestFilename != nil { + dest = filepath.Join(os.TempDir(), r.DestFilename(version, src.URL)) + } else { + ext := inferExt(src.URL) + dest = filepath.Join(os.TempDir(), fmt.Sprintf("%s-%s%s", r.DepName, version, ext)) + } + + if err := r.Fetcher.Download(ctx, src.URL, dest, src.PrimaryChecksum()); err != nil { + return fmt.Errorf("downloading %s: %w", r.DepName, err) + } + + if !r.StripTopLevelDir { + return nil + } + + // Use dest (already fragment-free) rather than src.URL to detect zip archives. + // PyPI download URLs may contain a #sha256=… fragment that would fool a + // suffix check on the raw URL. + if strings.HasSuffix(dest, ".zip") { + return archive.StripTopLevelDirFromZip(dest) + } + return archive.StripTopLevelDir(dest) +} + +// inferExt returns the file extension for a download URL, recognising .tar.gz +// as a two-part extension and falling back to the last path segment's suffix. +func inferExt(url string) string { + if strings.HasSuffix(url, ".tar.gz") { + return ".tar.gz" + } + parts := strings.Split(url, "/") + last := parts[len(parts)-1] + idx := strings.LastIndex(last, ".") + if idx < 0 { + return "" + } + return last[idx:] +} diff --git a/internal/recipe/ruby.go b/internal/recipe/ruby.go new file mode 100644 index 00000000..e76cc22c --- /dev/null +++ b/internal/recipe/ruby.go @@ -0,0 +1,82 @@ +package recipe + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/cloudfoundry/binary-builder/internal/apt" + "github.com/cloudfoundry/binary-builder/internal/archive" + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/portile" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// RubyRecipe builds Ruby via portile (configure/make/install) and strips +// incorrect_words.yaml from the resulting tarball. +type RubyRecipe struct { + Fetcher fetch.Fetcher +} + +func (r *RubyRecipe) Name() string { return "ruby" } +func (r *RubyRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "x64", Stack: ""} +} + +func (r *RubyRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, run runner.Runner, _ *output.OutData) error { + a := apt.New(run) + + // Install ruby build dependencies from stack config. + if err := a.Install(ctx, s.AptPackages["ruby_build"]...); err != nil { + return fmt.Errorf("ruby: apt install ruby_build: %w", err) + } + + builtPath := fmt.Sprintf("/app/vendor/ruby-%s", src.Version) + artifactPath := filepath.Join(mustCwd(), fmt.Sprintf("ruby-%s-linux-x64.tgz", src.Version)) + + p := &portile.Portile{ + Name: "ruby", + Version: src.Version, + URL: src.URL, + Checksum: src.PrimaryChecksum(), + Prefix: builtPath, + Options: []string{ + "--enable-load-relative", + "--disable-install-doc", + "--without-gmp", + }, + Runner: run, + Fetcher: r.Fetcher, + } + + if err := p.Cook(ctx); err != nil { + return fmt.Errorf("ruby: portile cook: %w", err) + } + + // Pack the installed tree flat (no top-level dir), matching Ruby builder's + // ArchiveRecipe#compress! which copies archive_files into tmpdir/ directly + // and tars from there, so archive root contains bin/, lib/, etc. + if err := run.Run("tar", "czf", artifactPath, "-C", builtPath, "."); err != nil { + return fmt.Errorf("ruby: packing artifact: %w", err) + } + + // Inject sources.yml into the artifact tarball at the archive root, matching + // Ruby's ArchiveRecipe#compress! which writes YAMLPresenter output into the + // tmpdir before running tar (alongside the archive_files). + // src.SHA256 is the sha256 of the downloaded source tarball, matching what + // YAMLPresenter computes via Digest::SHA256.file(local_path).hexdigest. + sourcesContent := buildSourcesYAML([]SourceEntry{{URL: src.URL, SHA256: src.SHA256}}) + if err := archive.InjectFile(artifactPath, "sources.yml", sourcesContent); err != nil { + return fmt.Errorf("ruby: injecting sources.yml: %w", err) + } + + // Remove incorrect_words.yaml from the tarball and any nested jars. + if err := archive.StripIncorrectWordsYAML(artifactPath); err != nil { + return fmt.Errorf("ruby: stripping incorrect_words.yaml: %w", err) + } + + return nil +} diff --git a/internal/recipe/simple.go b/internal/recipe/simple.go new file mode 100644 index 00000000..61274d20 --- /dev/null +++ b/internal/recipe/simple.go @@ -0,0 +1,129 @@ +package recipe + +import ( + "context" + "fmt" + "net/url" + "path" + "strings" + + "github.com/cloudfoundry/binary-builder/internal/fetch" + "github.com/cloudfoundry/binary-builder/internal/output" + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/cloudfoundry/binary-builder/internal/stack" +) + +// BowerRecipe downloads an npm tarball directly — simplest possible recipe. +type BowerRecipe struct { + Fetcher fetch.Fetcher +} + +func (b *BowerRecipe) Name() string { return "bower" } +func (b *BowerRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: ""} +} +func (b *BowerRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, r runner.Runner, out *output.OutData) error { + return (&RepackRecipe{ + DepName: "bower", + Meta: ArtifactMeta{OS: "linux", Arch: "noarch"}, + Fetcher: b.Fetcher, + }).Build(ctx, s, src, r, out) +} + +// YarnRecipe downloads yarn, strips 'v' prefix from version, strips top-level dir. +type YarnRecipe struct { + Fetcher fetch.Fetcher +} + +func (y *YarnRecipe) Name() string { return "yarn" } +func (y *YarnRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: ""} +} +func (y *YarnRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, r runner.Runner, out *output.OutData) error { + return (&RepackRecipe{ + DepName: "yarn", + Meta: ArtifactMeta{OS: "linux", Arch: "noarch"}, + Fetcher: y.Fetcher, + StripTopLevelDir: true, + StripVersionPrefix: "v", + }).Build(ctx, s, src, r, out) +} + +// PyPISourceRecipe downloads a PyPI source tarball and strips its top-level +// directory. It covers any dep published as a plain sdist on PyPI (e.g. +// setuptools, flit-core) where the artifact filename is the last path segment +// of the download URL and no compilation step is required. +type PyPISourceRecipe struct { + DepName string + Fetcher fetch.Fetcher +} + +func (p *PyPISourceRecipe) Name() string { return p.DepName } +func (p *PyPISourceRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: ""} +} +func (p *PyPISourceRecipe) Build(ctx context.Context, stk *stack.Stack, src *source.Input, r runner.Runner, out *output.OutData) error { + return (&RepackRecipe{ + DepName: p.DepName, + Meta: ArtifactMeta{OS: "linux", Arch: "noarch"}, + Fetcher: p.Fetcher, + StripTopLevelDir: true, + // PyPI sdist URLs end with the canonical filename (e.g. setuptools-69.0.3.tar.gz). + // Use url.Parse + path.Base to strip any query string or fragment before + // using the last path segment as the local filename. + DestFilename: func(_, rawURL string) string { + if u, err := url.Parse(rawURL); err == nil { + return path.Base(u.Path) + } + // Fallback: should not happen for well-formed URLs. + parts := strings.Split(rawURL, "/") + return parts[len(parts)-1] + }, + }).Build(ctx, stk, src, r, out) +} + +// RubygemsRecipe downloads rubygems and strips top-level dir. +type RubygemsRecipe struct { + Fetcher fetch.Fetcher +} + +func (rg *RubygemsRecipe) Name() string { return "rubygems" } +func (rg *RubygemsRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: ""} +} +func (rg *RubygemsRecipe) Build(ctx context.Context, s *stack.Stack, src *source.Input, r runner.Runner, out *output.OutData) error { + return (&RepackRecipe{ + DepName: "rubygems", + Meta: ArtifactMeta{OS: "linux", Arch: "noarch"}, + Fetcher: rg.Fetcher, + StripTopLevelDir: true, + }).Build(ctx, s, src, r, out) +} + +// MinicondaRecipe is a URL passthrough — no file produced, just sets outData. +type MinicondaRecipe struct { + Fetcher fetch.Fetcher +} + +func (m *MinicondaRecipe) Name() string { return "miniconda3-py39" } +func (m *MinicondaRecipe) Artifact() ArtifactMeta { + return ArtifactMeta{OS: "linux", Arch: "noarch", Stack: "any-stack"} +} + +func (m *MinicondaRecipe) Build(ctx context.Context, _ *stack.Stack, src *source.Input, r runner.Runner, outData *output.OutData) error { + // Miniconda is special: no file produced. We just verify the URL body + // and set outData.URL + outData.SHA256 directly. + body, err := m.Fetcher.ReadBody(ctx, src.URL) + if err != nil { + return fmt.Errorf("reading miniconda URL: %w", err) + } + + // Compute SHA256 of the body. + sha256 := computeSHA256(body) + + outData.URL = src.URL + outData.SHA256 = sha256 + + return nil +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go new file mode 100644 index 00000000..03b0f14d --- /dev/null +++ b/internal/runner/runner.go @@ -0,0 +1,186 @@ +// Package runner provides an interface for executing system commands, +// with a real implementation for production and a fake for testing. +package runner + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +// Runner is the interface for executing system commands. +// All packages that shell out accept this interface, enabling +// unit tests to inject FakeRunner without executing anything. +type Runner interface { + // Run executes a command and returns an error if it fails. + Run(name string, args ...string) error + + // RunWithEnv executes a command with additional environment variables. + RunWithEnv(env map[string]string, name string, args ...string) error + + // RunInDir executes a command in the specified directory. + RunInDir(dir string, name string, args ...string) error + + // RunInDirWithEnv executes a command in the specified directory with additional env vars. + RunInDirWithEnv(dir string, env map[string]string, name string, args ...string) error + + // Output executes a command and returns its stdout. + Output(name string, args ...string) (string, error) +} + +// RealRunner executes commands on the real system. +type RealRunner struct{} + +// Run executes a command, inheriting the current process's environment. +func (r *RealRunner) Run(name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("running %s %s: %w", name, strings.Join(args, " "), err) + } + return nil +} + +// RunWithEnv executes a command with additional environment variables +// merged into the current process environment. +func (r *RealRunner) RunWithEnv(env map[string]string, name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + if err := cmd.Run(); err != nil { + return fmt.Errorf("running %s %s: %w", name, strings.Join(args, " "), err) + } + return nil +} + +// RunInDir executes a command in the specified working directory. +func (r *RealRunner) RunInDir(dir string, name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Dir = dir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("running %s %s in %s: %w", name, strings.Join(args, " "), dir, err) + } + return nil +} + +// RunInDirWithEnv executes a command in the specified directory with +// additional environment variables merged into the current process environment. +func (r *RealRunner) RunInDirWithEnv(dir string, env map[string]string, name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Dir = dir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + if err := cmd.Run(); err != nil { + return fmt.Errorf("running %s %s in %s: %w", name, strings.Join(args, " "), dir, err) + } + return nil +} + +// Output executes a command and returns its combined stdout as a string. +func (r *RealRunner) Output(name string, args ...string) (string, error) { + cmd := exec.Command(name, args...) + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("running %s %s: %w", name, strings.Join(args, " "), err) + } + return strings.TrimSpace(string(out)), nil +} + +// Call records a single command invocation made through FakeRunner. +type Call struct { + Name string + Args []string + Env map[string]string + Dir string +} + +// String returns a human-readable representation of the call. +func (c Call) String() string { + parts := []string{c.Name} + parts = append(parts, c.Args...) + return strings.Join(parts, " ") +} + +// FakeRunner records all command invocations without executing them. +// Used in unit tests to verify the exact sequence, arguments, and +// environment of every system call. +type FakeRunner struct { + Calls []Call + OutputMap map[string]string // keyed by "name arg1 arg2..." → stdout + ErrorMap map[string]error // keyed by "name arg1 arg2..." → error +} + +// NewFakeRunner creates a FakeRunner with initialized maps. +func NewFakeRunner() *FakeRunner { + return &FakeRunner{ + OutputMap: make(map[string]string), + ErrorMap: make(map[string]error), + } +} + +func (f *FakeRunner) key(name string, args ...string) string { + parts := []string{name} + parts = append(parts, args...) + return strings.Join(parts, " ") +} + +// Run records the call and returns any configured error. +func (f *FakeRunner) Run(name string, args ...string) error { + f.Calls = append(f.Calls, Call{Name: name, Args: args}) + if err, ok := f.ErrorMap[f.key(name, args...)]; ok { + return err + } + return nil +} + +// RunWithEnv records the call with environment and returns any configured error. +func (f *FakeRunner) RunWithEnv(env map[string]string, name string, args ...string) error { + f.Calls = append(f.Calls, Call{Name: name, Args: args, Env: env}) + if err, ok := f.ErrorMap[f.key(name, args...)]; ok { + return err + } + return nil +} + +// RunInDir records the call with directory and returns any configured error. +func (f *FakeRunner) RunInDir(dir string, name string, args ...string) error { + f.Calls = append(f.Calls, Call{Name: name, Args: args, Dir: dir}) + if err, ok := f.ErrorMap[f.key(name, args...)]; ok { + return err + } + return nil +} + +// RunInDirWithEnv records the call with directory and env and returns any configured error. +func (f *FakeRunner) RunInDirWithEnv(dir string, env map[string]string, name string, args ...string) error { + f.Calls = append(f.Calls, Call{Name: name, Args: args, Dir: dir, Env: env}) + if err, ok := f.ErrorMap[f.key(name, args...)]; ok { + return err + } + return nil +} + +// Output records the call and returns any configured output or error. +func (f *FakeRunner) Output(name string, args ...string) (string, error) { + f.Calls = append(f.Calls, Call{Name: name, Args: args}) + key := f.key(name, args...) + if err, ok := f.ErrorMap[key]; ok { + return "", err + } + if out, ok := f.OutputMap[key]; ok { + return out, nil + } + return "", nil +} diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go new file mode 100644 index 00000000..3618b19e --- /dev/null +++ b/internal/runner/runner_test.go @@ -0,0 +1,113 @@ +package runner_test + +import ( + "errors" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/runner" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFakeRunnerRecordsCalls(t *testing.T) { + f := runner.NewFakeRunner() + + err := f.Run("apt-get", "-y", "install", "foo") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "apt-get", f.Calls[0].Name) + assert.Equal(t, []string{"-y", "install", "foo"}, f.Calls[0].Args) +} + +func TestFakeRunnerRunWithEnv(t *testing.T) { + f := runner.NewFakeRunner() + env := map[string]string{"DEBIAN_FRONTEND": "noninteractive"} + + err := f.RunWithEnv(env, "apt-get", "update") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "apt-get", f.Calls[0].Name) + assert.Equal(t, []string{"update"}, f.Calls[0].Args) + assert.Equal(t, "noninteractive", f.Calls[0].Env["DEBIAN_FRONTEND"]) +} + +func TestFakeRunnerRunInDir(t *testing.T) { + f := runner.NewFakeRunner() + + err := f.RunInDir("/tmp/build", "make", "install") + require.NoError(t, err) + + require.Len(t, f.Calls, 1) + assert.Equal(t, "make", f.Calls[0].Name) + assert.Equal(t, []string{"install"}, f.Calls[0].Args) + assert.Equal(t, "/tmp/build", f.Calls[0].Dir) +} + +func TestFakeRunnerOutputReturnsConfiguredValue(t *testing.T) { + f := runner.NewFakeRunner() + f.OutputMap["git describe --tags"] = "v1.2.3" + + out, err := f.Output("git", "describe", "--tags") + require.NoError(t, err) + assert.Equal(t, "v1.2.3", out) +} + +func TestFakeRunnerOutputReturnsEmptyForUnknown(t *testing.T) { + f := runner.NewFakeRunner() + + out, err := f.Output("unknown", "command") + require.NoError(t, err) + assert.Equal(t, "", out) +} + +func TestFakeRunnerErrorMapTriggersError(t *testing.T) { + f := runner.NewFakeRunner() + f.ErrorMap["make install"] = errors.New("make failed") + + err := f.Run("make", "install") + require.Error(t, err) + assert.Equal(t, "make failed", err.Error()) +} + +func TestFakeRunnerErrorMapForOutput(t *testing.T) { + f := runner.NewFakeRunner() + f.ErrorMap["git status"] = errors.New("not a git repo") + + _, err := f.Output("git", "status") + require.Error(t, err) + assert.Equal(t, "not a git repo", err.Error()) +} + +func TestFakeRunnerMultipleCalls(t *testing.T) { + f := runner.NewFakeRunner() + + _ = f.Run("apt-get", "update") + _ = f.Run("apt-get", "install", "-y", "gcc") + _ = f.RunInDir("/build", "make") + + require.Len(t, f.Calls, 3) + assert.Equal(t, "apt-get update", f.Calls[0].String()) + assert.Equal(t, "apt-get install -y gcc", f.Calls[1].String()) + assert.Equal(t, "make", f.Calls[2].String()) + assert.Equal(t, "/build", f.Calls[2].Dir) +} + +func TestCallString(t *testing.T) { + c := runner.Call{Name: "wget", Args: []string{"-q", "-O", "/tmp/file", "http://example.com"}} + assert.Equal(t, "wget -q -O /tmp/file http://example.com", c.String()) +} + +func TestRealRunnerRunSuccess(t *testing.T) { + r := &runner.RealRunner{} + err := r.Run("true") + require.NoError(t, err) +} + +func TestRealRunnerRunFailure(t *testing.T) { + r := &runner.RealRunner{} + err := r.Run("false") + require.Error(t, err) + assert.Contains(t, err.Error(), "false") +} diff --git a/internal/source/source_input.go b/internal/source/source_input.go new file mode 100644 index 00000000..d26a8ee8 --- /dev/null +++ b/internal/source/source_input.go @@ -0,0 +1,154 @@ +// Package source parses the Concourse source/data.json resource, +// handling both legacy and modern JSON formats. +package source + +import ( + "encoding/json" + "fmt" + "os" +) + +// Checksum represents a hash algorithm and its expected value. +type Checksum struct { + Algorithm string // "sha256", "sha512", "md5", "sha1" + Value string +} + +// Input represents the parsed source/data.json from a Concourse resource. +type Input struct { + Name string `json:"name"` + URL string `json:"url"` + Version string `json:"version"` + MD5 string `json:"md5"` + SHA256 string `json:"sha256"` + SHA512 string `json:"sha512"` + SHA1 string `json:"sha1"` + GitCommitSHA string `json:"git_commit_sha"` + Repo string `json:"repo"` + Type string `json:"type"` +} + +// legacyInput handles the older JSON format with different field names. +type legacyInput struct { + Name string `json:"name"` + SourceURI string `json:"source_uri"` + Version string `json:"version"` + SourceSHA string `json:"source_sha"` + Repo string `json:"repo"` + Type string `json:"type"` +} + +// modernInput handles the depwatcher JSON format with nested source + version objects: +// +// { +// "source": {"name": "composer", "type": "github_releases", "repo": "composer/composer"}, +// "version": {"url": "https://...", "ref": "2.7.1", "sha256": "...", "sha512": "..."} +// } +type modernInput struct { + Source struct { + Name string `json:"name"` + Repo string `json:"repo"` + Type string `json:"type"` + } `json:"source"` + Version struct { + URL string `json:"url"` + Ref string `json:"ref"` + SHA256 string `json:"sha256"` + SHA512 string `json:"sha512"` + MD5 string `json:"md5_digest"` + SHA1 string `json:"sha1"` + GitCommitSHA string `json:"git_commit_sha"` + } `json:"version"` +} + +// FromFile reads and parses a source data.json file. +// It auto-detects the format (legacy vs modern) based on the presence +// of a "source" key or "source_uri" key. +func FromFile(path string) (*Input, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading source file %q: %w", path, err) + } + + return Parse(data) +} + +// Parse parses source JSON data, auto-detecting the format. +func Parse(data []byte) (*Input, error) { + // Try to detect format by checking for key fields. + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("parsing source JSON: %w", err) + } + + // Modern format has a "source" object. + if _, hasSource := raw["source"]; hasSource { + return parseModern(data) + } + + // Legacy format has "source_uri". + if _, hasSourceURI := raw["source_uri"]; hasSourceURI { + return parseLegacy(data) + } + + // Fallback: try modern first, then legacy. + if input, err := parseModern(data); err == nil && input.URL != "" { + return input, nil + } + + return parseLegacy(data) +} + +func parseModern(data []byte) (*Input, error) { + var m modernInput + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("parsing modern source format: %w", err) + } + + return &Input{ + Name: m.Source.Name, + URL: m.Version.URL, + Version: m.Version.Ref, + SHA256: m.Version.SHA256, + SHA512: m.Version.SHA512, + MD5: m.Version.MD5, + SHA1: m.Version.SHA1, + GitCommitSHA: m.Version.GitCommitSHA, + Repo: m.Source.Repo, + Type: m.Source.Type, + }, nil +} + +func parseLegacy(data []byte) (*Input, error) { + var l legacyInput + if err := json.Unmarshal(data, &l); err != nil { + return nil, fmt.Errorf("parsing legacy source format: %w", err) + } + + return &Input{ + Name: l.Name, + URL: l.SourceURI, + Version: l.Version, + MD5: l.SourceSHA, // legacy format uses source_sha for MD5 + Repo: l.Repo, + Type: l.Type, + }, nil +} + +// PrimaryChecksum returns the strongest available checksum. +// Preference order: SHA512 > SHA256 > MD5 > SHA1. +func (i *Input) PrimaryChecksum() Checksum { + if i.SHA512 != "" { + return Checksum{Algorithm: "sha512", Value: i.SHA512} + } + if i.SHA256 != "" { + return Checksum{Algorithm: "sha256", Value: i.SHA256} + } + if i.MD5 != "" { + return Checksum{Algorithm: "md5", Value: i.MD5} + } + if i.SHA1 != "" { + return Checksum{Algorithm: "sha1", Value: i.SHA1} + } + return Checksum{} +} diff --git a/internal/source/source_input_test.go b/internal/source/source_input_test.go new file mode 100644 index 00000000..8fb4dcea --- /dev/null +++ b/internal/source/source_input_test.go @@ -0,0 +1,189 @@ +package source_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/source" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestParseModernFormat verifies parsing of the real depwatcher output format: +// +// { +// "source": {"name": "...", "type": "...", "repo": "..."}, +// "version": {"url": "...", "ref": "...", "sha256": "...", "sha512": "..."} +// } +func TestParseModernFormat(t *testing.T) { + data := []byte(`{ + "source": { + "name": "ruby", + "type": "github-releases", + "repo": "ruby/ruby" + }, + "version": { + "url": "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.6.tar.gz", + "ref": "3.3.6", + "sha256": "abc123", + "sha512": "def456", + "git_commit_sha": "deadbeef" + } + }`) + + input, err := source.Parse(data) + require.NoError(t, err) + + assert.Equal(t, "ruby", input.Name) + assert.Equal(t, "3.3.6", input.Version) + assert.Equal(t, "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.6.tar.gz", input.URL) + assert.Equal(t, "abc123", input.SHA256) + assert.Equal(t, "def456", input.SHA512) + assert.Equal(t, "deadbeef", input.GitCommitSHA) + assert.Equal(t, "github-releases", input.Type) + assert.Equal(t, "ruby/ruby", input.Repo) +} + +func TestParseModernFormatWithMD5(t *testing.T) { + data := []byte(`{ + "source": {"name": "node", "type": "node_lts"}, + "version": { + "url": "https://nodejs.org/dist/v20.11.0/node-v20.11.0.tar.gz", + "ref": "20.11.0", + "md5_digest": "md5hashvalue" + } + }`) + + input, err := source.Parse(data) + require.NoError(t, err) + + assert.Equal(t, "node", input.Name) + assert.Equal(t, "20.11.0", input.Version) + assert.Equal(t, "md5hashvalue", input.MD5) + assert.Empty(t, input.SHA256) +} + +func TestParseLegacyFormat(t *testing.T) { + data := []byte(`{ + "name": "python", + "version": "3.12.0", + "source_uri": "https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tgz", + "source_sha": "abc123md5" + }`) + + input, err := source.Parse(data) + require.NoError(t, err) + + assert.Equal(t, "python", input.Name) + assert.Equal(t, "3.12.0", input.Version) + assert.Equal(t, "https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tgz", input.URL) + assert.Equal(t, "abc123md5", input.MD5) + assert.Empty(t, input.SHA256) +} + +func TestPrimaryChecksumPrefersSHA512(t *testing.T) { + input := &source.Input{ + SHA512: "sha512val", + SHA256: "sha256val", + MD5: "md5val", + } + + cs := input.PrimaryChecksum() + assert.Equal(t, "sha512", cs.Algorithm) + assert.Equal(t, "sha512val", cs.Value) +} + +func TestPrimaryChecksumFallsBackToSHA256(t *testing.T) { + input := &source.Input{ + SHA256: "sha256val", + MD5: "md5val", + } + + cs := input.PrimaryChecksum() + assert.Equal(t, "sha256", cs.Algorithm) + assert.Equal(t, "sha256val", cs.Value) +} + +func TestPrimaryChecksumFallsBackToMD5(t *testing.T) { + input := &source.Input{ + MD5: "md5val", + } + + cs := input.PrimaryChecksum() + assert.Equal(t, "md5", cs.Algorithm) + assert.Equal(t, "md5val", cs.Value) +} + +func TestPrimaryChecksumFallsBackToSHA1(t *testing.T) { + input := &source.Input{ + SHA1: "sha1val", + } + + cs := input.PrimaryChecksum() + assert.Equal(t, "sha1", cs.Algorithm) + assert.Equal(t, "sha1val", cs.Value) +} + +func TestPrimaryChecksumEmpty(t *testing.T) { + input := &source.Input{} + + cs := input.PrimaryChecksum() + assert.Empty(t, cs.Algorithm) + assert.Empty(t, cs.Value) +} + +func TestFromFile(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "data.json") + data := []byte(`{ + "source": {"name": "node", "type": "node_lts"}, + "version": { + "url": "https://nodejs.org/dist/v20.11.0/node-v20.11.0.tar.gz", + "ref": "20.11.0", + "sha256": "abc123" + } + }`) + err := os.WriteFile(path, data, 0644) + require.NoError(t, err) + + input, err := source.FromFile(path) + require.NoError(t, err) + + assert.Equal(t, "node", input.Name) + assert.Equal(t, "20.11.0", input.Version) + assert.Equal(t, "abc123", input.SHA256) +} + +func TestFromFileMissing(t *testing.T) { + _, err := source.FromFile("/nonexistent/data.json") + require.Error(t, err) + assert.Contains(t, err.Error(), "reading source file") +} + +func TestParseMalformedJSON(t *testing.T) { + _, err := source.Parse([]byte("{invalid json")) + require.Error(t, err) + assert.Contains(t, err.Error(), "parsing source JSON") +} + +func TestParseModernWithRepo(t *testing.T) { + data := []byte(`{ + "source": { + "name": "libgdiplus", + "type": "github_releases", + "repo": "mono/libgdiplus" + }, + "version": { + "url": "https://github.com/mono/libgdiplus/archive/6.1.tar.gz", + "ref": "6.1", + "sha256": "abc123" + } + }`) + + input, err := source.Parse(data) + require.NoError(t, err) + + assert.Equal(t, "mono/libgdiplus", input.Repo) + assert.Equal(t, "6.1", input.Version) +} diff --git a/internal/stack/stack.go b/internal/stack/stack.go new file mode 100644 index 00000000..d30fa392 --- /dev/null +++ b/internal/stack/stack.go @@ -0,0 +1,133 @@ +// Package stack provides the Stack configuration struct and YAML loader. +// All Ubuntu-version-specific values live in stack YAML files — no stack +// names appear in Go source code. +package stack + +import ( + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" +) + +// GfortranConfig holds gfortran compiler settings for a specific stack. +type GfortranConfig struct { + Version int `yaml:"version"` + Bin string `yaml:"bin"` + LibPath string `yaml:"lib_path"` + // LibexecPath is the directory where GCC executables (e.g. f951) live. + // On Ubuntu 22.04 (jammy) executables share the same directory as the libs + // (LibPath), so LibexecPath can be left empty and LibPath is used as fallback. + // On Ubuntu 24.04 (noble) GCC moved executables to /usr/libexec/gcc/… + // while libs remain in /usr/lib/gcc/…, so LibexecPath must be set explicitly. + LibexecPath string `yaml:"libexec_path"` + Packages []string `yaml:"packages"` +} + +// GCCConfig holds GCC/g++ compiler settings for a specific stack. +type GCCConfig struct { + Version int `yaml:"version"` + Packages []string `yaml:"packages"` + PPA string `yaml:"ppa"` + // ToolPackages lists prerequisite apt packages needed before GCC setup + // (e.g. software-properties-common for add-apt-repository). Stored here + // rather than hardcoded in compiler.go so that future stacks can override + // the list without touching Go source. + ToolPackages []string `yaml:"tool_packages"` +} + +// CompilerConfig groups all compiler configurations. +type CompilerConfig struct { + Gfortran GfortranConfig `yaml:"gfortran"` + GCC GCCConfig `yaml:"gcc"` +} + +// BootstrapBinary holds the URL, SHA256 checksum, and optional fixed install +// directory for a build-time bootstrap binary (Go toolchain, JDK, Ruby). +// InstallDir is empty for Go because its extract path is version-scoped at +// runtime (/tmp/go-bootstrap-{version}). +type BootstrapBinary struct { + URL string `yaml:"url"` + SHA256 string `yaml:"sha256"` + InstallDir string `yaml:"install_dir"` +} + +// BootstrapConfig groups all build-time bootstrap binaries. +// Each language that requires a pre-built tool to compile from source lists it +// here so that recipes never hard-code URLs or checksums. +type BootstrapConfig struct { + Go BootstrapBinary `yaml:"go"` + JRuby BootstrapBinary `yaml:"jruby"` + Ruby BootstrapBinary `yaml:"ruby"` +} + +// PythonConfig holds Python-specific build settings. +type PythonConfig struct { + TCLVersion string `yaml:"tcl_version"` + UseForceYes bool `yaml:"use_force_yes"` +} + +// HTTPDSubDep holds the pinned version, download URL and SHA256 for a single +// HTTPD sub-dependency (APR, APR-Iconv, APR-Util, mod_auth_openidc). +type HTTPDSubDep struct { + Version string `yaml:"version"` + URL string `yaml:"url"` + SHA256 string `yaml:"sha256"` +} + +// HTTPDSubDepsConfig groups all HTTPD sub-dependency pinned versions. +type HTTPDSubDepsConfig struct { + APR HTTPDSubDep `yaml:"apr"` + APRIconv HTTPDSubDep `yaml:"apr_iconv"` + APRUtil HTTPDSubDep `yaml:"apr_util"` + ModAuthOpenidc HTTPDSubDep `yaml:"mod_auth_openidc"` +} + +// Symlink represents a filesystem symlink to create during builds. +type Symlink struct { + Src string `yaml:"src"` + Dst string `yaml:"dst"` +} + +// Stack holds all configuration for a specific Ubuntu stack (cflinuxfs4, cflinuxfs5, etc.). +// Every Ubuntu-version-specific value lives here — recipes read from this struct +// and never contain hardcoded stack names or version numbers. +type Stack struct { + Name string `yaml:"name"` + UbuntuVersion string `yaml:"ubuntu_version"` + UbuntuCodename string `yaml:"ubuntu_codename"` + DockerImage string `yaml:"docker_image"` + Bootstrap BootstrapConfig `yaml:"bootstrap"` + Compilers CompilerConfig `yaml:"compilers"` + AptPackages map[string][]string `yaml:"apt_packages"` + PHPSymlinks []Symlink `yaml:"php_symlinks"` + Python PythonConfig `yaml:"python"` + HTTPDSubDeps HTTPDSubDepsConfig `yaml:"httpd_sub_deps"` +} + +// Load reads a stack YAML file from stacksDir for the given stack name. +// Returns an error if the file does not exist or cannot be parsed. +func Load(stacksDir, name string) (*Stack, error) { + path := filepath.Join(stacksDir, name+".yaml") + + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("loading stack %q: %w", name, err) + } + + var s Stack + if err := yaml.Unmarshal(data, &s); err != nil { + return nil, fmt.Errorf("parsing stack %q: %w", name, err) + } + + if s.Name == "" { + return nil, fmt.Errorf("stack %q: name field is empty", name) + } + + if s.Name != name { + return nil, fmt.Errorf("stack file %q declares name %q (expected %q)", path, s.Name, name) + } + + return &s, nil +} diff --git a/internal/stack/stack_test.go b/internal/stack/stack_test.go new file mode 100644 index 00000000..729bbc55 --- /dev/null +++ b/internal/stack/stack_test.go @@ -0,0 +1,243 @@ +package stack_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/cloudfoundry/binary-builder/internal/stack" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func stacksDir(t *testing.T) string { + t.Helper() + // Walk up to find the stacks/ directory relative to the repo root. + dir, err := filepath.Abs("../../stacks") + require.NoError(t, err) + return dir +} + +func TestLoadCflinuxfs4(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + assert.Equal(t, "cflinuxfs4", s.Name) + assert.Equal(t, "22.04", s.UbuntuVersion) + assert.Equal(t, "jammy", s.UbuntuCodename) + assert.Equal(t, "cloudfoundry/cflinuxfs4", s.DockerImage) +} + +func TestLoadCflinuxfs5(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs5") + require.NoError(t, err) + + assert.Equal(t, "cflinuxfs5", s.Name) + assert.Equal(t, "24.04", s.UbuntuVersion) + assert.Equal(t, "noble", s.UbuntuCodename) + assert.Equal(t, "cloudfoundry/cflinuxfs5", s.DockerImage) +} + +func TestGfortranVersionCflinuxfs4(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + assert.Equal(t, 11, s.Compilers.Gfortran.Version) + assert.Equal(t, "/usr/bin/x86_64-linux-gnu-gfortran-11", s.Compilers.Gfortran.Bin) + assert.Equal(t, "/usr/lib/gcc/x86_64-linux-gnu/11", s.Compilers.Gfortran.LibPath) + // cflinuxfs4 (jammy): executables and libs share the same dir; libexec_path omitted. + assert.Equal(t, "", s.Compilers.Gfortran.LibexecPath) +} + +func TestGfortranVersionCflinuxfs5(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs5") + require.NoError(t, err) + + assert.Equal(t, 13, s.Compilers.Gfortran.Version) + assert.Equal(t, "/usr/bin/x86_64-linux-gnu-gfortran-13", s.Compilers.Gfortran.Bin) + assert.Equal(t, "/usr/lib/gcc/x86_64-linux-gnu/13", s.Compilers.Gfortran.LibPath) + // cflinuxfs5 (noble): GCC executables moved to /usr/libexec/gcc/… + assert.Equal(t, "/usr/libexec/gcc/x86_64-linux-gnu/13", s.Compilers.Gfortran.LibexecPath) +} + +func TestGCCPPACflinuxfs4(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + assert.Equal(t, "ppa:ubuntu-toolchain-r/test", s.Compilers.GCC.PPA) + assert.Equal(t, 12, s.Compilers.GCC.Version) + assert.Contains(t, s.Compilers.GCC.Packages, "gcc-12") + assert.Contains(t, s.Compilers.GCC.Packages, "g++-12") +} + +func TestGCCPPACflinuxfs5(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs5") + require.NoError(t, err) + + assert.Equal(t, "", s.Compilers.GCC.PPA) + assert.Equal(t, 14, s.Compilers.GCC.Version) + assert.Contains(t, s.Compilers.GCC.Packages, "gcc-14") + assert.Contains(t, s.Compilers.GCC.Packages, "g++-14") +} + +func TestPHPSymlinksCflinuxfs4HasLibldapR(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + found := false + for _, sym := range s.PHPSymlinks { + if sym.Dst == "/usr/lib/libldap_r.so" { + found = true + break + } + } + assert.True(t, found, "cflinuxfs4 should have libldap_r.so symlink") +} + +func TestPHPSymlinksCflinuxfs5NoLibldapR(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs5") + require.NoError(t, err) + + for _, sym := range s.PHPSymlinks { + assert.NotEqual(t, "/usr/lib/libldap_r.so", sym.Dst, + "cflinuxfs5 should NOT have libldap_r.so symlink (dropped in OpenLDAP 2.6)") + } +} + +func TestPythonUseForceYesCflinuxfs4(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + assert.True(t, s.Python.UseForceYes) +} + +func TestPythonUseForceYesCflinuxfs5(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs5") + require.NoError(t, err) + + assert.False(t, s.Python.UseForceYes) +} + +func TestPHPBuildPackagesCflinuxfs4(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + pkgs := s.AptPackages["php_build"] + assert.Contains(t, pkgs, "libdb-dev") + assert.NotContains(t, pkgs, "libdb5.3-dev") + assert.Contains(t, pkgs, "libzookeeper-mt-dev") +} + +func TestPHPBuildPackagesCflinuxfs5(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs5") + require.NoError(t, err) + + pkgs := s.AptPackages["php_build"] + assert.Contains(t, pkgs, "libdb5.3-dev") + assert.NotContains(t, pkgs, "libdb-dev") + assert.NotContains(t, pkgs, "libzookeeper-mt-dev") +} + +func TestRBuildPackagesCflinuxfs4(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + pkgs := s.AptPackages["r_build"] + assert.Contains(t, pkgs, "libpcre++-dev") + assert.Contains(t, pkgs, "libtiff5-dev") +} + +func TestRBuildPackagesCflinuxfs5(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs5") + require.NoError(t, err) + + pkgs := s.AptPackages["r_build"] + assert.NotContains(t, pkgs, "libpcre++-dev") + assert.Contains(t, pkgs, "libtiff-dev") + assert.NotContains(t, pkgs, "libtiff5-dev") +} + +func TestLoadMissingFile(t *testing.T) { + _, err := stack.Load(stacksDir(t), "nonexistent") + require.Error(t, err) + assert.Contains(t, err.Error(), "nonexistent") +} + +func TestLoadMalformedYAML(t *testing.T) { + tmpDir := t.TempDir() + err := os.WriteFile(filepath.Join(tmpDir, "bad.yaml"), []byte("{{invalid yaml"), 0644) + require.NoError(t, err) + + _, err = stack.Load(tmpDir, "bad") + require.Error(t, err) + assert.Contains(t, err.Error(), "parsing") +} + +func TestLoadEmptyName(t *testing.T) { + tmpDir := t.TempDir() + err := os.WriteFile(filepath.Join(tmpDir, "empty.yaml"), []byte("ubuntu_version: '22.04'\n"), 0644) + require.NoError(t, err) + + _, err = stack.Load(tmpDir, "empty") + require.Error(t, err) + assert.Contains(t, err.Error(), "name field is empty") +} + +func TestLoadNameMismatch(t *testing.T) { + tmpDir := t.TempDir() + err := os.WriteFile(filepath.Join(tmpDir, "wrong.yaml"), []byte("name: other\nubuntu_version: '22.04'\n"), 0644) + require.NoError(t, err) + + _, err = stack.Load(tmpDir, "wrong") + require.Error(t, err) + assert.Contains(t, err.Error(), "expected \"wrong\"") +} + +func TestJRubyConfigCflinuxfs4(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + assert.Contains(t, s.Bootstrap.JRuby.URL, "jammy") + assert.Equal(t, "/opt/java", s.Bootstrap.JRuby.InstallDir) +} + +func TestJRubyConfigCflinuxfs5(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs5") + require.NoError(t, err) + + // No noble JDK bucket exists yet; cflinuxfs5 intentionally uses the jammy + // Bellsoft JDK 8 build which is binary-compatible with Ubuntu 24.04. + assert.Contains(t, s.Bootstrap.JRuby.URL, "jammy") + assert.Equal(t, "/opt/java", s.Bootstrap.JRuby.InstallDir) +} + +func TestRubyBootstrapCflinuxfs4(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + assert.Contains(t, s.Bootstrap.Ruby.URL, "cflinuxfs4") + assert.Equal(t, "/opt/ruby", s.Bootstrap.Ruby.InstallDir) +} + +func TestGoBootstrapCflinuxfs4(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + assert.Contains(t, s.Bootstrap.Go.URL, "go.dev/dl/") + assert.Contains(t, s.Bootstrap.Go.URL, "linux-amd64.tar.gz") +} + +func TestGoBootstrapCflinuxfs5(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs5") + require.NoError(t, err) + + assert.Contains(t, s.Bootstrap.Go.URL, "go.dev/dl/") + assert.Contains(t, s.Bootstrap.Go.URL, "linux-amd64.tar.gz") +} + +func TestPythonTCLVersion(t *testing.T) { + s, err := stack.Load(stacksDir(t), "cflinuxfs4") + require.NoError(t, err) + + assert.Equal(t, "8.6", s.Python.TCLVersion) +} diff --git a/lib/archive_recipe.rb b/lib/archive_recipe.rb deleted file mode 100644 index 8038a1cb..00000000 --- a/lib/archive_recipe.rb +++ /dev/null @@ -1,42 +0,0 @@ -# encoding: utf-8 -require 'tmpdir' -require_relative 'yaml_presenter' - -class ArchiveRecipe - def initialize(recipe) - @recipe = recipe - end - - def compress! - return if @recipe.archive_files.empty? - - @recipe.setup_tar if @recipe.respond_to? :setup_tar - - Dir.mktmpdir do |dir| - archive_path = File.join(dir, @recipe.archive_path_name) - FileUtils.mkdir_p(archive_path) - - @recipe.archive_files.each do |glob| - `cp -r #{glob} #{archive_path}` - end - - File.write("#{dir}/sources.yml", YAMLPresenter.new(@recipe).to_yaml) - - print "Running 'archive' for #{@recipe.name} #{@recipe.version}... " - if @recipe.archive_filename.split('.').last == 'zip' - output_dir = Dir.pwd - - Dir.chdir(dir) do - `zip #{File.join(output_dir, @recipe.archive_filename)} -r .` - end - else - if @recipe.name == 'php' - `ls -A #{dir} | xargs tar --dereference -czf #{@recipe.archive_filename} -C #{dir}` - else - `ls -A #{dir} | xargs tar czf #{@recipe.archive_filename} -C #{dir}` - end - end - puts 'OK' - end - end -end diff --git a/lib/geoip_downloader.rb b/lib/geoip_downloader.rb deleted file mode 100644 index 5e666a58..00000000 --- a/lib/geoip_downloader.rb +++ /dev/null @@ -1,147 +0,0 @@ -# encoding: utf-8 -require "net/http" -require "uri" -require "digest" -require "tempfile" - -class MaxMindGeoIpUpdater - @@FREE_LICENSE = '000000000000' - @@FREE_USER = '999999' - - def initialize(user_id, license, output_dir) - @proto = 'http' - @host = 'updates.maxmind.com' - @user_id = user_id - @license = license - @output_dir = output_dir - @client_ip = nil - @challenge_digest = nil - end - - def self.FREE_LICENSE - @@FREE_LICENSE - end - - def self.FREE_USER - @@FREE_USER - end - - def get_filename(product_id) - uri = URI.parse("#{@proto}://#{@host}/app/update_getfilename") - uri.query = URI.encode_www_form({ :product_id => product_id}) - resp = Net::HTTP.get_response(uri) - resp.body - end - - def client_ip - @client_ip ||= begin - uri = URI.parse("#{@proto}://#{@host}/app/update_getipaddr") - resp = Net::HTTP.get_response(uri) - resp.body - end - end - - def download_database(db_digest, challenge_digest, product_id, file_path) - uri = URI.parse("#{@proto}://#{@host}/app/update_secure") - uri.query = URI.encode_www_form({ - :db_md5 => db_digest, - :challenge_md5 => challenge_digest, - :user_id => @user_id, - :edition_id => product_id - }) - - Net::HTTP.start(uri.host, uri.port) do |http| - req = Net::HTTP::Get.new(uri.request_uri) - - http.request(req) do |resp| - file = Tempfile.new('geoip_db_download') - begin - if resp['content-type'] == 'text/plain; charset=utf-8' - puts "\tAlready up-to-date." - else - resp.read_body do |chunk| - file.write(chunk) - end - file.rewind - extract_file(file, file_path) - puts "\tDatabase updated." - end - ensure - file.close() - file.unlink() - end - end - end - end - - def download_free_database(product_id, file_path) - product_uris = { - "GeoLite-Legacy-IPv6-City" => "http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz", - "GeoLite-Legacy-IPv6-Country" => "http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz" - } - - if !product_uris.include?(product_id) - puts "\tProduct '#{product_id}' is not available under free license. Available products are: #{product_uris.keys().join(', ')}." - else - uri = URI.parse(product_uris[product_id]) - Net::HTTP.start(uri.host, uri.port) do |http| - req = Net::HTTP::Get.new(uri.request_uri) - - http.request(req) do |resp| - file = Tempfile.new('geoip_db_download') - begin - resp.read_body do |chunk| - file.write(chunk) - end - file.rewind - extract_file(file, file_path) - puts "\tDatabase updated." - ensure - file.close() - file.unlink() - end - end - end - end - end - - def extract_file(file, file_path) - gz = Zlib::GzipReader.new(file) - begin - File.open(file_path, 'w') do |out| - IO.copy_stream(gz, out) - end - ensure - gz.close - end - end - - def download_product(product_id) - puts "Downloading..." - file_name = get_filename(product_id) - file_path = File.join(@output_dir, file_name) - db_digest = db_digest(file_path) - puts "\tproduct_id: #{product_id}" - puts "\tfile_name: #{file_name}" - puts "\tip: #{client_ip}" - puts "\tdb: #{db_digest}" - - if @license == @@FREE_LICENSE - # As of April 1, 2018, free legacy databases are no longer available through the GeoIP update - # API. Therefore, we'll fetch them from static URLs they've provided. - # This will NOT work using free GeoIP2 databases. - download_free_database(product_id, file_path) - else - puts "\tchallenge: #{challenge_digest}" - download_database(db_digest, challenge_digest, product_id, file_path) - end - end - - def db_digest(path) - return File::exist?(path) ? Digest::MD5.file(path) : '00000000000000000000000000000000' - end - - def challenge_digest - return Digest::MD5.hexdigest("#{@license}#{client_ip}") - end -end diff --git a/lib/install_go.rb b/lib/install_go.rb deleted file mode 100644 index f9c45653..00000000 --- a/lib/install_go.rb +++ /dev/null @@ -1,34 +0,0 @@ -require 'net/http' -require 'json' - -def install_go_compiler - - url = URI('https://go.dev/dl/?mode=json') - res = Net::HTTP.get(url) - latest_go = JSON.parse(res).first - - go_version = latest_go['version'].delete_prefix('go') - go_sha256 = "" - - latest_go['files'].each do |file| - if file['filename'] == "go#{go_version}.linux-amd64.tar.gz" - go_sha256 = file['sha256'] - break - end - end - - Dir.chdir("/usr/local") do - go_download = "https://go.dev/dl/go#{go_version}.linux-amd64.tar.gz" - go_tar = "go.tar.gz" - - system("curl -L #{go_download} -o #{go_tar}") - - downloaded_sha = Digest::SHA256.file(go_tar).hexdigest - - if go_sha256 != downloaded_sha - raise "sha256 verification failed: expected #{go_sha256}, got #{downloaded_sha}" - end - - system("tar xf #{go_tar}") - end -end diff --git a/lib/openssl_replace.rb b/lib/openssl_replace.rb deleted file mode 100644 index b0b16ecf..00000000 --- a/lib/openssl_replace.rb +++ /dev/null @@ -1,29 +0,0 @@ -require_relative '../recipe/base' - -class OpenSSLReplace - - def self.run(*args) - system({'DEBIAN_FRONTEND' => 'noninteractive'}, *args) - raise "Could not run #{args}" unless $?.success? - end - - def self.replace_openssl() - filebase = 'OpenSSL_1_1_0g' - filename = "#{filebase}.tar.gz" - openssltar = "https://github.com/openssl/openssl/archive/#{filename}" - - Dir.mktmpdir do |dir| - run('wget', openssltar) - run('tar', 'xf', filename) - Dir.chdir("openssl-#{filebase}") do - run("./config", - "--prefix=/usr", - "--libdir=/lib/x86_64-linux-gnu", - "--openssldir=/include/x86_64-linux-gnu/openssl") - run('make') - run('make', 'install') - end - end - end - -end \ No newline at end of file diff --git a/lib/utils.rb b/lib/utils.rb deleted file mode 100644 index cf38ce7f..00000000 --- a/lib/utils.rb +++ /dev/null @@ -1,56 +0,0 @@ -require 'net/http' - -module HTTPHelper - class << self - def download_with_follow_redirects(uri) - uri = URI(uri) - Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |httpRequest| - response = httpRequest.request_get(uri) - if response.is_a?(Net::HTTPRedirection) - download_with_follow_redirects(response['location']) - else - response - end - end - end - - def download(uri, filename, digest_algorithm, sha) - response = download_with_follow_redirects(uri) - if response.code == '200' - Sha.verify_digest(response.body, digest_algorithm, sha) - File.write(filename, response.body) - else - str = "Failed to download #{uri} with code #{response.code} error: \n#{response.body}" - raise str - end - end - - def read_file(url) - uri = URI.parse(url) - response = Net::HTTP.get_response(uri) - response.body if response.code == '200' - end - end -end - -module Sha - class << self - def verify_digest(content, algorithm, expected_digest) - file_digest = get_digest(content, algorithm) - raise "sha256 verification failed: expected #{expected_digest}, got #{file_digest}" if expected_digest != file_digest - end - - def get_digest(content, algorithm) - case algorithm - when 'sha256' - Digest::SHA2.new(256).hexdigest(content) - when 'md5' - Digest::MD5.hexdigest(content) - when 'sha1' - Digest::SHA1.hexdigest(content) - else - raise 'Unknown digest algorithm' - end - end - end -end diff --git a/lib/yaml_presenter.rb b/lib/yaml_presenter.rb deleted file mode 100644 index 367ec6be..00000000 --- a/lib/yaml_presenter.rb +++ /dev/null @@ -1,25 +0,0 @@ -# encoding: utf-8 -require 'yaml' -require 'digest' - -class YAMLPresenter - def initialize(recipe) - @recipe = recipe - end - - def to_yaml - @recipe.send(:files_hashs).collect do |file| - if file.has_key?(:git) - { - 'url' => file[:url], - 'git_commit_sha' => file[:git][:commit_sha] - } - else - { - 'url' => file[:url], - 'sha256' => Digest::SHA256.file(file[:local_path]).hexdigest.force_encoding('UTF-8') - } - end - end.to_yaml - end -end diff --git a/recipe/base.rb b/recipe/base.rb deleted file mode 100644 index 5cd9346f..00000000 --- a/recipe/base.rb +++ /dev/null @@ -1,48 +0,0 @@ -# encoding: utf-8 -require 'mini_portile' -require 'net/ftp' -require 'tmpdir' -require 'fileutils' -require_relative 'determine_checksum' -require_relative '../lib/yaml_presenter' - -class BaseRecipe < MiniPortile - def initialize(name, version, options = {}) - super name, version - - options.each do |key, value| - instance_variable_set("@#{key}", value) - end - - @files = [{ - url: url - }.merge(DetermineChecksum.new(options).to_h)] - end - - def configure_options - [] - end - - def compile - execute('compile', [make_cmd, '-j4']) - end - - def archive_filename - "#{name}-#{version}-linux-x64.tgz" - end - - def archive_files - [] - end - - def archive_path_name - '' - end - - private - - # NOTE: https://www.virtualbox.org/ticket/10085 - def tmp_path - "/tmp/#{@host}/ports/#{@name}/#{@version}" - end -end diff --git a/recipe/bundler.rb b/recipe/bundler.rb deleted file mode 100644 index bef9c1b6..00000000 --- a/recipe/bundler.rb +++ /dev/null @@ -1,59 +0,0 @@ -# encoding: utf-8 -require 'mini_portile' -require_relative 'base' - -class BundlerRecipe < BaseRecipe - def url - "https://rubygems.org/downloads/bundler-#{version}.gem" - end - - def cook - download unless downloaded? - extract - compile - end - - def compile - current_dir = ENV['PWD'] - puts current_dir - Dir.mktmpdir("bundler-#{version}") do |tmpdir| - Dir.chdir(tmpdir) do |dir| - FileUtils.rm_rf("#{tmpdir}/*") - - in_gem_env(tmpdir) do - system("unset RUBYOPT; gem install bundler --version #{version} --no-document --env-shebang") - replace_shebangs(tmpdir) - system("rm -f bundler-#{version}.gem") - system("rm -rf cache/bundler-#{version}.gem") - system("tar czvf #{current_dir}/#{archive_filename} .") - puts "#{current_dir}/#{archive_filename}" - end - end - end - end - - def archive_filename - "#{name}-#{version}.tgz" - end - - private - - def in_gem_env(gem_home, &block) - old_gem_home = ENV['GEM_HOME'] - old_gem_path = ENV['GEM_PATH'] - ENV['GEM_HOME'] = ENV['GEM_PATH'] = gem_home.to_s - - yield - - ENV['GEM_HOME'] = old_gem_home - ENV['GEM_PATH'] = old_gem_path - end - - def replace_shebangs(dir) - Dir.glob("#{dir}/bin/*").each do |bin_script| - original_contents = File.read(bin_script) - new_contents = original_contents.gsub(/^#!.*ruby.*/, '#!/usr/bin/env ruby') - File.open(bin_script, 'w') { |file| file.puts(new_contents) } - end - end -end diff --git a/recipe/dep.rb b/recipe/dep.rb deleted file mode 100644 index 6a86ead9..00000000 --- a/recipe/dep.rb +++ /dev/null @@ -1,43 +0,0 @@ -# encoding: utf-8 -require_relative 'base' - -class DepRecipe < BaseRecipe - attr_reader :name, :version - - def cook - download unless downloaded? - extract - - install_go_compiler - - FileUtils.rm_rf("#{tmp_path}/dep") - FileUtils.mv(Dir.glob("#{tmp_path}/dep-*").first, "#{tmp_path}/dep") - Dir.chdir("#{tmp_path}/dep") do - system( - {"GOPATH" => "#{tmp_path}/dep/deps/_workspace:/tmp"}, - "/usr/local/go/bin/go get -asmflags -trimpath ./..." - ) or raise "Could not install dep" - end - FileUtils.mv("#{tmp_path}/dep/LICENSE", "/tmp/LICENSE") - end - - def archive_files - ['/tmp/bin/dep', '/tmp/LICENSE'] - end - - def archive_path_name - 'bin' - end - - def url - "https://github.com/golang/dep/archive/#{version}.tar.gz" - end - - def go_recipe - @go_recipe ||= GoRecipe.new(@name, @version) - end - - def tmp_path - '/tmp/src/github.com/golang' - end -end diff --git a/recipe/determine_checksum.rb b/recipe/determine_checksum.rb deleted file mode 100644 index c6e206c2..00000000 --- a/recipe/determine_checksum.rb +++ /dev/null @@ -1,13 +0,0 @@ -# encoding: utf-8 -class DetermineChecksum - def initialize(options) - @options = options - end - - def to_h - checksum_type = ([:md5, :sha256, :gpg, :git] & @options.keys).first - { - checksum_type => @options[checksum_type] - } - end -end diff --git a/recipe/glide.rb b/recipe/glide.rb deleted file mode 100644 index c439aa36..00000000 --- a/recipe/glide.rb +++ /dev/null @@ -1,46 +0,0 @@ -# encoding: utf-8 -require_relative 'base' - -class GlideRecipe < BaseRecipe - attr_reader :name, :version - - def cook - download unless downloaded? - extract - - install_go_compiler - - FileUtils.rm_rf("#{tmp_path}/glide") - FileUtils.mv(Dir.glob("#{tmp_path}/glide-*").first, "#{tmp_path}/glide") - Dir.chdir("#{tmp_path}/glide") do - system( - {"GOPATH" => "/tmp", - "PATH" => "#{ENV["PATH"]}:/usr/local/go/bin"}, - "/usr/local/go/bin/go build" - ) or raise "Could not install glide" - end - - FileUtils.mv("#{tmp_path}/glide/glide", "/tmp/glide") - FileUtils.mv("#{tmp_path}/glide/LICENSE", "/tmp/LICENSE") - end - - def archive_files - ['/tmp/glide', '/tmp/LICENSE'] - end - - def archive_path_name - 'bin' - end - - def url - "https://github.com/Masterminds/glide/archive/#{version}.tar.gz" - end - - def go_recipe - @go_recipe ||= GoRecipe.new(@name, @version) - end - - def tmp_path - '/tmp/src/github.com/Masterminds' - end -end diff --git a/recipe/go.rb b/recipe/go.rb deleted file mode 100644 index 439a7957..00000000 --- a/recipe/go.rb +++ /dev/null @@ -1,48 +0,0 @@ -# encoding: utf-8 -require_relative 'base' -require_relative '../lib/utils' - -class GoRecipe < BaseRecipe - attr_reader :name, :version - - def cook - download unless downloaded? - extract - - # Installs go1.24.2 to $HOME/go1.24 - go124_sha256 = '68097bd680839cbc9d464a0edce4f7c333975e27a90246890e9f1078c7e702ad' - - Dir.chdir("#{ENV['HOME']}") do - go_download_uri = "https://go.dev/dl/go1.24.2.linux-amd64.tar.gz" - go_tar = "go.tar.gz" - HTTPHelper.download(go_download_uri, go_tar, "sha256", go124_sha256) - - system("tar xf #{go_tar}") - system("mv ./go ./go1.24") - end - - # The GOROOT_BOOTSTRAP defaults to $HOME/go1.4 so we need to update it for this command - Dir.chdir("#{tmp_path}/go/src") do - system( - 'GOROOT_BOOTSTRAP=$HOME/go1.24 ./make.bash' - ) or raise "Could not install go" - end - end - - def archive_files - ["#{tmp_path}/go/*"] - end - - def archive_path_name - 'go' - end - - def archive_filename - "#{name}#{version}.linux-amd64.tar.gz" - end - - def url - "https://go.dev/dl/go#{version}.src.tar.gz" - end - -end diff --git a/recipe/godep.rb b/recipe/godep.rb deleted file mode 100644 index 20d87ac9..00000000 --- a/recipe/godep.rb +++ /dev/null @@ -1,43 +0,0 @@ -# encoding: utf-8 -require_relative 'base' - -class GodepMeal < BaseRecipe - attr_reader :name, :version - - def cook - download unless downloaded? - extract - - install_go_compiler - - FileUtils.rm_rf("#{tmp_path}/godep") - FileUtils.mv(Dir.glob("#{tmp_path}/godep-*").first, "#{tmp_path}/godep") - Dir.chdir("#{tmp_path}/godep") do - system( - {"GOPATH" => "#{tmp_path}/godep/Godeps/_workspace:/tmp"}, - "/usr/local/go/bin/go get ./..." - ) or raise "Could not install godep" - end - FileUtils.mv("#{tmp_path}/godep/License", "/tmp/License") - end - - def archive_files - ['/tmp/bin/godep', '/tmp/License'] - end - - def archive_path_name - 'bin' - end - - def url - "https://github.com/tools/godep/archive/#{version}.tar.gz" - end - - def go_recipe - @go_recipe ||= GoRecipe.new(@name, @version) - end - - def tmp_path - '/tmp/src/github.com/tools' - end -end diff --git a/recipe/httpd_meal.rb b/recipe/httpd_meal.rb deleted file mode 100644 index 4e8f93d5..00000000 --- a/recipe/httpd_meal.rb +++ /dev/null @@ -1,234 +0,0 @@ -# encoding: utf-8 -require_relative 'base' - -class AprRecipe < BaseRecipe - def url - "https://archive.apache.org/dist/apr/apr-#{version}.tar.gz" - end -end - -class AprIconvRecipe < BaseRecipe - def configure_options - [ - "--with-apr=#{@apr_path}/bin/apr-1-config" - ] - end - - def url - "https://archive.apache.org/dist/apr/apr-iconv-#{version}.tar.gz" - end -end - -class AprUtilRecipe < BaseRecipe - def configure_options - [ - "--with-apr=#{@apr_path}", - "--with-iconv=#{@apr_iconv_path}", - '--with-crypto', - '--with-openssl', - '--with-mysql', - '--with-pgsql', - '--with-gdbm', - '--with-ldap' - ] - end - - def url - "https://archive.apache.org/dist/apr/apr-util-#{version}.tar.gz" - end -end - -class HTTPdRecipe < BaseRecipe - def computed_options - [ - '--prefix=/app/httpd', - "--with-apr=#{@apr_path}", - "--with-apr-util=#{@apr_util_path}", - '--with-ssl=/usr/lib/x86_64-linux-gnu', - '--enable-mpms-shared=worker event', - '--enable-mods-shared=reallyall', - '--disable-isapi', - '--disable-dav', - '--disable-dialup' - ] - end - - def install - return if installed? - execute('install', [make_cmd, 'install', "prefix=#{path}"]) - end - - def url - "https://archive.apache.org/dist/httpd/httpd-#{version}.tar.bz2" - end - - def archive_files - ["#{path}/*"] - end - - def archive_path_name - 'httpd' - end - - def setup_tar - system <<-eof - cd #{path} - - rm -rf cgi-bin/ error/ icons/ include/ man/ manual/ htdocs/ - rm -rf conf/extra/* conf/httpd.conf conf/httpd.conf.bak conf/magic conf/original - - mkdir -p lib - cp "#{@apr_path}/lib/libapr-1.so.0" ./lib - cp "#{@apr_util_path}/lib/libaprutil-1.so.0" ./lib - mkdir -p "./lib/apr-util-1" - cp "#{@apr_util_path}/lib/apr-util-1/"*.so ./lib/apr-util-1/ - mkdir -p "./lib/iconv" - cp "#{@apr_iconv_path}/lib/libapriconv-1.so.0" ./lib - cp "#{@apr_iconv_path}/lib/iconv/"*.so ./lib/iconv/ - cp /usr/lib/x86_64-linux-gnu/libcjose.so* ./lib/ - cp /usr/lib/x86_64-linux-gnu/libhiredis.so* ./lib/ - cp /usr/lib/x86_64-linux-gnu/libjansson.so* ./lib/ - eof - end -end - -class ModAuthOpenidcRecipe < BaseRecipe - def url - "https://github.com/zmartzone/mod_auth_openidc/releases/download/v#{version}/mod_auth_openidc-#{version}.tar.gz" - end - - def configure_options - ENV['APR_LIBS'] = `#{@apr_path}/bin/apr-1-config --link-ld --libs` - ENV['APR_CFLAGS'] = `#{@apr_path}/bin/apr-1-config --cflags --includes` - [ - "--with-apxs2=#{@httpd_path}/bin/apxs" - ] - end -end - -class HTTPdMeal - attr_reader :name, :version - - def initialize(name, version, options = {}) - @name = name - @version = version - @options = options - update_git - end - - def cook - run('mkdir /app') - run('apt update') or raise 'Failed to apt update' - run('apt-get install -y libldap2-dev') or raise 'Failed to install libldap2-dev' - - apr_recipe.cook - apr_iconv_recipe.cook - apr_util_recipe.cook - httpd_recipe.cook - - # this symlink is needed so that modules can call `apxs` - # putting it here because we only need to do it once - system <<-eof - cd /app - if ! [ -L "/app/httpd" ]; then - ln -s "#{httpd_recipe.path}" httpd - fi - eof - - run('apt-get install -y libjansson-dev libcjose-dev libhiredis-dev') or raise 'Failed to install additional dependencies' - mod_auth_openidc_recipe.cook - end - - def url - httpd_recipe.url - end - - def archive_files - httpd_recipe.archive_files - end - - def archive_path_name - httpd_recipe.archive_path_name - end - - def archive_filename - httpd_recipe.archive_filename - end - - def setup_tar - httpd_recipe.setup_tar - end - - private - - def run(command) - output = `#{command}` - if $?.success? - return true - else - STDOUT.puts "ERROR, output was:" - STDOUT.puts output - return false - end - end - - def latest_github_version(repo) - puts "Getting latest tag from #{repo}..." - repo = "https://github.com/#{repo}" - return `git -c 'versionsort.suffix=-' ls-remote --exit-code --refs --sort='version:refname' --tags #{repo} '*.*.*' | tail -1 | cut -d/ --fields=3`.strip - end - - def update_git - # This is done because we rely on git's ls-remote that was introduced in 2.18. - # cflinuxfs3/bionic comes with an older version of git. - puts "Updating git to latest version..." - system <<-EOF - #!/bin/sh - apt-get -y update - apt-get -y install software-properties-common - add-apt-repository -y ppa:git-core/ppa - apt-get -y update && apt-get -y install git - EOF - end - - def files_hashs - hashes = httpd_recipe.send(:files_hashs) + - apr_recipe.send(:files_hashs) + - apr_iconv_recipe.send(:files_hashs) + - apr_util_recipe.send(:files_hashs) + - mod_auth_openidc_recipe.send(:files_hashs) - - hashes - end - - def mod_auth_openidc_recipe - @mod_auth_openidc ||= ModAuthOpenidcRecipe.new('mod-auth-openidc', '2.3.8', - httpd_path: httpd_recipe.path, - apr_path: apr_recipe.path, - md5: 'd6abc2f68dabf5d2557400af2499f500') - end - - def httpd_recipe - @http_recipe ||= HTTPdRecipe.new(@name, @version, { - apr_path: apr_recipe.path, - apr_util_path: apr_util_recipe.path, - apr_iconv_path: apr_iconv_recipe.path - }.merge(DetermineChecksum.new(@options).to_h)) - end - - def apr_util_recipe - apr_util_version = latest_github_version("apache/apr-util") - @apr_util_recipe ||= AprUtilRecipe.new('apr-util', apr_util_version, apr_path: apr_recipe.path, - apr_iconv_path: apr_iconv_recipe.path) - end - - def apr_iconv_recipe - apr_iconv_version = latest_github_version("apache/apr-iconv") - @apr_iconv_recipe ||= AprIconvRecipe.new('apr-iconv', apr_iconv_version, apr_path: apr_recipe.path) - end - - def apr_recipe - apr_version = latest_github_version("apache/apr") - @apr_recipe ||= AprRecipe.new('apr', apr_version) - end -end diff --git a/recipe/hwc.rb b/recipe/hwc.rb deleted file mode 100644 index c4df3486..00000000 --- a/recipe/hwc.rb +++ /dev/null @@ -1,54 +0,0 @@ -# encoding: utf-8 -require_relative 'base' -require_relative '../lib/install_go' -require 'yaml' -require 'digest' - -class HwcRecipe < BaseRecipe - attr_reader :name, :version - - def cook - download unless downloaded? - extract - - install_go_compiler - - system <<-eof - sudo apt-get update - sudo apt-get -y upgrade - sudo apt-get -y install mingw-w64 - eof - - FileUtils.rm_rf("#{tmp_path}/hwc") - FileUtils.mv(Dir.glob("#{tmp_path}/hwc-*").first, "#{tmp_path}/hwc") - Dir.chdir("#{tmp_path}/hwc") do - system( - { 'PATH' => "#{ENV['PATH']}:/usr/local/go/bin" }, - "./bin/release-binaries.bash amd64 windows #{version} #{tmp_path}/hwc" - ) or raise 'Could not build hwc amd64' - system( - { 'PATH' => "#{ENV['PATH']}:/usr/local/go/bin" }, - "./bin/release-binaries.bash 386 windows #{version} #{tmp_path}/hwc" - ) or raise 'Could not build hwc 386' - end - - FileUtils.mv("#{tmp_path}/hwc/hwc-windows-amd64", '/tmp/hwc.exe') - FileUtils.mv("#{tmp_path}/hwc/hwc-windows-386", '/tmp/hwc_x86.exe') - end - - def archive_files - ['/tmp/hwc.exe', '/tmp/hwc_x86.exe'] - end - - def url - "https://github.com/cloudfoundry/hwc/archive/#{version}.tar.gz" - end - - def tmp_path - '/tmp/src/code.cloudfoundry.org' - end - - def archive_filename - "#{name}-#{version}-windows-x86-64.zip" - end -end diff --git a/recipe/jruby.rb b/recipe/jruby.rb deleted file mode 100644 index 0e436f2b..00000000 --- a/recipe/jruby.rb +++ /dev/null @@ -1,45 +0,0 @@ -# encoding: utf-8 -require 'mini_portile' -require_relative 'base' - -class JRubyRecipe < BaseRecipe - def archive_files - [ - "#{work_path}/bin", - "#{work_path}/lib" - ] - end - - def url - "https://repo1.maven.org/maven2/org/jruby/jruby-dist/#{jruby_version}/jruby-dist-#{jruby_version}-src.zip" - end - - def cook - download unless downloaded? - extract_zip - compile - end - - def compile - execute('compile', ['mvn', '-P', '!truffle', "-Djruby.default.ruby.version=#{ruby_version}"]) - end - - def extract_zip - files_hashs.each do |file| - verify_file(file) - - filename = File.basename(file[:local_path]) - message "Unzipping #{filename} into #{tmp_path}... " - FileUtils.mkdir_p tmp_path - execute('unzip', ["unzip", "-o", file[:local_path], "-d", tmp_path], {:cd => Dir.pwd, :initial_message => false}) - end - end - - def ruby_version - @ruby_version ||= version.match(/.*-ruby-(\d+\.\d+)/)[1] - end - - def jruby_version - @jruby_version ||= version.match(/(.*)-ruby-\d+\.\d+/)[1] - end -end diff --git a/recipe/jruby_meal.rb b/recipe/jruby_meal.rb deleted file mode 100644 index 329310c9..00000000 --- a/recipe/jruby_meal.rb +++ /dev/null @@ -1,77 +0,0 @@ -# encoding: utf-8 -require_relative 'jruby' -require_relative 'maven' -require 'fileutils' -require 'digest' - -class JRubyMeal - attr_reader :name, :version - - def initialize(name, version, options = {}) - @name = name - @version = version - @options = options - end - - def cook - # We compile against the OpenJDK8 that the java buildpack team builds - # This is the openjdk-jdk that contains the openjdk-jre used in the ruby buildpack - # Ubuntu Trusty itself does not provide openjdk 8 - - java_jdk_dir = '/opt/java' - java_jdk_tar_file = File.join(java_jdk_dir, 'openjdk-8-jdk.tar.gz') - java_jdk_bin_dir = File.join(java_jdk_dir, 'bin') - java_jdk_sha256 = '1210b14353f9a911aad84182e36be229a30991c5ca6b8b1a932aad426e106276' - java_buildpack_java_sdk = "https://java-buildpack.cloudfoundry.org/openjdk-jdk/trusty/x86_64/openjdk-jdk-1.8.0_222-trusty.tar.gz" - - FileUtils.mkdir_p(java_jdk_dir) - raise "Downloading openjdk-8-jdk failed." unless system("wget #{java_buildpack_java_sdk} -O #{java_jdk_tar_file}") - - downloaded_sha = Digest::SHA256.file(java_jdk_tar_file).hexdigest - - if java_jdk_sha256 != downloaded_sha - raise "sha256 verification failed: expected #{java_jdk_sha256}, got #{downloaded_sha}" - end - - raise "Untarring openjdk-8-jdk failed." unless system("tar xvf #{java_jdk_tar_file} -C #{java_jdk_dir}") - - ENV['JAVA_HOME'] = java_jdk_dir - ENV['PATH'] = "#{ENV['PATH']}:#{java_jdk_bin_dir}" - - maven.cook - maven.activate - - jruby.cook - end - - def url - jruby.url - end - - def archive_files - jruby.archive_files - end - - def archive_path_name - jruby.archive_path_name - end - - def archive_filename - jruby.archive_filename - end - - private - - def files_hashs - maven.send(:files_hashs) + - jruby.send(:files_hashs) - end - - def jruby - @jruby ||= JRubyRecipe.new(@name, @version, @options) - end - - def maven - @maven ||= MavenRecipe.new('maven', '3.6.3', md5: '42e4430d19894524e5d026217f2b3ecd') - end -end diff --git a/recipe/maven.rb b/recipe/maven.rb deleted file mode 100644 index b66efd34..00000000 --- a/recipe/maven.rb +++ /dev/null @@ -1,47 +0,0 @@ -# encoding: utf-8 -require_relative 'base' - -class MavenRecipe < BaseRecipe - def url - "https://archive.apache.org/dist/maven/maven-3/#{version}/source/apache-maven-#{version}-src.tar.gz" - end - - def cook - download unless downloaded? - extract - - #install maven 3.6.3 to $HOME/apache-maven-3.6.3 - sha512 = 'c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0' - - Dir.chdir("#{ENV['HOME']}") do - maven_download = "https://archive.apache.org/dist/maven/maven-3/#{version}/binaries/apache-maven-#{version}-bin.tar.gz" - maven_tar = "apache-maven-#{version}-bin.tar.gz" - system("curl -L #{maven_download} -o #{maven_tar}") - - downloaded_sha = Digest::SHA512.file(maven_tar).hexdigest - - if sha512 != downloaded_sha - raise "sha512 verification failed: expected #{sha512}, got #{downloaded_sha}" - end - - system("tar xf #{maven_tar}") - end - - old_path = ENV['PATH'] - ENV['PATH'] = "#{ENV['HOME']}/apache-maven-3.6.3/bin:#{old_path}" - - install - ENV['PATH'] = old_path - FileUtils.rm_rf(File.join(ENV['HOME'], 'apache-maven-3.6.3')) - end - - def install - FileUtils.rm_rf(path) - execute('install', [ - 'mvn', - "-DdistributionTargetDir=#{path}", - 'clean', - 'package' - ]) - end -end diff --git a/recipe/nginx.rb b/recipe/nginx.rb deleted file mode 100644 index 9c118ccf..00000000 --- a/recipe/nginx.rb +++ /dev/null @@ -1,54 +0,0 @@ -# encoding: utf-8 -require_relative '../lib/openssl_replace' -require_relative 'base' - -class NginxRecipe < BaseRecipe - - def initialize(name, version, options = {}) - super name, version, options - # override openssl in container - OpenSSLReplace.replace_openssl - end - - def computed_options - [ - '--prefix=/', - '--error-log-path=stderr', - '--with-http_ssl_module', - '--with-http_realip_module', - '--with-http_gunzip_module', - '--with-http_gzip_static_module', - '--with-http_auth_request_module', - '--with-http_random_index_module', - '--with-http_secure_link_module', - '--with-http_stub_status_module', - '--without-http_uwsgi_module', - '--without-http_scgi_module', - '--with-pcre', - '--with-pcre-jit', - '--with-cc-opt=-fPIE -pie', - '--with-ld-opt=-fPIE -pie -z now', - ] - end - - def install - return if installed? - execute('install', [make_cmd, 'install', "DESTDIR=#{path}"]) - end - - def archive_files - ["#{path}/*"] - end - - def archive_path_name - 'nginx' - end - - def setup_tar - `rm -Rf #{path}/html/ #{path}/conf/*` - end - - def url - "http://nginx.org/download/nginx-#{version}.tar.gz" - end -end diff --git a/recipe/node.rb b/recipe/node.rb deleted file mode 100644 index e5ade38b..00000000 --- a/recipe/node.rb +++ /dev/null @@ -1,42 +0,0 @@ -# encoding: utf-8 -require 'mini_portile' -require 'fileutils' -require_relative 'base' - -class NodeRecipe < BaseRecipe - - def computed_options - if Gem::Version.new(version) >= Gem::Version.new('6.0.0') - %w[--prefix=/ --openssl-use-def-ca-store] - else - ['--prefix=/'] - end - end - - def install - execute('install', [make_cmd, 'install', "DESTDIR=#{dest_dir}", 'PORTABLE=1']) - end - - def archive_files - [dest_dir] - end - - def setup_tar - FileUtils.cp( - "#{work_path}/LICENSE", - dest_dir - ) - end - - def url - "https://nodejs.org/dist/v#{version}/node-v#{version}.tar.gz" - end - - def dest_dir - "/tmp/node-v#{version}-linux-x64" - end - - def configure - execute('configure', %w(python3 configure) + computed_options) - end -end diff --git a/recipe/php_common_recipes.rb b/recipe/php_common_recipes.rb deleted file mode 100644 index 9a65657f..00000000 --- a/recipe/php_common_recipes.rb +++ /dev/null @@ -1,539 +0,0 @@ -# encoding: utf-8 -require_relative 'base' -require_relative '../lib/geoip_downloader' -require 'uri' - -class BasePHPModuleRecipe < BaseRecipe - def initialize(name, version, options = {}) - super name, version, options - - @files = [{ - url: url, - local_path: local_path, - }.merge(DetermineChecksum.new(options).to_h)] - end - - def local_path - File.join(archives_path, File.basename(url)) - end - - # override this method to allow local_path to be specified - # this prevents recipes with the same versions downloading colliding files (such as `v1.0.0.tar.gz`) - def files_hashs - @files.map do |file| - hash = case file - when String - { :url => file } - when Hash - file.dup - else - raise ArgumentError, "files must be an Array of Stings or Hashs" - end - - hash[:local_path] = local_path - hash - end - end -end - -class PeclRecipe < BasePHPModuleRecipe - def url - "http://pecl.php.net/get/#{name}-#{version}.tgz" - end - - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config" - ] - end - - def configure - return if configured? - - md5_file = File.join(tmp_path, 'configure.md5') - digest = Digest::MD5.hexdigest(computed_options.to_s) - File.open(md5_file, 'w') { |f| f.write digest } - - execute('phpize', 'phpize') - execute('configure', %w(sh configure) + computed_options) - end -end - -class AmqpPeclRecipe < PeclRecipe - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config" - ] - end -end - -class PkgConfigLibRecipe < BasePHPModuleRecipe - def cook - exists = system("PKG_CONFIG_PATH=$PKG_CONFIG_PATH:#{pkg_path} pkg-config #{pkgcfg_name} --exists") - if ! exists - super() - end - end - - def pkg_path - "#{File.expand_path(port_path)}/lib/pkgconfig/" - end -end - -class MaxMindRecipe < PeclRecipe - def work_path - File.join(tmp_path, "maxminddb-#{version}", 'ext') - end -end - -class HiredisRecipe < PkgConfigLibRecipe - def url - "https://github.com/redis/hiredis/archive/v#{version}.tar.gz" - end - - def local_path - "hiredis-#{version}.tar.gz" - end - - def configure - end - - def install - return if installed? - - execute('install', ['bash', '-c', "LIBRARY_PATH=lib PREFIX='#{path}' #{make_cmd} install"]) - end - - def pkgcfg_name - "hiredis" - end -end - -class LibSodiumRecipe < PkgConfigLibRecipe - def url - "https://github.com/jedisct1/libsodium/archive/#{version}-RELEASE.tar.gz" - end - - def pkgcfg_name - "libsodium" - end -end - -class IonCubeRecipe < BaseRecipe - def url - "http://downloads3.ioncube.com/loader_downloads/ioncube_loaders_lin_x86-64_#{version}.tar.gz" - end - - def configure; end - - def compile; end - - def install; end - - def self.build_ioncube?(php_version) - true - end - - def path - tmp_path - end -end - -class LibRdKafkaRecipe < PkgConfigLibRecipe - def url - "https://github.com/edenhill/librdkafka/archive/v#{version}.tar.gz" - end - - def pkgcfg_name - "rdkafka" - end - - def local_path - "librdkafka-#{version}.tar.gz" - end - - def work_path - File.join(tmp_path, "librdkafka-#{version}") - end - - def configure_prefix - '--prefix=/usr' - end - - def configure - return if configured? - - md5_file = File.join(tmp_path, 'configure.md5') - digest = Digest::MD5.hexdigest(computed_options.to_s) - File.open(md5_file, 'w') { |f| f.write digest } - - execute('configure', %w(bash ./configure) + computed_options) - end -end - -class CassandraCppDriverRecipe < PkgConfigLibRecipe - def url - "https://github.com/datastax/cpp-driver/archive/#{version}.tar.gz" - end - - def pkgcfg_name - "cassandra" - end - - def local_path - "cassandra-cpp-driver-#{version}.tar.gz" - end - - def configure - end - - def compile - execute('compile', ['bash', '-c', 'mkdir -p build && cd build && cmake .. && make']) - end - - def install - execute('install', ['bash', '-c', 'cd build && make install']) - end -end - -class LuaPeclRecipe < PeclRecipe - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config", - "--with-lua=#{@lua_path}" - ] - end -end - -class LuaRecipe < BaseRecipe - def url - "http://www.lua.org/ftp/lua-#{version}.tar.gz" - end - - def configure - end - - def compile - execute('compile', ['bash', '-c', "#{make_cmd} linux MYCFLAGS=-fPIC"]) - end - - def install - return if installed? - - execute('install', ['bash', '-c', "#{make_cmd} install INSTALL_TOP=#{path}"]) - end -end - -class MemcachedPeclRecipe < PeclRecipe - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config", - "--with-libmemcached-dir", - '--enable-memcached-sasl', - '--enable-memcached-msgpack', - '--enable-memcached-igbinary', - '--enable-memcached-json' - ] - end -end - -class FakePeclRecipe < PeclRecipe - def url - "file://#{@php_source}/ext/#{name}-#{version}.tar.gz" - end - - def download - # this copys an extension folder out of the PHP source director (i.e. `ext/`) - # it pretends to download it by making a zip of the extension files - # that way the rest of the PeclRecipe works normally - files_hashs.each do |file| - path = URI(file[:url]).path.rpartition('-')[0] # only need path before the `-`, see url above - system <<-eof - tar czf "#{file[:local_path]}" -C "#{File.dirname(path)}" "#{File.basename(path)}" - eof - end - end -end - - -class Gd74FakePeclRecipe < FakePeclRecipe - # how to build gd.so in PHP 7.4 changed dramatically - # In 7.4+, you can just use libgd from Ubuntu - def configure_options - [ - "--with-external-gd" - ] - end -end - -class OdbcRecipe < FakePeclRecipe - def configure_options - [ - "--with-unixODBC=shared,/usr" - ] - end - - def patch - system <<-eof - cd #{work_path} - echo 'AC_DEFUN([PHP_ALWAYS_SHARED],[])dnl' > temp.m4 - echo >> temp.m4 - cat config.m4 >> temp.m4 - mv temp.m4 config.m4 - eof - end - - def setup_tar - system <<-eof - cp -a /usr/lib/x86_64-linux-gnu/libodbc.so* #{@php_path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libodbcinst.so* #{@php_path}/lib/ - eof - end -end - -class SodiumRecipe < FakePeclRecipe - def configure_options - ENV['LDFLAGS'] = "-L#{@libsodium_path}/lib" - ENV['PKG_CONFIG_PATH'] = "#{@libsodium_path}/lib/pkgconfig" - sodium_flag = "--with-sodium=#{@libsodium_path}" - [ - "--with-php-config=#{@php_path}/bin/php-config", - sodium_flag - ] - end - - def setup_tar - system <<-eof - cp -a #{@libsodium_path}/lib/libsodium.so* #{@php_path}/lib/ - eof - end -end - -class PdoOdbcRecipe < FakePeclRecipe - def configure_options - [ - "--with-pdo-odbc=unixODBC,/usr" - ] - end - - def setup_tar - system <<-eof - cp -a /usr/lib/x86_64-linux-gnu/libodbc.so* #{@php_path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libodbcinst.so* #{@php_path}/lib/ - eof - end - -end - -class OraclePdoRecipe < FakePeclRecipe - def configure_options - [ - "--with-pdo-oci=shared,instantclient,/oracle,#{OraclePdoRecipe.oracle_version}" - ] - end - - def self.oracle_version - Dir["/oracle/*"].select {|i| i.match(/libclntsh\.so\./) }.map {|i| i.sub(/.*libclntsh\.so\./, '')}.first - end - - def setup_tar - system <<-eof - cp -an /oracle/libclntshcore.so.12.1 #{@php_path}/lib - cp -an /oracle/libclntsh.so #{@php_path}/lib - cp -an /oracle/libclntsh.so.12.1 #{@php_path}/lib - cp -an /oracle/libipc1.so #{@php_path}/lib - cp -an /oracle/libmql1.so #{@php_path}/lib - cp -an /oracle/libnnz12.so #{@php_path}/lib - cp -an /oracle/libociicus.so #{@php_path}/lib - cp -an /oracle/libons.so #{@php_path}/lib - eof - end -end - -class OraclePeclRecipe < PeclRecipe - def configure_options - [ - "--with-oci8=shared,instantclient,/oracle" - ] - end - - def self.oracle_sdk? - File.directory?('/oracle') - end - - def setup_tar - system <<-eof - cp -an /oracle/libclntshcore.so.12.1 #{@php_path}/lib - cp -an /oracle/libclntsh.so #{@php_path}/lib - cp -an /oracle/libclntsh.so.12.1 #{@php_path}/lib - cp -an /oracle/libipc1.so #{@php_path}/lib - cp -an /oracle/libmql1.so #{@php_path}/lib - cp -an /oracle/libnnz12.so #{@php_path}/lib - cp -an /oracle/libociicus.so #{@php_path}/lib - cp -an /oracle/libons.so #{@php_path}/lib - eof - end -end - -class PHPIRedisRecipe < PeclRecipe - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config", - '--enable-phpiredis', - "--with-hiredis-dir=#{@hiredis_path}" - ] - end - - def url - "https://github.com/nrk/phpiredis/archive/v#{version}.tar.gz" - end - - def local_path - "phpiredis-#{version}.tar.gz" - end -end - -class RedisPeclRecipe < PeclRecipe - def configure_options - [ - "--with-php-config=#{@php_path}/bin/php-config", - "--enable-redis-igbinary", - "--enable-redis-lzf", - "--with-liblzf=no" - ] - end -end - -# TODO: Remove after PHP 7 is out of support -class PHPProtobufPeclRecipe < PeclRecipe - def url - "https://github.com/allegro/php-protobuf/archive/v#{version}.tar.gz" - end - - def local_path - "php-protobuf-#{version}.tar.gz" - end -end - -class TidewaysXhprofRecipe < PeclRecipe - def url - "https://github.com/tideways/php-xhprof-extension/archive/v#{version}.tar.gz" - end - - def local_path - "tideways-xhprof-#{version}.tar.gz" - end -end - -class EnchantFakePeclRecipe < FakePeclRecipe - def patch - super - system <<-eof - cd #{work_path} - sed -i 's|#include "../spl/spl_exceptions.h"|#include |' enchant.c - eof - end -end - -class RabbitMQRecipe < PkgConfigLibRecipe - def url - "https://github.com/alanxz/rabbitmq-c/archive/v#{version}.tar.gz" - end - - def pkgcfg_name - "librabbitmq" - end - - def local_path - "rabbitmq-#{version}.tar.gz" - end - - def work_path - File.join(tmp_path, "rabbitmq-c-#{@version}") - end - - def configure - end - - def compile - execute('compile', ['bash', '-c', 'cmake .']) - execute('compile', ['bash', '-c', 'cmake --build .']) - execute('compile', ['bash', '-c', 'cmake -DCMAKE_INSTALL_PREFIX=/usr/local .']) - execute('compile', ['bash', '-c', 'cmake --build . --target install']) - end -end - -class SnmpRecipe - attr_reader :name, :version - - def initialize(name, version, options) - @name = name - @version = version - @options = options - end - - def files_hashs - [] - end - - def cook - system <<-eof - cd #{@php_path} - mkdir -p mibs - cp "/usr/lib/x86_64-linux-gnu/libnetsnmp.so.30" lib/ - # copy mibs that are packaged freely - cp -r /usr/share/snmp/mibs/* mibs - # copy mibs downloader & smistrip, will download un-free mibs - cp /usr/bin/download-mibs bin - cp /usr/bin/smistrip bin - sed -i "s|^CONFDIR=/etc/snmp-mibs-downloader|CONFDIR=\$HOME/php/mibs/conf|" bin/download-mibs - sed -i "s|^SMISTRIP=/usr/bin/smistrip|SMISTRIP=\$HOME/php/bin/smistrip|" bin/download-mibs - # copy mibs download config - cp -R /etc/snmp-mibs-downloader mibs/conf - sed -i "s|^DIR=/usr/share/doc|DIR=\$HOME/php/mibs/originals|" mibs/conf/iana.conf - sed -i "s|^DEST=iana|DEST=|" mibs/conf/iana.conf - sed -i "s|^DIR=/usr/share/doc|DIR=\$HOME/php/mibs/originals|" mibs/conf/ianarfc.conf - sed -i "s|^DEST=iana|DEST=|" mibs/conf/ianarfc.conf - sed -i "s|^DIR=/usr/share/doc|DIR=\$HOME/php/mibs/originals|" mibs/conf/rfc.conf - sed -i "s|^DEST=ietf|DEST=|" mibs/conf/rfc.conf - sed -i "s|^BASEDIR=/var/lib/mibs|BASEDIR=\$HOME/php/mibs|" mibs/conf/snmp-mibs-downloader.conf - # copy data files - # TODO: these are gone or have moved, commenting out for now - # mkdir mibs/originals - # cp -R /usr/share/doc/mibiana mibs/originals - # cp -R /usr/share/doc/mibrfcs mibs/originals - eof - end -end - -class SuhosinPeclRecipe < PeclRecipe - def url - "https://github.com/sektioneins/suhosin/archive/#{version}.tar.gz" - end -end - -class TwigPeclRecipe < PeclRecipe - def url - "https://github.com/twigphp/Twig/archive/v#{version}.tar.gz" - end - - def work_path - "#{super}/ext/twig" - end -end - -class XcachePeclRecipe < PeclRecipe - def url - "http://xcache.lighttpd.net/pub/Releases/#{version}/xcache-#{version}.tar.gz" - end -end - -class XhprofPeclRecipe < PeclRecipe - def url - "https://github.com/phacility/xhprof/archive/#{version}.tar.gz" - end - - def work_path - "#{super}/extension" - end -end diff --git a/recipe/php_meal.rb b/recipe/php_meal.rb deleted file mode 100644 index 790f61b0..00000000 --- a/recipe/php_meal.rb +++ /dev/null @@ -1,242 +0,0 @@ -# encoding: utf-8 -require_relative 'php_common_recipes' -require_relative 'php_recipe' - -class PhpMeal - attr_reader :name, :version - - def initialize(name, version, options) - @name = name - @version = version - version_parts = version.split('.') - @major_version = version_parts[0] - @minor_version = version_parts[1] - @options = options - @native_modules = [] - @extensions = [] - - create_native_module_recipes - create_extension_recipes - - (@native_modules + @extensions).each do |recipe| - recipe.instance_variable_set('@php_path', php_recipe.path) - recipe.instance_variable_set('@php_source', "#{php_recipe.send(:tmp_path)}/php-#{@version}") - - if recipe.is_a? FakePeclRecipe - recipe.instance_variable_set('@version', @version) - recipe.instance_variable_set('@files', [{url: recipe.url, md5: nil}]) - end - end - end - - def cook - system <<-eof - DIFF=$(expr $(date +'%s') - $(date -r /tmp/apt-last-updated +'%s')) - if [ -z $DIFF ] || [ $DIFF -gt 86400 ]; then - apt-get update - apt-get -y upgrade - apt-get -y install #{apt_packages} - - # because default bionic gives us an old uncompatible libtidy - apt-get -y install software-properties-common - add-apt-repository -y ppa:savoury1/backports - apt-get update - apt-get install -y libtidy-dev - - touch /tmp/apt-last-updated - fi - #{install_libuv} - #{symlink_commands} - eof - - if OraclePeclRecipe.oracle_sdk? - Dir.chdir('/oracle') do - system "ln -s libclntsh.so.* libclntsh.so" - end - end - - php_recipe.cook - php_recipe.activate - - # native libraries - @native_modules.each do |recipe| - recipe.cook - end - - # php extensions - @extensions.each do |recipe| - recipe.cook if should_cook?(recipe) - end - end - - def url - php_recipe.url - end - - def archive_files - php_recipe.archive_files - end - - def archive_path_name - php_recipe.archive_path_name - end - - def archive_filename - php_recipe.archive_filename - end - - def setup_tar - php_recipe.setup_tar - if OraclePeclRecipe.oracle_sdk? - @extensions.detect{|r| r.name=='oci8'}.setup_tar - @extensions.detect{|r| r.name=='pdo_oci'}.setup_tar - end - @extensions.detect{|r| r.name=='odbc'}&.setup_tar - @extensions.detect{|r| r.name=='pdo_odbc'}&.setup_tar - @extensions.detect{|r| r.name=='sodium'}&.setup_tar - end - - private - - def create_native_module_recipes - return unless @options[:php_extensions_file] - php_extensions_hash = YAML.safe_load_file(@options[:php_extensions_file]) - - php_extensions_hash['native_modules'].each do |hash| - klass = Kernel.const_get(hash['klass']) - - @native_modules << klass.new( - hash['name'], - hash['version'], - md5: hash['md5'] - ) - end - end - - def create_extension_recipes - return unless @options[:php_extensions_file] - php_extensions_hash = YAML.safe_load_file(@options[:php_extensions_file]) - - php_extensions_hash['extensions'].each do |hash| - klass = Kernel.const_get(hash['klass']) - - @extensions << klass.new( - hash['name'], - hash['version'], - md5: hash['md5'] - ) - end - - @extensions.each do |recipe| - case recipe.name - when 'amqp' - recipe.instance_variable_set('@rabbitmq_path', @native_modules.detect{|r| r.name=='rabbitmq'}.work_path) - when 'lua' - recipe.instance_variable_set('@lua_path', @native_modules.detect{|r| r.name=='lua'}.path) - when 'phpiredis' - recipe.instance_variable_set('@hiredis_path', @native_modules.detect{|r| r.name=='hiredis'}.path) - when 'sodium' - recipe.instance_variable_set('@libsodium_path', @native_modules.detect{|r| r.name=='libsodium'}.path) - end - end - end - - def apt_packages - %w(automake - firebird-dev - libaspell-dev - libc-client2007e-dev - libcurl4-openssl-dev - libedit-dev - libenchant-dev - libexpat1-dev - libgd-dev - libgdbm-dev - libgeoip-dev - libgmp-dev - libgpgme11-dev - libjpeg-dev - libkrb5-dev - libldap2-dev - libmaxminddb-dev - libmcrypt-dev - libmemcached-dev - libonig-dev - libpng-dev - libpspell-dev - librecode-dev - libsasl2-dev - libsnmp-dev - libsqlite3-dev - libssh2-1-dev - libssl-dev - libtool - libwebp-dev - libxml2-dev - libzip-dev - libzookeeper-mt-dev - snmp-mibs-downloader - unixodbc-dev).join(" ") - end - - def install_libuv - %q(( - if [ "$(pkg-config libuv --print-provides | awk '{print $3}')" != "1.12.0" ]; then - cd /tmp - wget http://dist.libuv.org/dist/v1.12.0/libuv-v1.12.0.tar.gz - tar zxf libuv-v1.12.0.tar.gz - cd libuv-v1.12.0 - sh autogen.sh - ./configure - make install - fi - ) - ) - end - - def symlink_commands - [ "sudo ln -s /usr/include/x86_64-linux-gnu/curl /usr/local/include/curl", - "sudo ln -fs /usr/include/x86_64-linux-gnu/gmp.h /usr/include/gmp.h", - "sudo ln -fs /usr/lib/x86_64-linux-gnu/libldap.so /usr/lib/libldap.so", - "sudo ln -fs /usr/lib/x86_64-linux-gnu/libldap_r.so /usr/lib/libldap_r.so"].join("\n") - end - - def should_cook?(recipe) - case recipe.name - when 'ioncube' - IonCubeRecipe.build_ioncube?(version) - when 'oci8', 'pdo_oci' - OraclePeclRecipe.oracle_sdk? - else - true - end - end - - def files_hashs - native_module_hashes = @native_modules.map do |recipe| - recipe.send(:files_hashs) - end.flatten - - extension_hashes = @extensions.map do |recipe| - recipe.send(:files_hashs) if should_cook?(recipe) - end.flatten.compact - - extension_hashes + native_module_hashes - end - - def php_recipe - php_recipe_options = {} - - hiredis_recipe = @native_modules.detect{|r| r.name=='hiredis'} - libmemcached_recipe = @native_modules.detect{|r| r.name=='libmemcached'} - ioncube_recipe = @extensions.detect{|r| r.name=='ioncube'} - - php_recipe_options[:hiredis_path] = hiredis_recipe.path unless hiredis_recipe.nil? - php_recipe_options[:libmemcached_path] = libmemcached_recipe.path unless libmemcached_recipe.nil? - php_recipe_options[:ioncube_path] = ioncube_recipe.path unless ioncube_recipe.nil? - - php_recipe_options.merge(DetermineChecksum.new(@options).to_h) - - @php_recipe ||= PhpRecipe.new(@name, @version, php_recipe_options) - end -end diff --git a/recipe/php_recipe.rb b/recipe/php_recipe.rb deleted file mode 100644 index 0676ced0..00000000 --- a/recipe/php_recipe.rb +++ /dev/null @@ -1,126 +0,0 @@ -# encoding: utf-8 -require_relative 'php_common_recipes' - -class PhpRecipe < BaseRecipe - def configure_options - [ - '--disable-static', - '--enable-shared', - '--enable-ftp=shared', - '--enable-sockets=shared', - '--enable-soap=shared', - '--enable-fileinfo=shared', - '--enable-bcmath', - '--enable-calendar', - '--enable-intl', - '--with-kerberos', - '--with-bz2=shared', - '--with-curl=shared', - '--enable-dba=shared', - "--with-password-argon2=/usr/lib/x86_64-linux-gnu", - '--with-cdb', - '--with-gdbm', - '--with-mysqli=shared', - '--enable-pdo=shared', - '--with-pdo-sqlite=shared,/usr', - '--with-pdo-mysql=shared,mysqlnd', - '--with-pdo-pgsql=shared', - '--with-pgsql=shared', - '--with-pspell=shared', - '--with-gettext=shared', - '--with-gmp=shared', - '--with-imap=shared', - '--with-imap-ssl=shared', - '--with-ldap=shared', - '--with-ldap-sasl', - '--with-zlib=shared', - '--with-libzip=/usr/local/lib', - '--with-xsl=shared', - '--with-snmp=shared', - '--enable-mbstring=shared', - '--enable-mbregex', - '--enable-exif=shared', - '--with-openssl=shared', - '--enable-fpm', - '--enable-pcntl=shared', - '--enable-sysvsem=shared', - '--enable-sysvshm=shared', - '--enable-sysvmsg=shared', - '--enable-shmop=shared', - ] - end - - def url - "https://github.com/php/web-php-distributions/raw/master/php-#{version}.tar.gz" - end - - def archive_files - ["#{port_path}/*"] - end - - def archive_path_name - 'php' - end - - def configure - return if configured? - - md5_file = File.join(tmp_path, 'configure.md5') - digest = Digest::MD5.hexdigest(computed_options.to_s) - File.open(md5_file, 'w') { |f| f.write digest } - - # LIBS=-lz enables using zlib when configuring - execute('configure', ['bash', '-c', "LIBS=-lz ./configure #{computed_options.join ' '}"]) - end - - def major_version - @major_version ||= version.match(/^(\d+\.\d+)/)[1] - end - - def zts_path - Dir["#{path}/lib/php/extensions/no-debug-non-zts-*"].first - end - - def setup_tar - lib_dir = '/usr/lib/x86_64-linux-gnu' - argon_dir = '/usr/lib/x86_64-linux-gnu' - - system <<-eof - cp -a /usr/local/lib/x86_64-linux-gnu/librabbitmq.so* #{path}/lib/ - cp -a #{@hiredis_path}/lib/libhiredis.so* #{path}/lib/ - cp -a /usr/lib/libc-client.so* #{path}/lib/ - cp -a /usr/lib/libmcrypt.so* #{path}/lib - cp -a #{lib_dir}/libaspell.so* #{path}/lib - cp -a #{lib_dir}/libpspell.so* #{path}/lib - cp -a /usr/lib/x86_64-linux-gnu/libmemcached.so* #{path}/lib/ - cp -a /usr/local/lib/x86_64-linux-gnu/libcassandra.so* #{path}/lib - cp -a /usr/local/lib/libuv.so* #{path}/lib - cp -a #{argon_dir}/libargon2.so* #{path}/lib - cp -a /usr/lib/librdkafka.so* #{path}/lib - cp -a /usr/lib/x86_64-linux-gnu/libzip.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libGeoIP.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libgpgme.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libassuan.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libgpg-error.so* #{path}/lib/ - cp -a /usr/lib/libtidy*.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libtidy*.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libenchant.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libfbclient.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/librecode.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libtommath.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libmaxminddb.so* #{path}/lib/ - cp -a /usr/lib/x86_64-linux-gnu/libssh2.so* #{path}/lib/ - eof - - if IonCubeRecipe.build_ioncube?(version) - system "cp #{@ioncube_path}/ioncube/ioncube_loader_lin_#{major_version}.so #{zts_path}/ioncube.so" - end - - system <<-eof - # Remove unused files - rm "#{path}/etc/php-fpm.conf.default" - rm "#{path}/bin/php-cgi" - find "#{path}/lib/php/extensions" -name "*.a" -type f -delete - eof - end -end diff --git a/recipe/python.rb b/recipe/python.rb deleted file mode 100644 index 48bd0c52..00000000 --- a/recipe/python.rb +++ /dev/null @@ -1,78 +0,0 @@ -# encoding: utf-8 -require 'fileutils' -require 'mini_portile' -require_relative '../lib/openssl_replace' -require_relative 'base' - -class PythonRecipe < BaseRecipe - def initialize(name, version, options = {}) - super name, version, options - # override openssl in container - OpenSSLReplace.replace_openssl - end - - def computed_options - [ - '--enable-shared', - '--with-ensurepip=yes', - '--with-dbmliborder=bdb:gdbm', - '--with-tcltk-includes="-I/usr/include/tcl8.6"', - '--with-tcltk-libs="-L/usr/lib/x86_64-linux-gnu -ltcl8.6 -L/usr/lib/x86_64-linux-gnu -ltk8.6"', - "--prefix=#{prefix_path}", - '--enable-unicode=ucs4' - ] - end - - def cook - install_apt('libdb-dev libgdbm-dev tk8.6-dev') - - run('apt-get -y --force-yes -d install --reinstall libtcl8.6 libtk8.6 libxss1') or raise 'Failed to download libtcl8.6 libtk8.6 libxss1' - FileUtils.mkdir_p prefix_path - Dir.glob('/var/cache/apt/archives/lib{tcl8.6,tk8.6,xss1}_*.deb').each do |path| - STDOUT.puts("dpkg -x #{path} #{prefix_path}") - run("dpkg -x #{path} #{prefix_path}") or raise "Could not extract #{path}" - end - - super - end - - def archive_files - ["#{prefix_path}/*"] - end - - def setup_tar - unless File.exist?("#{prefix_path}/bin/python") - File.symlink('./python3', "#{prefix_path}/bin/python") - end - end - - def url - "https://www.python.org/ftp/python/#{version}/Python-#{version}.tgz" - end - - def prefix_path - '/app/.heroku/vendor' - end - - private - - def install_apt(packages) - STDOUT.print "Running 'install dependencies' for #{@name} #{@version}... " - if run("sudo apt-get update && sudo apt-get -y install #{packages}") - STDOUT.puts "OK" - else - raise "Failed to complete install dependencies task" - end - end - - def run(command) - output = `#{command}` - if $?.success? - return true - else - STDOUT.puts "ERROR, output was:" - STDOUT.puts output - return false - end - end -end diff --git a/recipe/ruby.rb b/recipe/ruby.rb deleted file mode 100644 index 58f32c78..00000000 --- a/recipe/ruby.rb +++ /dev/null @@ -1,50 +0,0 @@ -# encoding: utf-8 -require 'mini_portile' -require_relative 'base' - -class RubyRecipe < BaseRecipe - def computed_options - [ - '--enable-load-relative', - '--disable-install-doc', - 'debugflags=-g', - "--prefix=#{prefix_path}", - "--without-gmp" - ] - end - - def cook - run('apt-get update') or raise 'Failed to apt-get update' - run('apt-get -y install libffi-dev') or raise 'Failed to install libffi-dev' - super - end - - def prefix_path - "/app/vendor/ruby-#{version}" - end - - def minor_version - version.match(/(\d+\.\d+)\./)[1] - end - - def archive_files - ["#{prefix_path}/*"] - end - - def url - "https://cache.ruby-lang.org/pub/ruby/#{minor_version}/ruby-#{version}.tar.gz" - end - - private - - def run(command) - output = `#{command}` - if $?.success? - return true - else - STDOUT.puts "ERROR, output was:" - STDOUT.puts output - return false - end - end -end diff --git a/spec/assets/binary-exerciser.sh b/spec/assets/binary-exerciser.sh deleted file mode 100755 index df4c3c99..00000000 --- a/spec/assets/binary-exerciser.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set +e - -tar_name=$1; shift - -mkdir -p /tmp/binary-exerciser -current_dir=`pwd` -cd /tmp/binary-exerciser - -tar xzf $current_dir/${tar_name} -eval $(printf '%q ' "$@") diff --git a/spec/assets/bundler-exerciser.sh b/spec/assets/bundler-exerciser.sh deleted file mode 100755 index 96949e07..00000000 --- a/spec/assets/bundler-exerciser.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set +e - -tar_name=$1; shift - -current_dir=`pwd` -tmpdir=$(mktemp -d /tmp/binary-builder.XXXXXXXX) -cd $tmpdir - -tar xzf $current_dir/${tar_name} --touch - -export GEM_HOME=$tmpdir -export GEM_PATH=$tmpdir - -eval $(printf '%q ' "$@") diff --git a/spec/assets/jruby-exerciser.sh b/spec/assets/jruby-exerciser.sh deleted file mode 100755 index 852079bf..00000000 --- a/spec/assets/jruby-exerciser.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set +e - -mkdir -p /tmp/binary-exerciser -current_dir=`pwd` -cd /tmp/binary-exerciser - -tar xzf $current_dir/jruby-9.2.8.0-ruby-2.5-linux-x64.tgz -JAVA_HOME=/opt/java -PATH=$PATH:$JAVA_HOME/bin -./bin/jruby -e 'puts "#{RUBY_PLATFORM} #{RUBY_VERSION}"' diff --git a/spec/assets/php-exerciser.sh b/spec/assets/php-exerciser.sh deleted file mode 100755 index 18c0d2ea..00000000 --- a/spec/assets/php-exerciser.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -tar_name=$1; shift -current_dir=$(pwd) -mkdir -p /tmp/binary-exerciser -cd /tmp/binary-exerciser - -tar xzf "$current_dir/$tar_name" -export LD_LIBRARY_PATH="$PWD/php/lib" -eval "$(printf '%q ' "$@")" diff --git a/spec/assets/php-extensions.yml b/spec/assets/php-extensions.yml deleted file mode 100644 index f075c9a8..00000000 --- a/spec/assets/php-extensions.yml +++ /dev/null @@ -1,198 +0,0 @@ ---- -native_modules: -- name: rabbitmq - version: 0.10.0 - md5: 6f09f0cb07cea221657a768bd9c7dff7 - klass: RabbitMQRecipe -- name: lua - version: 5.4.0 - md5: dbf155764e5d433fc55ae80ea7060b60 - klass: LuaRecipe -- name: hiredis - version: 1.0.0 - md5: 209ae570cdee65a5143ea6db8ac07fe3 - klass: HiredisRecipe -- name: snmp - version: nil - md5: nil - klass: SnmpRecipe -- name: librdkafka - version: 1.5.2 - md5: f5272e30ab6556967ed82a58d2ad35e1 - klass: LibRdKafkaRecipe -- name: libsodium - version: 1.0.18 - md5: 3ca9ebc13b6b4735acae0a6a4c4f9a95 - klass: LibSodiumRecipe -extensions: -- name: apcu - version: 5.1.19 - md5: a868ee0b4179fb240cf6eb7e49723794 - klass: PeclRecipe -- name: igbinary - version: 3.1.6 - md5: 9d1e0530025c7f129f46c81f7581af98 - klass: PeclRecipe -- name: gnupg - version: 1.4.0 - md5: 2354cb56168d8ea0f643e548e139d013 - klass: PeclRecipe -- name: imagick - version: 3.4.4 - md5: 6d3a7048ab73b0fab931f28c484dbf76 - klass: PeclRecipe -- name: LZF - version: 1.6.8 - md5: 0677fba342c89795de6c694f3e72ba1d - klass: PeclRecipe -- name: mailparse - version: 3.1.1 - md5: 17d77e3c03c25acaf51926a9020a4596 - klass: PeclRecipe -- name: mongodb - version: 1.9.0 - md5: 780f206f6a9399b5a4cabfd304f6ecff - klass: PeclRecipe -- name: msgpack - version: 2.1.2 - md5: 9bcaad416fc2b3c6ffd6966e0ae30313 - klass: PeclRecipe -- name: oauth - version: 2.0.7 - md5: 8ea6eb5ac6de8a4ed399980848c04c0c - klass: PeclRecipe -- name: odbc - version: nil - md5: nil - klass: OdbcRecipe -- name: pdo_odbc - version: nil - md5: nil - klass: PdoOdbcRecipe -- name: pdo_sqlsrv - version: 5.8.1 - md5: e687989a47cefd6cf9005e1f41637289 - klass: PeclRecipe -- name: rdkafka - version: 4.1.1 - md5: f4b39d7f6b4c489bfb4c5532f71046c2 - klass: PeclRecipe -- name: redis - version: 5.3.2 - md5: 8531a792d43a60dd03f87eec7b65b381 - klass: RedisPeclRecipe -- name: ssh2 - version: 1.2 - md5: ae62ba2d4a7bbd5eff34daa8ed9f6ed6 - klass: PeclRecipe -- name: sqlsrv - version: 5.8.1 - md5: 1a237f847d4466a85f7bfdb6b2fd5ecd - klass: PeclRecipe -- name: stomp - version: 2.0.2 - md5: 507c30184fde736e924cee20c56df061 - klass: PeclRecipe -- name: xdebug - version: 3.0.1 - md5: 52891d89de6829fa8dba1132b8c66f75 - klass: PeclRecipe -- name: yaf - version: 3.2.5 - md5: 8cd86db117f65131d212cac1c60065a3 - klass: PeclRecipe -- name: yaml - version: 2.2.0 - md5: cd8d34b87d9e147691d66590c403ba46 - klass: PeclRecipe -- name: memcached - version: 3.1.5 - md5: eb535a7551aad6bff0d836a4dec9c4fa - klass: MemcachedPeclRecipe -- name: sodium - version: nil - md5: nil - klass: SodiumRecipe - -#bundled w/PHP -- name: tidy - version: nil - md5: nil - klass: FakePeclRecipe -- name: enchant - version: nil - md5: nil - klass: FakePeclRecipe -- name: pdo_firebird - version: nil - md5: nil - klass: FakePeclRecipe -- name: readline - version: nil - md5: nil - klass: FakePeclRecipe -- name: xmlrpc - version: nil - md5: nil - klass: FakePeclRecipe -- name: zip - version: nil - md5: nil - klass: FakePeclRecipe - -#non-standard -- name: amqp - version: 1.10.2 - md5: 1163a52a495cab5210a6a2c8deb60064 - klass: AmqpPeclRecipe -- name: ioncube - version: 10.4.4 - md5: 1e6b0d8a8db6c5536c99bd7e67eb6a4f - klass: IonCubeRecipe -- name: lua - version: 2.0.7 - md5: a37402f8b10753a80db56b61e2e70c29 - klass: LuaPeclRecipe -- name: maxminddb - version: 1.8.0 - md5: ed95e0d8914ff5dd340fa6cf47b2b921 - klass: MaxMindRecipe -- name: psr - version: 1.0.1 - md5: 9e44b2f2e271bf57c5dbf6b6b07a8acf - klass: PeclRecipe -- name: phalcon - version: 4.1.0 - md5: c13b72a724820027ec9889f0dca0670d - klass: PeclRecipe -- name: phpiredis - version: 1.0.1 - md5: 09a9bdb347c70832d3e034655b604064 - klass: PHPIRedisRecipe -- name: phpprotobufpecl - version: 0.12.4 - md5: 77a77a429af4a5ff97778ccafeffa43a - klass: PHPProtobufPeclRecipe -- name: tideways_xhprof - version: 5.0.2 - md5: 374cf4ff7ba03401a279777abe94815d #curl -sL https://github.com/tideways/php-xhprof-extension/archive/v5.0.2.tar.gz | md5sum | cut -d ' ' -f 1 - klass: TidewaysXhprofRecipe -- name: solr - version: 2.5.1 - md5: 29fc866198d61bccdbc4c4f53fb7ef06 - klass: PeclRecipe - -- name: gd - version: nil - md5: nil - klass: Gd74FakePeclRecipe - -#Oracle client stuff. Not included unless libs are present -- name: oci8 - version: 2.2.0 - md5: 678d2a647881cd8e5b458c669dcce215 - klass: OraclePeclRecipe -- name: pdo_oci - version: nil - md5: nil - klass: OraclePdoRecipe diff --git a/spec/integration/bundler_spec.rb b/spec/integration/bundler_spec.rb deleted file mode 100644 index 6646c7b8..00000000 --- a/spec/integration/bundler_spec.rb +++ /dev/null @@ -1,32 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when bundler is specified' do - before(:all) do - run_binary_builder('bundler', '1.11.2', '--sha256=c7aa8ffe0af6e0c75d0dad8dd7749cb8493b834f0ed90830d4843deb61906768') - @binary_tarball_location = File.join(Dir.pwd, 'bundler-1.11.2.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, replaces the shebangs, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - bundler_version_cmd = "./spec/assets/bundler-exerciser.sh bundler-1.11.2.tgz ./bin/bundle -v" - output, status = run(bundler_version_cmd) - - expect(status).to be_success - expect(output).to include('Bundler version 1.11.2') - - shebang = `tar -O -xf #{@binary_tarball_location} ./bin/bundle | head -n1`.chomp - expect(shebang).to eq("#!/usr/bin/env ruby") - shebang = `tar -O -xf #{@binary_tarball_location} ./bin/bundler | head -n1`.chomp - expect(shebang).to eq("#!/usr/bin/env ruby") - end - end -end - diff --git a/spec/integration/dep_spec.rb b/spec/integration/dep_spec.rb deleted file mode 100644 index 12945c62..00000000 --- a/spec/integration/dep_spec.rb +++ /dev/null @@ -1,30 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when dep is specified' do - before(:all) do - run_binary_builder('dep', 'v0.3.0', '--sha256=7d816ffb14f57c4b01352676998a8cda9e4fb24eaec92bd79526e1045c5a0c83') - @binary_tarball_location = File.join(Dir.pwd, 'dep-v0.3.0-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - dep_version_cmd = './spec/assets/binary-exerciser.sh dep-v0.3.0-linux-x64.tgz ./bin/dep ensure -examples' - output, status = run(dep_version_cmd) - - expect(status).to be_success - expect(output).to include('dep ensure') - end - - it 'includes the license in the tar file.' do - expect(tar_contains_file('bin/LICENSE')).to eq true - end - end -end diff --git a/spec/integration/glide_spec.rb b/spec/integration/glide_spec.rb deleted file mode 100644 index cac15fc8..00000000 --- a/spec/integration/glide_spec.rb +++ /dev/null @@ -1,31 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when glide is specified' do - before(:all) do - run_binary_builder('glide', 'v0.11.0', '--sha256=7a7023aff20ba695706a262b8c07840ee28b939ea6358efbb69ab77da04f0052') - @binary_tarball_location = File.join(Dir.pwd, 'glide-v0.11.0-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - glide_version_cmd = "./spec/assets/binary-exerciser.sh glide-v0.11.0-linux-x64.tgz ./bin/glide -v" - output, status = run(glide_version_cmd) - - expect(status).to be_success - expect(output).to include('glide version 0.11.0') - end - - it 'includes the license in the tar file.' do - expect(tar_contains_file('bin/LICENSE')).to eq true - end - end -end - diff --git a/spec/integration/go_spec.rb b/spec/integration/go_spec.rb deleted file mode 100644 index 3e6e50d4..00000000 --- a/spec/integration/go_spec.rb +++ /dev/null @@ -1,30 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when go is specified' do - before(:all) do - run_binary_builder('go', '1.6.3', '--sha256=6326aeed5f86cf18f16d6dc831405614f855e2d416a91fd3fdc334f772345b00') - @binary_tarball_location = File.join(Dir.pwd, 'go1.6.3.linux-amd64.tar.gz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - go_version_cmd = './spec/assets/binary-exerciser.sh go1.6.3.linux-amd64.tar.gz GOROOT=/tmp/binary-exerciser/go ./go/bin/go version' - output, status = run(go_version_cmd) - - expect(status).to be_success - expect(output).to include('go1.6.3') - end - - it 'includes the license in the tar file.' do - expect(tar_contains_file('go/LICENSE')).to eq true - end - end -end diff --git a/spec/integration/godep_spec.rb b/spec/integration/godep_spec.rb deleted file mode 100644 index 27716d54..00000000 --- a/spec/integration/godep_spec.rb +++ /dev/null @@ -1,30 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when godep is specified' do - before(:all) do - run_binary_builder('godep', 'v14', '--sha256=0f212bcf903d5b01db0e93a4218b79f228c6f080d5a409dd4e2ec5edfbc2aad5') - @binary_tarball_location = File.join(Dir.pwd, 'godep-v14-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - godep_version_cmd = './spec/assets/binary-exerciser.sh godep-v14-linux-x64.tgz ./bin/godep version' - output, status = run(godep_version_cmd) - - expect(status).to be_success - expect(output).to include('v14') - end - - it 'includes the license in the tar file.' do - expect(tar_contains_file('bin/License')).to eq true - end - end -end diff --git a/spec/integration/httpd_spec.rb b/spec/integration/httpd_spec.rb deleted file mode 100644 index b5cf11a6..00000000 --- a/spec/integration/httpd_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when httpd is specified' do - before(:all) do - run_binary_builder('httpd', '2.4.41', '--sha256=133d48298fe5315ae9366a0ec66282fa4040efa5d566174481077ade7d18ea40') - @binary_tarball_location = Dir.glob(File.join(Dir.pwd, 'httpd-2.4.41-linux-x64*.tgz')).first - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - httpd_version_cmd = %(env LD_LIBRARY_PATH=/tmp/binary-exerciser/lib ./spec/assets/binary-exerciser.sh #{File.basename(@binary_tarball_location)} ./httpd/bin/httpd -v) - - output, status = run(httpd_version_cmd) - - expect(status).to be_success - expect(output).to include('2.4.41') - end - - it 'copies in *.so files for some of the compiled extensions' do - expect(tar_contains_file('httpd/lib/libapr-1.so.0')).to eq true - expect(tar_contains_file('httpd/lib/libaprutil-1.so.0')).to eq true - expect(tar_contains_file('httpd/lib/libapriconv-1.so.0')).to eq true - expect(tar_contains_file('httpd/lib/apr-util-1/apr_ldap.so')).to eq true - expect(tar_contains_file('httpd/lib/iconv/utf-8.so')).to eq true - end - end -end diff --git a/spec/integration/hwc_spec.rb b/spec/integration/hwc_spec.rb deleted file mode 100644 index 5ac1f7aa..00000000 --- a/spec/integration/hwc_spec.rb +++ /dev/null @@ -1,45 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' -require 'tmpdir' - -describe 'building a binary', :integration do - context 'when hwc is specified' do - - before(:all) do - run_binary_builder('hwc', '20.0.0', '--sha256=643fd1225881bd6206eec205ba818cf60be00bd3a1029c86b0e5bf74a3a978ab') - @binary_zip_location = File.join(Dir.pwd, 'hwc-20.0.0-windows-x86-64.zip') - @unzip_dir = Dir.mktmpdir - end - - after(:all) do - FileUtils.rm(@binary_zip_location) - FileUtils.rm_rf(@unzip_dir) - end - - it 'builds the specified binary, zips it, and places it in your current working directory' do - expect(File).to exist(@binary_zip_location) - - zip_file_cmd = "file hwc-20.0.0-windows-x86-64.zip" - output, status = run(zip_file_cmd) - - expect(status).to be_success - expect(output).to include('Zip archive data') - end - - it 'builds a windows binary' do - Dir.chdir(@unzip_dir) do - FileUtils.cp(@binary_zip_location, Dir.pwd) - system "unzip hwc-20.0.0-windows-x86-64.zip" - file_output = `file hwc.exe` - expect(file_output).to include('hwc.exe: PE32+ executable') - expect(file_output).to include('for MS Windows') - - file_output = `file hwc_x86.exe` - expect(file_output).to include('hwc_x86.exe: PE32 executable') - expect(file_output).to include('for MS Windows') - end - end - end -end - diff --git a/spec/integration/jruby_spec.rb b/spec/integration/jruby_spec.rb deleted file mode 100644 index 5c530a83..00000000 --- a/spec/integration/jruby_spec.rb +++ /dev/null @@ -1,26 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when jruby is specified' do - before(:all) do - output = run_binary_builder('jruby', '9.2.8.0-ruby-2.5', '--sha256=287ae0e946c2d969613465c738cc3b09098f9f25805893ab707dce19a7b98c43') - @binary_tarball_location = File.join(Dir.pwd, 'jruby-9.2.8.0-ruby-2.5-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - jruby_version_cmd = './spec/assets/jruby-exerciser.sh' - output, status = run(jruby_version_cmd) - - expect(status).to be_success - expect(output).to include('java 2.5.3') - end - end -end diff --git a/spec/integration/nodejs_spec.rb b/spec/integration/nodejs_spec.rb deleted file mode 100644 index 2f459490..00000000 --- a/spec/integration/nodejs_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when node allows openssl-use-def-ca-store' do - before(:all) do - run_binary_builder('node', '8.8.1', '--sha256=1725bbbe623d6a13ee14522730dfc90eac1c9ebe9a0a8f4c3322a402dd7e75a2') - @binary_tarball_location = File.join(Dir.pwd, 'node-8.8.1-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - node_version_cmd = "./spec/assets/binary-exerciser.sh node-8.8.1-linux-x64.tgz node-v8.8.1-linux-x64/bin/node -e 'console.log(process.version)'" - - output, status = run(node_version_cmd) - - expect(status).to be_success - expect(output).to include('v8.8.1') - end - end - - context 'when node DOES NOT allow openssl-use-def-ca-store' do - before(:all) do - run_binary_builder('node', '4.8.5', '--sha256=23980b1d31c6b0e05eff2102ffa0059a6f7a93e27e5288eb5551b9b003ec0c07') - @binary_tarball_location = File.join(Dir.pwd, 'node-4.8.5-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - node_version_cmd = "./spec/assets/binary-exerciser.sh node-4.8.5-linux-x64.tgz node-v4.8.5-linux-x64/bin/node -e 'console.log(process.version)'" - - output, status = run(node_version_cmd) - - expect(status).to be_success - expect(output).to include('v4.8.5') - end - end -end diff --git a/spec/integration/php7_with_oracle_spec.rb b/spec/integration/php7_with_oracle_spec.rb deleted file mode 100644 index fc31687b..00000000 --- a/spec/integration/php7_with_oracle_spec.rb +++ /dev/null @@ -1,44 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' -require 'open-uri' - -describe 'building a binary', :run_oracle_php_tests do - context 'when php7.4 is specified with oracle libraries' do - before(:all) do - extensions_file = File.join('spec', 'assets', 'php-extensions.yml') - - run_binary_builder('php', '7.4.0', "--sha256=004a1a8176176ee1b5c112e73d705977507803f425f9e48cb4a84f42b22abf22 --php-extensions-file=#{extensions_file}") - @binary_tarball_location = Dir.glob(File.join(Dir.pwd, 'php-7.4.0-linux-x64.tgz')).first - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'can load the oci8.so and pdo_oci.so PHP extensions' do - expect(File).to exist(@binary_tarball_location) - php_oracle_module_arguments = '-dextension=oci8.so -dextension=pdo_oci.so -dextension=pdo.so' - php_info_modules_command = '-r "phpinfo(INFO_MODULES);"' - - php_info_with_oracle_modules = %{./spec/assets/php-exerciser.sh #{File.basename(@binary_tarball_location)} ./php/bin/php #{php_oracle_module_arguments} #{php_info_modules_command}} - - output, status = run(php_info_with_oracle_modules) - - expect(status).to be_success - expect(output).to include('OCI8 Support => enabled') - expect(output).to include('PDO Driver for OCI 8 and later => enabled') - end - - it 'copies in the oracle *.so files ' do - expect(tar_contains_file('php/lib/libclntshcore.so.12.1')).to eq true - expect(tar_contains_file('php/lib/libclntsh.so')).to eq true - expect(tar_contains_file('php/lib/libclntsh.so.12.1')).to eq true - expect(tar_contains_file('php/lib/libipc1.so')).to eq true - expect(tar_contains_file('php/lib/libmql1.so')).to eq true - expect(tar_contains_file('php/lib/libnnz12.so')).to eq true - expect(tar_contains_file('php/lib/libociicus.so')).to eq true - expect(tar_contains_file('php/lib/libons.so')).to eq true - end - end -end diff --git a/spec/integration/ruby_spec.rb b/spec/integration/ruby_spec.rb deleted file mode 100644 index 9c7d19a3..00000000 --- a/spec/integration/ruby_spec.rb +++ /dev/null @@ -1,33 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'fileutils' - -describe 'building a binary', :integration do - context 'when ruby is specified' do - before(:all) do - run_binary_builder('ruby', '2.6.5', '--sha256=66976b716ecc1fd34f9b7c3c2b07bbd37631815377a2e3e85a5b194cfdcbed7d') - @binary_tarball_location = File.join(Dir.pwd, 'ruby-2.6.5-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'builds the specified binary, tars it, and places it in your current working directory' do - expect(File).to exist(@binary_tarball_location) - - ruby_version_cmd = "./spec/assets/binary-exerciser.sh ruby-2.6.5-linux-x64.tgz ./bin/ruby -e 'puts RUBY_VERSION'" - output, status = run(ruby_version_cmd) - - expect(status).to be_success - expect(output).to include('2.6.5') - - libgmp_cmd = "./spec/assets/binary-exerciser.sh ruby-2.6.5-linux-x64.tgz grep LIBS= lib/pkgconfig/ruby-2.6.pc" - output, status = run(libgmp_cmd) - - expect(status).to be_success - expect(output).to include('LIBS=') - expect(output).not_to include('lgmp') - end - end -end diff --git a/spec/integration/url_output_spec.rb b/spec/integration/url_output_spec.rb deleted file mode 100644 index 70532f16..00000000 --- a/spec/integration/url_output_spec.rb +++ /dev/null @@ -1,37 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'yaml' - -describe 'building a binary', :integration do - context 'when a recipe is specified' do - before(:all) do - @output, = run_binary_builder('glide', 'v0.11.0', '--sha256=7a7023aff20ba695706a262b8c07840ee28b939ea6358efbb69ab77da04f0052') - @binary_tarball_location = File.join(Dir.pwd, 'glide-v0.11.0-linux-x64.tgz') - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'prints the url of the source used to build the binary to stdout' do - puts @output - expect(@output).to include('Source URL: https://github.com/Masterminds/glide/archive/v0.11.0.tar.gz') - end - end - - context 'when a meal is specified' do - before(:all) do - @output, = run_binary_builder('httpd', '2.4.41', '--sha256=133d48298fe5315ae9366a0ec66282fa4040efa5d566174481077ade7d18ea40') - @binary_tarball_location = Dir.glob(File.join(Dir.pwd, 'httpd-2.4.41-linux-x64*.tgz')).first - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'prints the url of the source used to build the binary to stdout' do - puts @output - expect(@output).to include('Source URL: https://archive.apache.org/dist/httpd/httpd-2.4.41.tar.bz2') - end - end -end diff --git a/spec/integration/yaml_flag_spec.rb b/spec/integration/yaml_flag_spec.rb deleted file mode 100644 index 37bfb7b0..00000000 --- a/spec/integration/yaml_flag_spec.rb +++ /dev/null @@ -1,96 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require 'yaml' - -describe 'building a binary', :integration do - context 'when a recipe is specified' do - before(:all) do - @output, _ = run_binary_builder('go', '1.6.3', '--sha256=6326aeed5f86cf18f16d6dc831405614f855e2d416a91fd3fdc334f772345b00') - @tarball_name = 'go1.6.3.linux-amd64.tar.gz' - @binary_tarball_location = File.join(Dir.pwd, @tarball_name) - end - - after(:all) do - FileUtils.rm(@binary_tarball_location) - end - - it 'prints a yaml representation of the source used to build the binary to stdout' do - yaml_source = @output.match(/Source YAML:(.*)/m)[1] - expect(YAML.safe_load(yaml_source)).to eq([ - { - "sha256"=>"6326aeed5f86cf18f16d6dc831405614f855e2d416a91fd3fdc334f772345b00", - "url"=>"https://storage.googleapis.com/golang/go1.6.3.src.tar.gz" - } - ]) - end - - it 'includes the yaml representation of the source inside the resulting tarball' do - yaml_source = `tar xzf #{@tarball_name} -O sources.yml` - expect(YAML.safe_load(yaml_source)).to eq([ - { - "sha256"=>"6326aeed5f86cf18f16d6dc831405614f855e2d416a91fd3fdc334f772345b00", - "url"=>"https://storage.googleapis.com/golang/go1.6.3.src.tar.gz" - } - ]) - end - end - - context 'when a meal is specified' do - before(:all) do - @output, = run_binary_builder('httpd', '2.4.41', '--sha256=133d48298fe5315ae9366a0ec66282fa4040efa5d566174481077ade7d18ea40') - @binary_tarball_location = Dir.glob(File.join(Dir.pwd, 'httpd-2.4.41-linux-x64*.tgz')).first - end - - it 'prints a yaml representation of the source used to build the binary to stdout' do - yaml_source = @output.match(/Source YAML:(.*)/m)[1] - expect(YAML.safe_load(yaml_source)).to match_array([ - { - "sha256" => "133d48298fe5315ae9366a0ec66282fa4040efa5d566174481077ade7d18ea40", - "url" => "https://archive.apache.org/dist/httpd/httpd-2.4.41.tar.bz2" - }, - { - "sha256" => "48e9dbf45ae3fdc7b491259ffb6ccf7d63049ffacbc1c0977cced095e4c2d5a2", - "url" => "https://apache.osuosl.org/apr/apr-1.7.0.tar.gz" - }, - { - "sha256" => "ce94c7722ede927ce1e5a368675ace17d96d60ff9b8918df216ee5c1298c6a5e", - "url" => "https://apache.osuosl.org/apr/apr-iconv-1.2.2.tar.gz" - }, - { - "sha256" => "b65e40713da57d004123b6319828be7f1273fbc6490e145874ee1177e112c459", - "url" => "https://apache.osuosl.org/apr/apr-util-1.6.1.tar.gz" - }, - { - "sha256" => "0f078444fed34085bc83e27eb3439556718f50dcea275307ffb66d498bdabb8f", - "url" => "https://github.com/zmartzone/mod_auth_openidc/releases/download/v2.3.8/mod_auth_openidc-2.3.8.tar.gz" - } - ]) - end - - it 'includes the yaml representation of the source inside the resulting tarball' do - yaml_source = `tar xzf httpd-2.4.41-linux-x64.tgz sources.yml -O` - expect(YAML.safe_load(yaml_source)).to match_array([ - { - "sha256" => "133d48298fe5315ae9366a0ec66282fa4040efa5d566174481077ade7d18ea40", - "url" => "https://archive.apache.org/dist/httpd/httpd-2.4.41.tar.bz2" - }, - { - "sha256" => "48e9dbf45ae3fdc7b491259ffb6ccf7d63049ffacbc1c0977cced095e4c2d5a2", - "url" => "https://apache.osuosl.org/apr/apr-1.7.0.tar.gz" - }, - { - "sha256" => "ce94c7722ede927ce1e5a368675ace17d96d60ff9b8918df216ee5c1298c6a5e", - "url" => "https://apache.osuosl.org/apr/apr-iconv-1.2.2.tar.gz" - }, - { - "sha256" => "b65e40713da57d004123b6319828be7f1273fbc6490e145874ee1177e112c459", - "url" => "https://apache.osuosl.org/apr/apr-util-1.6.1.tar.gz" - }, - { - "sha256" => "0f078444fed34085bc83e27eb3439556718f50dcea275307ffb66d498bdabb8f", - "url" => "https://github.com/zmartzone/mod_auth_openidc/releases/download/v2.3.8/mod_auth_openidc-2.3.8.tar.gz" - } - ]) - end - end -end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb deleted file mode 100644 index 10b797fd..00000000 --- a/spec/spec_helper.rb +++ /dev/null @@ -1,140 +0,0 @@ -# encoding: utf-8 -require 'fileutils' -require 'open3' -require 'tmpdir' - -RSpec.configure do |config| - config.color = true - config.tty = true - - if RUBY_PLATFORM.include?('darwin') - DOCKER_CONTAINER_NAME = "test-suite-binary-builder-#{Time.now.to_i}".freeze - - config.before(:all, :integration) do - directory_mapping = "-v #{Dir.pwd}:/binary-builder" - setup_docker_container(DOCKER_CONTAINER_NAME, directory_mapping) - end - - config.after(:all, :integration) do - cleanup_docker_artifacts(DOCKER_CONTAINER_NAME) - end - - config.before(:all, :run_oracle_php_tests) do - dir_to_contain_oracle = File.join(Dir.pwd, 'oracle_client_libs') - FileUtils.mkdir_p(dir_to_contain_oracle) - setup_oracle_libs(dir_to_contain_oracle) - - oracle_dir = File.join(dir_to_contain_oracle, 'oracle') - directory_mapping = "-v #{Dir.pwd}:/binary-builder -v #{oracle_dir}:/oracle" - setup_docker_container(DOCKER_CONTAINER_NAME, directory_mapping) - end - - config.after(:all, :run_oracle_php_tests) do - cleanup_docker_artifacts(DOCKER_CONTAINER_NAME) - - dir_containing_oracle = File.join(Dir.pwd, 'oracle_client_libs') - FileUtils.rm_rf(dir_containing_oracle) - end - - config.before(:all, :run_geolite_php_tests) do - directory_mapping = "-v #{Dir.pwd}:/binary-builder" - setup_docker_container(DOCKER_CONTAINER_NAME, directory_mapping) - - file_to_enable_geolite_db = File.join(Dir.pwd, 'BUNDLE_GEOIP_LITE') - File.open(file_to_enable_geolite_db, 'w') { |f| f.puts "true" } - end - - config.after(:all, :run_geolite_php_tests) do - cleanup_docker_artifacts(DOCKER_CONTAINER_NAME) - - file_to_enable_geolite_db = File.join(Dir.pwd, 'BUNDLE_GEOIP_LITE') - FileUtils.rm(file_to_enable_geolite_db) - end - else - config.before(:all, :run_oracle_php_tests) do - setup_oracle_libs('/') - end - - config.before(:all, :run_geolite_php_tests) do - file_to_enable_geolite_db = File.join(Dir.pwd, 'BUNDLE_GEOIP_LITE') - File.open(file_to_enable_geolite_db, 'w') { |f| f.puts "true" } - end - - config.after(:all, :run_geolite_php_tests) do - file_to_enable_geolite_db = File.join(Dir.pwd, 'BUNDLE_GEOIP_LITE') - FileUtils.rm(file_to_enable_geolite_db) - end - end - - - def cleanup_docker_artifacts(docker_container_name) - `docker stop #{docker_container_name}` - `docker rm #{docker_container_name}` - - Dir['*deb*'].each do |deb_file| - FileUtils.rm(deb_file) - end - end - - def setup_oracle_libs(dir_to_contain_oracle) - Dir.chdir(dir_to_contain_oracle) do - s3_bucket = ENV['ORACLE_LIBS_AWS_BUCKET'] - libs_filename = ENV['ORACLE_LIBS_FILENAME'] - - ## If AWS_ASSUME_ROLE_ARN is provides, switch to aws assume-role mode - if ENV['AWS_ASSUME_ROLE_ARN'] && !ENV['AWS_ASSUME_ROLE_ARN'].empty? - system <<-eof - uuid=$(cat /proc/sys/kernel/random/uuid) - RESULT="$(aws sts assume-role --role-arn "${AWS_ASSUME_ROLE_ARN}" --role-session-name "binary-builder-spec-${uuid}")" - export AWS_ACCESS_KEY_ID="$(echo "${RESULT}" |jq -r .Credentials.AccessKeyId)" - export AWS_SECRET_ACCESS_KEY="$(echo "${RESULT}" |jq -r .Credentials.SecretAccessKey)" - export AWS_SESSION_TOKEN="$(echo "${RESULT}" |jq -r .Credentials.SessionToken)" - aws s3 cp s3://#{s3_bucket}/#{libs_filename} . - eof - else - system "aws s3 cp s3://#{s3_bucket}/#{libs_filename} ." - end - system "tar -xvf #{libs_filename}" - end - end - - def setup_docker_container(docker_container_name, directory_mapping) - docker_image = "cloudfoundry/#{ENV.fetch('STACK', 'cflinuxfs3')}" - `docker run --name #{docker_container_name} -dit #{directory_mapping} -e CCACHE_DIR=/binary-builder/.ccache -w /binary-builder #{docker_image} sh -c 'env PATH=/usr/lib/ccache:$PATH bash'` - `docker exec #{docker_container_name} apt-get -y install ccache` - `docker exec #{docker_container_name} gem install bundler --no-ri --no-rdoc` - `docker exec #{docker_container_name} bundle install -j4` - end - - def run(cmd) - cmd = "docker exec #{DOCKER_CONTAINER_NAME} #{cmd}" if RUBY_PLATFORM.include?('darwin') - - Bundler.with_clean_env do - Open3.capture2e(cmd).tap do |output, status| - expect(status).to be_success, (lambda do - puts "command output: #{output}" - puts "expected command to return a success status code, got: #{status}" - end) - end - end - end - - def run_binary_builder(binary_name, binary_version, flags) - binary_builder_cmd = "bundle exec ./bin/binary-builder --name=#{binary_name} --version=#{binary_version} #{flags}" - run(binary_builder_cmd) - end - - def tar_contains_file(filename) - expect(@binary_tarball_location).to be - - o, status = Open3.capture2e("tar --wildcards --list --verbose -f #{@binary_tarball_location} #{filename}") - return false unless status.success? - o.split(/[\r\n]+/).all? do |line| - m = line.match(/(\S+) -> (\S+)$/) - return true unless m - oldfile, newfile = m[1,2] - return false if newfile.start_with?('/') - tar_contains_file(File.dirname(oldfile) + '/' + newfile) - end - end -end diff --git a/spec/unit/archive_recipe_spec.rb b/spec/unit/archive_recipe_spec.rb deleted file mode 100644 index cadb578f..00000000 --- a/spec/unit/archive_recipe_spec.rb +++ /dev/null @@ -1,36 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require_relative '../../lib/archive_recipe' -require_relative '../../recipe/base' - -describe ArchiveRecipe do - class FakeRecipe < BaseRecipe - def url; end - - def archive_files - [1] - end - end - - context 'when the recipe has #setup_tar' do - it 'it invokes' do - recipe = FakeRecipe.new('fake', '1.1.1') - def recipe.setup_tar; end - allow(YAMLPresenter).to receive(:new).and_return('') - - expect(recipe).to receive(:setup_tar) - described_class.new(recipe).compress! - end - end - - context 'when the recipe does not have #setup_tar' do - it 'does not invoke' do - recipe = FakeRecipe.new('fake', '1.1.1') - allow(YAMLPresenter).to receive(:new).and_return('') - - expect do - described_class.new(recipe).compress! - end.not_to raise_error - end - end -end diff --git a/spec/unit/yaml_spec.rb b/spec/unit/yaml_spec.rb deleted file mode 100644 index 6d80d122..00000000 --- a/spec/unit/yaml_spec.rb +++ /dev/null @@ -1,29 +0,0 @@ -# encoding: utf-8 -require 'spec_helper' -require_relative '../../lib/yaml_presenter' - -describe YAMLPresenter do - it 'encodes the SHA256 as a raw string' do - recipe = double(:recipe, files_hashs: [ - { - local_path: File.expand_path(__FILE__) - } - ]) - presenter = described_class.new(recipe) - expect(presenter.to_yaml).to_not include "!binary |-\n" - end - - context 'the source is a github repo' do - it 'displays the git commit sha' do - recipe = double(:recipe, files_hashs: [ - { - git: {commit_sha: 'a_mocked_commit_sha'}, - local_path: File.expand_path(__FILE__) - } - ]) - presenter = described_class.new(recipe) - expect(presenter.to_yaml).to_not include "!binary |-\n" - expect(presenter.to_yaml).to include 'a_mocked_commit_sha' - end - end -end diff --git a/stacks/cflinuxfs4.yaml b/stacks/cflinuxfs4.yaml new file mode 100644 index 00000000..06d431db --- /dev/null +++ b/stacks/cflinuxfs4.yaml @@ -0,0 +1,154 @@ +name: cflinuxfs4 +ubuntu_version: "22.04" +ubuntu_codename: jammy +docker_image: cloudfoundry/cflinuxfs4 + +apt_packages: + php_build: + - automake + - firebird-dev + - libaspell-dev + - libc-client2007e-dev + - libcurl4-openssl-dev + - libdb-dev + - libedit-dev + - libenchant-2-dev + - libexpat1-dev + - libgdbm-dev + - libgeoip-dev + - libgmp-dev + - libgpgme11-dev + - libjpeg-dev + - libkrb5-dev + - libldap2-dev + - libmagickwand-dev + - libmagickcore-dev + - libmaxminddb-dev + - libmcrypt-dev + - libmemcached-dev + - libonig-dev + - libpng-dev + - libpspell-dev + - librecode-dev + - libsasl2-dev + - libsnmp-dev + - libsqlite3-dev + - libssh2-1-dev + - libssl-dev + - libtidy-dev + - libtool + - libwebp-dev + - libxml2-dev + - libzip-dev + - libzookeeper-mt-dev + - snmp-mibs-downloader + - sqlite3 + - unixodbc-dev + r_build: + - gfortran + - libbz2-dev + - liblzma-dev + - libpcre++-dev + - libpcre2-dev + - libcurl4-openssl-dev + - libsodium-dev + - libharfbuzz-dev + - libfribidi-dev + - default-jre + - libgfortran-12-dev + - libfreetype6-dev + - libpng-dev + - libtiff5-dev + - libjpeg-dev + - libwebp-dev + ruby_build: + - libffi-dev + python_build: + - libdb-dev + - libgdbm-dev + - tk8.6-dev + node_build: [] + httpd_build: + - libldap2-dev + httpd_mod_auth_build: + - libjansson-dev + - libcjose-dev + - libhiredis-dev + libgdiplus_build: + - automake + - libtool + - libglib2.0-dev + - libcairo2-dev + libunwind_build: + - autoconf + - automake + - libtool + hwc_build: + - mingw-w64 + pip_build: + - python3 + - python3-pip + python_deb_extras: + - libxss1 + +bootstrap: + go: + url: https://go.dev/dl/go1.24.2.linux-amd64.tar.gz + sha256: 68097bd680839cbc9d464a0edce4f7c333975e27a90246890e9f1078c7e702ad + jruby: + url: https://java-buildpack.cloudfoundry.org/openjdk-jdk/jammy/x86_64/bellsoft-jdk8u452%2B11-linux-amd64.tar.gz + sha256: d87d70d286150e662f12650ed22d91f1663c267b27ab3b0775eed1416ef3bd12 + install_dir: /opt/java + ruby: + url: https://buildpacks.cloudfoundry.org/dependencies/ruby/ruby_3.3.6_linux_x64_cflinuxfs4_e4311262.tgz + sha256: e4311262803d0e1a01c83001e3e98ad58700b5cc36eff12433f24312b11e3dc9 + install_dir: /opt/ruby + +compilers: + gfortran: + version: 11 + bin: /usr/bin/x86_64-linux-gnu-gfortran-11 + lib_path: /usr/lib/gcc/x86_64-linux-gnu/11 + packages: + - gfortran + - libgfortran-12-dev + gcc: + version: 12 + packages: + - gcc-12 + - g++-12 + ppa: "ppa:ubuntu-toolchain-r/test" + tool_packages: + - software-properties-common + +httpd_sub_deps: + apr: + version: "1.7.4" + url: https://archive.apache.org/dist/apr/apr-1.7.4.tar.gz + sha256: a4137dd82a185076fa50ba54232d920a17c6469c30b0876569e1c2a05ff311d9 + apr_iconv: + version: "1.2.2" + url: https://archive.apache.org/dist/apr/apr-iconv-1.2.2.tar.gz + sha256: ce94c7722ede927ce1e5a368675ace17d96d60ff9b8918df216ee5c1298c6a5e + apr_util: + version: "1.6.3" + url: https://archive.apache.org/dist/apr/apr-util-1.6.3.tar.gz + sha256: 2b74d8932703826862ca305b094eef2983c27b39d5c9414442e9976a9acf1983 + mod_auth_openidc: + version: "2.3.8" + url: https://github.com/OpenIDC/mod_auth_openidc/releases/download/v2.3.8/mod_auth_openidc-2.3.8.tar.gz + sha256: 0f078444fed34085bc83e27eb3439556718f50dcea275307ffb66d498bdabb8f + +php_symlinks: + - src: /usr/include/x86_64-linux-gnu/curl + dst: /usr/local/include/curl + - src: /usr/include/x86_64-linux-gnu/gmp.h + dst: /usr/include/gmp.h + - src: /usr/lib/x86_64-linux-gnu/libldap.so + dst: /usr/lib/libldap.so + - src: /usr/lib/x86_64-linux-gnu/libldap_r.so + dst: /usr/lib/libldap_r.so + +python: + tcl_version: "8.6" + use_force_yes: true diff --git a/stacks/cflinuxfs5.yaml b/stacks/cflinuxfs5.yaml new file mode 100644 index 00000000..cfd398c7 --- /dev/null +++ b/stacks/cflinuxfs5.yaml @@ -0,0 +1,152 @@ +name: cflinuxfs5 +ubuntu_version: "24.04" +ubuntu_codename: noble +docker_image: cloudfoundry/cflinuxfs5 + +apt_packages: + php_build: + - automake + - firebird-dev + - libaspell-dev + - libc-client2007e-dev + - libcurl4-openssl-dev + - libdb5.3-dev + - libedit-dev + - libenchant-2-dev + - libexpat1-dev + - libgdbm-dev + - libgeoip-dev + - libgmp-dev + - libgpgme11-dev + - libjpeg-dev + - libkrb5-dev + - libldap2-dev + - libmagickwand-dev + - libmagickcore-dev + - libmaxminddb-dev + - libmcrypt-dev + - libmemcached-dev + - libonig-dev + - libpng-dev + - libpspell-dev + - librecode-dev + - libsasl2-dev + - libsnmp-dev + - libsqlite3-dev + - libssh2-1-dev + - libssl-dev + - libtidy-dev + - libtool + - libwebp-dev + - libxml2-dev + - libzip-dev + # libzookeeper-mt-dev omitted — not available on Ubuntu 24.04 + - snmp-mibs-downloader + - sqlite3 + - unixodbc-dev + r_build: + - gfortran + - libbz2-dev + - liblzma-dev + - libpcre2-dev + - libcurl4-openssl-dev + - libsodium-dev + - libharfbuzz-dev + - libfribidi-dev + - default-jre + - libgfortran-13-dev + - libfreetype6-dev + - libpng-dev + - libtiff-dev + - libjpeg-dev + - libwebp-dev + ruby_build: + - libffi-dev + python_build: + - libdb5.3-dev + - libgdbm-dev + - tk8.6-dev + node_build: [] + httpd_build: + - libldap2-dev + - libjansson-dev + # libcjose-dev omitted — needs verification on Ubuntu 24.04 + - libhiredis-dev + libgdiplus_build: + - automake + - libtool + - libglib2.0-dev + - libcairo2-dev + libunwind_build: + - autoconf + - automake + - libtool + hwc_build: + - mingw-w64 + pip_build: + - python3 + - python3-pip + python_deb_extras: + - libxss1 + +bootstrap: + go: + url: https://go.dev/dl/go1.24.2.linux-amd64.tar.gz + sha256: 68097bd680839cbc9d464a0edce4f7c333975e27a90246890e9f1078c7e702ad + jruby: + url: https://java-buildpack.cloudfoundry.org/openjdk-jdk/jammy/x86_64/bellsoft-jdk8u452%2B11-linux-amd64.tar.gz + sha256: d87d70d286150e662f12650ed22d91f1663c267b27ab3b0775eed1416ef3bd12 + install_dir: /opt/java + ruby: + url: https://buildpacks.cloudfoundry.org/dependencies/ruby/ruby_3.3.6_linux_x64_cflinuxfs4_e4311262.tgz + sha256: e4311262803d0e1a01c83001e3e98ad58700b5cc36eff12433f24312b11e3dc9 + install_dir: /opt/ruby + +compilers: + gfortran: + version: 13 + bin: /usr/bin/x86_64-linux-gnu-gfortran-13 + lib_path: /usr/lib/gcc/x86_64-linux-gnu/13 + libexec_path: /usr/libexec/gcc/x86_64-linux-gnu/13 + packages: + - gfortran + - libgfortran-13-dev + gcc: + version: 14 + packages: + - gcc-14 + - g++-14 + ppa: "" + tool_packages: + - software-properties-common + +httpd_sub_deps: + apr: + version: "1.7.4" + url: https://archive.apache.org/dist/apr/apr-1.7.4.tar.gz + sha256: a4137dd82a185076fa50ba54232d920a17c6469c30b0876569e1c2a05ff311d9 + apr_iconv: + version: "1.2.2" + url: https://archive.apache.org/dist/apr/apr-iconv-1.2.2.tar.gz + sha256: ce94c7722ede927ce1e5a368675ace17d96d60ff9b8918df216ee5c1298c6a5e + apr_util: + version: "1.6.3" + url: https://archive.apache.org/dist/apr/apr-util-1.6.3.tar.gz + sha256: 2b74d8932703826862ca305b094eef2983c27b39d5c9414442e9976a9acf1983 + mod_auth_openidc: + version: "2.3.8" + url: https://github.com/OpenIDC/mod_auth_openidc/releases/download/v2.3.8/mod_auth_openidc-2.3.8.tar.gz + sha256: 0f078444fed34085bc83e27eb3439556718f50dcea275307ffb66d498bdabb8f + +php_symlinks: + - src: /usr/include/x86_64-linux-gnu/curl + dst: /usr/local/include/curl + - src: /usr/include/x86_64-linux-gnu/gmp.h + dst: /usr/include/gmp.h + - src: /usr/lib/x86_64-linux-gnu/libldap.so + dst: /usr/lib/libldap.so + # libldap_r removed — dropped in OpenLDAP 2.6 (Ubuntu 24.04) + +python: + tcl_version: "8.6" + use_force_yes: false diff --git a/test/exerciser/exerciser_test.go b/test/exerciser/exerciser_test.go new file mode 100644 index 00000000..9c24ec8d --- /dev/null +++ b/test/exerciser/exerciser_test.go @@ -0,0 +1,333 @@ +//go:build integration + +// Package exerciser_test verifies that built artifacts are functional. +// +// Each test extracts a tarball inside a Docker container running the target stack +// and asserts that the binary self-reports the expected version string (or that +// expected files are present for library/tool artifacts). +// +// Usage: +// +// ARTIFACT=/tmp/ruby_3.3.6_linux_x64_cflinuxfs4_e4311262.tgz \ +// STACK=cflinuxfs4 \ +// go test -tags integration ./test/exerciser/ -run TestRubyBinary -v +// +// The ARTIFACT and STACK environment variables must be set. If either is absent +// the test is skipped (not failed), so the harness is safe to import in CI +// pipelines that gate on the integration build tag. +package exerciser_test + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// runInContainer extracts ARTIFACT inside a Docker container running IMAGE and +// executes cmdArgs. It returns the combined stdout+stderr output. +func runInContainer(t *testing.T, tarball, stack string, cmdArgs ...string) string { + t.Helper() + + _, thisFile, _, _ := runtime.Caller(0) + runScript := filepath.Join(filepath.Dir(thisFile), "run.sh") + + args := append([]string{tarball, stack}, cmdArgs...) + cmd := exec.Command(runScript, args...) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("exerciser command failed: %v\noutput:\n%s", err, out) + } + return string(out) +} + +// artifact returns the ARTIFACT env var or skips the test. +func artifact(t *testing.T) string { + t.Helper() + a := os.Getenv("ARTIFACT") + if a == "" { + t.Skip("ARTIFACT env var not set") + } + return a +} + +// stackEnv returns the STACK env var or skips the test. +func stackEnv(t *testing.T) string { + t.Helper() + s := os.Getenv("STACK") + if s == "" { + t.Skip("STACK env var not set") + } + return s +} + +// assertContains fails the test if output does not contain want. +func assertContains(t *testing.T, output, want string) { + t.Helper() + if !strings.Contains(output, want) { + t.Errorf("expected output to contain %q\ngot:\n%s", want, output) + } +} + +// ── Compiled deps ────────────────────────────────────────────────────────── + +func TestRubyBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./bin/ruby", "-e", "puts RUBY_VERSION") + assertContains(t, out, os.Getenv("VERSION")) +} + +func TestPythonBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./bin/python3", "--version") + assertContains(t, out, "Python "+os.Getenv("VERSION")) +} + +func TestNodeBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", + "node-v*/bin/node -e 'console.log(process.version)'") + assertContains(t, out, "v"+os.Getenv("VERSION")) +} + +func TestGoBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./go/bin/go", "version") + assertContains(t, out, "go"+os.Getenv("VERSION")) +} + +func TestNginxBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", + "env LD_LIBRARY_PATH=./lib ./nginx/sbin/nginx -v 2>&1") + assertContains(t, out, "nginx/"+os.Getenv("VERSION")) +} + +func TestNginxStaticBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./nginx/sbin/nginx", "-v") + assertContains(t, out, "nginx/"+os.Getenv("VERSION")) +} + +func TestOpenrestyBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./nginx/sbin/nginx", "-v") + assertContains(t, out, "openresty/"+os.Getenv("VERSION")) +} + +func TestHTTPDBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", + "env LD_LIBRARY_PATH=./lib ./httpd/bin/httpd -v 2>&1") + assertContains(t, out, "Apache/"+os.Getenv("VERSION")) +} + +func TestJRubyBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./bin/jruby", "--version") + // JRuby version is the prefix of the full artifact version (e.g. "9.4.5.0") + version := os.Getenv("VERSION") + if idx := strings.Index(version, "-ruby-"); idx != -1 { + version = version[:idx] + } + assertContains(t, out, version) +} + +func TestBundlerBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./bin/bundle", "--version") + assertContains(t, out, "Bundler version "+os.Getenv("VERSION")) +} + +func TestRBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./bin/R", "--version") + assertContains(t, out, "R version "+os.Getenv("VERSION")) +} + +func TestLibunwindFiles(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", "ls lib/libunwind.so*") + if strings.TrimSpace(out) == "" { + t.Error("expected lib/libunwind.so* to exist but found nothing") + } +} + +func TestLibgdiplusFiles(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", "ls lib/libgdiplus.so*") + if strings.TrimSpace(out) == "" { + t.Error("expected lib/libgdiplus.so* to exist but found nothing") + } +} + +func TestDepBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./dep", "version") + assertContains(t, out, os.Getenv("VERSION")) +} + +func TestGlideBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./glide", "--version") + assertContains(t, out, os.Getenv("VERSION")) +} + +func TestGodepBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./godep", "version") + assertContains(t, out, "v"+os.Getenv("VERSION")) +} + +func TestHWCBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "file", "hwc.exe") + assertContains(t, out, "PE32+") +} + +// ── Repack / simple deps ─────────────────────────────────────────────────── + +func TestPipBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./bin/pip", "--version") + assertContains(t, out, "pip "+os.Getenv("VERSION")) +} + +func TestPipenvBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./bin/pipenv", "--version") + assertContains(t, out, "pipenv, version "+os.Getenv("VERSION")) +} + +func TestSetuptoolsFiles(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", "ls setuptools-*.dist-info/") + if strings.TrimSpace(out) == "" { + t.Error("expected setuptools-*.dist-info/ directory to exist but found nothing") + } +} + +func TestYarnBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./bin/yarn", "--version") + assertContains(t, out, os.Getenv("VERSION")) +} + +func TestRubygemsFiles(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", "ls rubygems-*/") + if strings.TrimSpace(out) == "" { + t.Error("expected rubygems-*/ directory to exist but found nothing") + } +} + +func TestDotnetSDKBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./dotnet", "--version") + assertContains(t, out, os.Getenv("VERSION")) +} + +func TestDotnetRuntimeBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "./dotnet", "--version") + assertContains(t, out, os.Getenv("VERSION")) +} + +func TestDotnetAspnetcoreFiles(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", + "ls shared/Microsoft.AspNetCore.App/") + if strings.TrimSpace(out) == "" { + t.Error("expected shared/Microsoft.AspNetCore.App/ to exist but found nothing") + } +} + +// ── Passthrough deps ─────────────────────────────────────────────────────── + +func TestComposerBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", + "php composer.phar --version 2>&1 || echo 'php not available, checking file exists'") + // Composer is a phar — check it's present if php is not available + if !strings.Contains(out, "Composer version "+os.Getenv("VERSION")) { + out2 := runInContainer(t, a, s, "bash", "-c", "ls composer.phar") + if strings.TrimSpace(out2) == "" { + t.Error("composer.phar not found in artifact") + } + } +} + +func TestTomcatFiles(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", "ls bin/catalina.sh") + if strings.TrimSpace(out) == "" { + t.Error("expected bin/catalina.sh to exist but found nothing") + } +} + +func TestOpenJDKBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", "./bin/java -version 2>&1") + assertContains(t, out, os.Getenv("VERSION")) +} + +func TestZuluBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", "./bin/java -version 2>&1") + assertContains(t, out, os.Getenv("VERSION")) +} + +func TestSAPMachineBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", "./bin/java -version 2>&1") + assertContains(t, out, os.Getenv("VERSION")) +} + +func TestJProfilerFiles(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", "ls bin/jprofiler") + if strings.TrimSpace(out) == "" { + t.Error("expected bin/jprofiler to exist but found nothing") + } +} + +func TestYourKitFiles(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", "ls lib/yjp.jar") + if strings.TrimSpace(out) == "" { + t.Error("expected lib/yjp.jar to exist but found nothing") + } +} + +// ── PHP — extended assertions ────────────────────────────────────────────── + +func TestPHPBinary(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", + "export LD_LIBRARY_PATH=$PWD/php/lib && ./php/bin/php --version 2>&1") + assertContains(t, out, "PHP "+os.Getenv("VERSION")) +} + +func TestPHPNativeModules(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", + "export LD_LIBRARY_PATH=$PWD/php/lib && ./php/bin/php -m 2>&1") + for _, mod := range []string{"date", "json", "pcre", "Reflection", "SPL", "standard"} { + if !strings.Contains(out, mod) { + t.Errorf("expected PHP native module %q to be present\noutput:\n%s", mod, out) + } + } +} + +func TestPHPKeyExtensions(t *testing.T) { + a, s := artifact(t), stackEnv(t) + out := runInContainer(t, a, s, "bash", "-c", + "export LD_LIBRARY_PATH=$PWD/php/lib && ./php/bin/php -m 2>&1") + for _, ext := range []string{"curl", "gd", "mbstring", "mysqli", "pdo_mysql"} { + if !strings.Contains(out, ext) { + t.Errorf("expected PHP extension %q to be present\noutput:\n%s", ext, out) + } + } +} diff --git a/test/exerciser/run.sh b/test/exerciser/run.sh new file mode 100644 index 00000000..b8093388 --- /dev/null +++ b/test/exerciser/run.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# binary-builder/test/exerciser/run.sh +# Usage: run.sh +# +# Extracts the tarball inside the target stack Docker container and runs +# the given command. Used to verify that a built artifact actually works. +# +# Examples: +# ./run.sh /tmp/ruby_3.3.6_linux_x64_cflinuxfs4_e4311262.tgz cflinuxfs4 \ +# ./bin/ruby -e 'puts RUBY_VERSION' +# +# ./run.sh /tmp/php_8.3.0_linux_x64_cflinuxfs4_abcd1234.tgz cflinuxfs4 \ +# bash -c 'LD_LIBRARY_PATH=$PWD/php/lib ./php/bin/php --version' + +set -euo pipefail + +TARBALL="${1:?tarball path required}" +STACK="${2:?stack required}" +shift 2 + +IMAGE="cloudfoundry/${STACK}" +TARBALL_ABS="$(realpath "${TARBALL}")" +TARBALL_NAME="$(basename "${TARBALL_ABS}")" + +# Quote each remaining arg so the docker bash -c invocation is safe +CMD_ARGS=( "$@" ) +CMD_QUOTED="" +for arg in "${CMD_ARGS[@]}"; do + CMD_QUOTED="${CMD_QUOTED} $(printf '%q' "${arg}")" +done + +docker run --rm \ + -v "${TARBALL_ABS}:/tmp/${TARBALL_NAME}" \ + "${IMAGE}" \ + bash -c " + set -euo pipefail + mkdir -p /tmp/exerciser + cd /tmp/exerciser + tar xzf /tmp/${TARBALL_NAME} + ${CMD_QUOTED} + " diff --git a/test/parity/compare-builds.sh b/test/parity/compare-builds.sh new file mode 100755 index 00000000..1f6a2aaa --- /dev/null +++ b/test/parity/compare-builds.sh @@ -0,0 +1,489 @@ +#!/usr/bin/env bash +# binary-builder/test/parity/compare-builds.sh +# Usage: compare-builds.sh --dep --data-json [--stack ] +# [--sub-deps-dir ] +# +# Runs both the Ruby builder and the Go builder inside the target stack Docker +# container with the same real depwatcher data.json, then diffs every observable +# output. Exits 1 on any mismatch. +# +# All output is tee'd to a log file: +# /tmp/parity-logs/--.log +# Check progress without re-running: tail -f /tmp/parity-logs/-*.log +# +# The data.json must be in the real depwatcher modern format: +# { +# "source": {"name": "...", "type": "...", "repo": "..."}, +# "version": {"url": "...", "ref": "...", "sha256": "...", "sha512": "..."} +# } +# +# For the `r` dep, pass --sub-deps-dir pointing to a directory containing: +# source-forecast-latest/data.json +# source-plumber-latest/data.json +# source-rserve-latest/data.json +# source-shiny-latest/data.json +# These are mounted into both containers at the working directory. +# +# The Ruby builder clones the ruby-builder-final tag inside the container (for +# the cflinuxfs4/ Ruby source tree) and mounts the local buildpacks-ci working +# tree (task scripts stay in sync with local changes). The Go builder mounts +# the local binary-builder working tree so that in-progress changes are tested. + +set -euo pipefail + +DEP="" +DATA_JSON="" +STACK="cflinuxfs4" +SUB_DEPS_DIR="" + +# Parse args +while [[ $# -gt 0 ]]; do + case $1 in + --dep) DEP="$2"; shift 2 ;; + --data-json) DATA_JSON="$2"; shift 2 ;; + --stack) STACK="$2"; shift 2 ;; + --sub-deps-dir) SUB_DEPS_DIR="$2"; shift 2 ;; + *) echo "Unknown arg: $1"; exit 1 ;; + esac +done + +[[ -n "${DEP}" ]] || { echo "Usage: compare-builds.sh --dep --data-json [--stack ] [--sub-deps-dir ]"; exit 1; } +[[ -n "${DATA_JSON}" ]] || { echo "--data-json is required"; exit 1; } +[[ -f "${DATA_JSON}" ]] || { echo "data.json not found: ${DATA_JSON}"; exit 1; } + +VERSION=$(jq -r '.version.ref // .version // "unknown"' "${DATA_JSON}") +IMAGE="cloudfoundry/${STACK}" + +# Resolve binary-builder repo root (two levels up from this script). +# Used by the Go builder to mount the local working tree. +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +RUBY_OUT="$(mktemp -d)" +GO_OUT="$(mktemp -d)" +SOURCE_DIR="$(mktemp -d)" # pre-downloaded source tarballs for libunwind/dotnet/etc. +DATA_JSON_ABS="$(realpath "${DATA_JSON}")" +SUB_DEPS_DIR_ABS="" +[[ -n "${SUB_DEPS_DIR}" ]] && SUB_DEPS_DIR_ABS="$(realpath "${SUB_DEPS_DIR}")" + +# ── Logging setup ───────────────────────────────────────────────────────────── +# All output (stdout + stderr) goes to both the terminal and a persistent log +# file so long-running builds can be inspected without re-running. +LOG_DIR="/tmp/parity-logs" +mkdir -p "${LOG_DIR}" +LOG_FILE="${LOG_DIR}/${DEP}-${VERSION}-${STACK}.log" +# Redirect all further output through tee; preserve stderr on the terminal too. +exec > >(tee "${LOG_FILE}") 2>&1 +echo "==> Log file: ${LOG_FILE}" + +cleanup() { + local _exit=$? + # Docker runs as root inside containers so output files may be root-owned. + # Use a throwaway container to chmod before removing; ignore any errors. + for dir in "${RUBY_OUT}" "${GO_OUT}" "${SOURCE_DIR}"; do + [[ -d "${dir}" ]] && docker run --rm -v "${dir}:/cleanup:z" busybox \ + chmod -R a+rwX /cleanup 2>/dev/null || true + done + rm -rf "${RUBY_OUT}" "${GO_OUT}" "${SOURCE_DIR}" 2>/dev/null || true + exit "${_exit}" +} +trap cleanup EXIT + +echo "==> Parity test: ${DEP} ${VERSION} on ${STACK}" + +# ── Source pre-download ──────────────────────────────────────────────────────── +# Some deps (libunwind, dotnet-*) expect a pre-downloaded source tarball in +# source/ alongside data.json. We download it here on the host and mount the +# source/ dir into both containers. +prepare_source() { + local url + url=$(jq -r '.version.url // empty' "${DATA_JSON_ABS}") + [[ -z "${url}" ]] && return 0 + + # Only download for deps that need a pre-placed source file. + # These are deps where the builder reads source/*.tar.gz or source/ + # rather than downloading itself. + case "${DEP}" in + libunwind|dotnet-sdk|dotnet-runtime|dotnet-aspnetcore|jprofiler-profiler|your-kit-profiler) + local filename + filename=$(basename "${url}") + echo "--> Pre-downloading source: ${url}" + curl -fsSL -o "${SOURCE_DIR}/${filename}" "${url}" + chmod a+r "${SOURCE_DIR}/${filename}" + echo " Saved: ${SOURCE_DIR}/${filename}" + ;; + *) + # All other deps fetch their own source inside the build + ;; + esac +} + +# ── Ruby builder ───────────────────────────────────────────────────────────── +# +# The Ruby builder clones the ruby-builder-final tag inside the container (for +# the cflinuxfs4/ Ruby source tree) and mounts the local buildpacks-ci working +# tree (task scripts stay in sync with local changes). The Go builder mounts +# the local binary-builder working tree so that in-progress changes are tested. + +run_ruby_builder() { + echo "--> Running Ruby builder..." + + mkdir -p "${RUBY_OUT}/artifact" "${RUBY_OUT}/dep-metadata" "${RUBY_OUT}/builds" + chmod -R o+rwx "${RUBY_OUT}" + + # Build sub-deps volume mount args (for r dep). + local ruby_subdeps_args=() + if [[ -n "${SUB_DEPS_DIR_ABS}" ]]; then + ruby_subdeps_args=(-v "${SUB_DEPS_DIR_ABS}:/tmp/host-sub-deps:ro,z") + fi + + docker run --rm \ + -v "${REPO_ROOT}/../buildpacks-ci:/buildpacks-ci-ro:ro,z" \ + -v "${DATA_JSON_ABS}:/tmp/data.json:ro,z" \ + -v "${SOURCE_DIR}:/tmp/host-source:ro,z" \ + -v "${RUBY_OUT}:/out:z" \ + "${ruby_subdeps_args[@]}" \ + -e STACK="${STACK}" \ + "${IMAGE}" \ + bash -c ' + set -euo pipefail + + apt-get update -qq + apt-get install -y -qq git + + # Copy buildpacks-ci to a writable location so tasks that write files + # (e.g. php_extensions/php-final-extensions.yml) can do so freely. + cp -a /buildpacks-ci-ro /buildpacks-ci + + # Clone the ruby-builder-final tag for the Ruby source tree (cflinuxfs4/ + # Gemfile, Gemfile.lock, bin/binary-builder, etc.). This tag points to + # the last commit on main before the Go builder was merged, so the parity + # baseline stays stable regardless of what main contains afterwards. + echo "--> Cloning binary-builder ruby-builder-final tag..." + git clone --depth=1 --branch ruby-builder-final https://github.com/cloudfoundry/binary-builder.git /srv/binary-builder + + RUBY_VERSION="3.4.6" + if ! command -v ruby &>/dev/null || ! ruby --version | grep -q "3.4"; then + apt-get install -y -qq wget build-essential zlib1g-dev libssl-dev libreadline-dev libyaml-dev libffi-dev + pushd /tmp + wget -q "https://cache.ruby-lang.org/pub/ruby/3.4/ruby-${RUBY_VERSION}.tar.gz" + tar -xzf "ruby-${RUBY_VERSION}.tar.gz" + cd "ruby-${RUBY_VERSION}" + ./configure --disable-install-doc + make -j$(nproc) + make install + popd + rm -rf "/tmp/ruby-${RUBY_VERSION}"* + fi + + # Set up Concourse-style task directory layout. + mkdir -p /task/source /task/artifacts "/task/builds-artifacts/binary-builds-new/'"${DEP}"'" /task/dep-metadata + cp /tmp/data.json /task/source/data.json + # Copy any pre-downloaded source files (libunwind tarball, dotnet tarball, etc.) + cp /tmp/host-source/* /task/source/ 2>/dev/null || true + ln -sf /srv/binary-builder /task/binary-builder + ln -sf /buildpacks-ci /task/buildpacks-ci + + # For r dep: copy sub-dep data.json dirs into task working directory. + if [[ -d /tmp/host-sub-deps ]]; then + cp -r /tmp/host-sub-deps/source-*-latest /task/ 2>/dev/null || true + fi + + cd /task + STACK='"${STACK}"' SKIP_COMMIT=true ruby buildpacks-ci/tasks/build-binary-new-cflinuxfs4/build.rb + + cp artifacts/* /out/artifact/ 2>/dev/null || true + cp dep-metadata/* /out/dep-metadata/ 2>/dev/null || true + cp "builds-artifacts/binary-builds-new/'"${DEP}"'"/*.json /out/builds/ 2>/dev/null || true + ' +} + +# ── Go builder ─────────────────────────────────────────────────────────────── + +run_go_builder() { + echo "--> Running Go builder..." + + mkdir -p "${GO_OUT}/artifact" "${GO_OUT}/dep-metadata" "${GO_OUT}/builds" + chmod -R o+rwx "${GO_OUT}" + + GO_VERSION="1.25.7" + + # Build sub-deps volume mount args (for r dep). + local go_subdeps_args=() + if [[ -n "${SUB_DEPS_DIR_ABS}" ]]; then + go_subdeps_args=(-v "${SUB_DEPS_DIR_ABS}:/tmp/host-sub-deps:ro,z") + fi + + # Write the in-container script to a temp file so we avoid quoting conflicts + # between the outer double-quoted bash -c string and the python3 -c blocks. + local go_script + go_script=$(mktemp) + cat > "${go_script}" </dev/null; then + apt-get update -qq + # zstd is required so the system tar can auto-detect compression when + # mise extracts the Go toolchain tarball inside this container. + apt-get install -y -qq curl ca-certificates zstd + curl -fsSL https://mise.run | MISE_QUIET=1 sh +fi +export PATH="\${HOME}/.local/bin:\${PATH}" +mise use --global go@${GO_VERSION} +export PATH="\${HOME}/.local/share/mise/shims:\${PATH}" + +go version >&2 + +# Compile binary-builder from source (must run from module root). +cd /binary-builder +go build -buildvcs=false -o /usr/local/bin/binary-builder ./cmd/binary-builder + +# Set up working directory. +mkdir -p /tmp/workdir/source +# Copy any pre-downloaded source files (libunwind tarball, dotnet tarball, etc.) +cp /tmp/host-source/* /tmp/workdir/source/ 2>/dev/null || true +# For r dep: copy sub-dep data.json dirs into workdir (Go recipe reads them from CWD). +if [[ -d /tmp/host-sub-deps ]]; then + cp -r /tmp/host-sub-deps/source-*-latest /tmp/workdir/ 2>/dev/null || true +fi +cd /tmp/workdir + +# Run binary-builder: JSON summary written to /out/summary.json via --output-file. +# Build subprocess output (compiler, make, etc.) flows to stdout/stderr (visible +# in logs) without corrupting the structured JSON output file. +binary-builder build \ + --stack "${STACK}" \ + --source-file /tmp/data.json \ + --stacks-dir /binary-builder/stacks \ + --output-file /out/summary.json + +summary=\$(cat /out/summary.json) + +# Move artifact from CWD to /out/artifact/. +artifact_file=\$(jq -r '.artifact_path' <<<"\${summary}") +mv "/tmp/workdir/\${artifact_file}" /out/artifact/ + +# Write dep-metadata JSON file. +# Format mirrors the Ruby builder's out_data.to_json output: +# version, source, url, sha256 (and git_commit_sha / sub_dependencies when present). +# Do NOT add extra fields (name, uri, source_sha256) that Ruby doesn't write — +# the parity comparison diffs these files field-by-field. +dep_meta_file="/out/dep-metadata/\${artifact_file}_metadata.json" +jq '{ + version: .version, + source: (.source // {}), + url: (.url // ""), + sha256: (.sha256 // "") +} + if (.git_commit_sha and .git_commit_sha != "") then {git_commit_sha: .git_commit_sha} else {} end + + if (.sub_dependencies | length) > 0 then {sub_dependencies: .sub_dependencies} else {} end' \ + <<<"\${summary}" > "\${dep_meta_file}" + +# Write builds-artifacts JSON file (binary-builds-new//--.json). +mkdir -p "/out/builds" +_version=\$(jq -r '.version' <<<"\${summary}") +builds_file="/out/builds/${DEP}-\${_version}-${STACK}.json" +jq '{ + url: (.url // ""), + sha256: (.sha256 // ""), + source: (.source // {}), + source_sha256: (.source.sha256 // ""), + sub_dependencies: (.sub_dependencies // {}) +}' <<<"\${summary}" > "\${builds_file}" +GOBUILDER_SCRIPT + + docker run --rm \ + -v "${REPO_ROOT}:/binary-builder:z" \ + -v "${DATA_JSON_ABS}:/tmp/data.json:ro,z" \ + -v "${SOURCE_DIR}:/tmp/host-source:ro,z" \ + -v "${GO_OUT}:/out:z" \ + -v "${go_script}:/tmp/run-go-builder.sh:ro,z" \ + "${go_subdeps_args[@]}" \ + -e STACK="${STACK}" \ + "${IMAGE}" \ + bash /tmp/run-go-builder.sh + + rm -f "${go_script}" +} + +# ── Compare ────────────────────────────────────────────────────────────────── + +compare_outputs() { + local mismatches=0 + + # --- Artifact filename pattern (strip 8-char SHA prefix) --- + ruby_artifact=$(ls "${RUBY_OUT}/artifact/" 2>/dev/null | head -1) + go_artifact=$(ls "${GO_OUT}/artifact/" 2>/dev/null | head -1) + + # URL-passthrough deps produce no artifact file — + # both builders set outData.URL directly pointing to the original download. + # For these deps, no artifact is expected; just compare builds JSON. + URL_PASSTHROUGH_DEPS=() + is_url_passthrough=false + for pt_dep in "${URL_PASSTHROUGH_DEPS[@]}"; do + if [[ "${DEP}" == "${pt_dep}" ]]; then + is_url_passthrough=true + break + fi + done + + if [[ "${is_url_passthrough}" == "true" ]]; then + if [[ -n "${ruby_artifact}" || -n "${go_artifact}" ]]; then + echo "WARN: URL-passthrough dep ${DEP} unexpectedly produced an artifact file" + echo " Ruby: ${ruby_artifact:-none} Go: ${go_artifact:-none}" + else + echo " OK: no artifact file (URL-passthrough dep)" + fi + else + if [[ -z "${ruby_artifact}" ]]; then + echo "FAIL: Ruby builder produced no artifact" + return 1 + fi + if [[ -z "${go_artifact}" ]]; then + echo "FAIL: Go builder produced no artifact" + return 1 + fi + fi + + if [[ "${is_url_passthrough}" == "false" ]]; then + ruby_pattern=$(echo "${ruby_artifact}" | sed 's/_[0-9a-f]\{8\}\./_./') + go_pattern=$(echo "${go_artifact}" | sed 's/_[0-9a-f]\{8\}\./_./') + + if [[ "${ruby_pattern}" != "${go_pattern}" ]]; then + echo "MISMATCH: artifact filename pattern" + echo " Ruby: ${ruby_artifact}" + echo " Go: ${go_artifact}" + mismatches=$((mismatches + 1)) + else + echo " OK: artifact filename pattern (${go_pattern})" + fi + + # --- Tar/zip contents (sorted file list) --- + ruby_ext="${ruby_artifact##*.}" + go_ext="${go_artifact##*.}" + + if [[ "${ruby_ext}" == "tgz" || "${ruby_ext}" == "gz" ]]; then + ruby_files=$(tar -tzf "${RUBY_OUT}/artifact/${ruby_artifact}" 2>/dev/null | sort) + go_files=$(tar -tzf "${GO_OUT}/artifact/${go_artifact}" 2>/dev/null | sort) + elif [[ "${ruby_ext}" == "zip" ]]; then + ruby_files=$(unzip -l "${RUBY_OUT}/artifact/${ruby_artifact}" 2>/dev/null | awk 'NR>3{print $4}' | sort) + go_files=$(unzip -l "${GO_OUT}/artifact/${go_artifact}" 2>/dev/null | awk 'NR>3{print $4}' | sort) + else + ruby_files="" + go_files="" + fi + + # Known Ruby builder bug: SnmpRecipe sets @php_path = nil (constructor takes + # name/version/options only, no php_path), so "cd #{@php_path}" expands to + # "cd" (empty) and the mibs/conf copy runs in the wrong directory — the + # snmp-mibs-downloader tree never appears in the Ruby artifact. The Go + # builder is correct. Filter from both sides before comparing. + ruby_files=$(echo "${ruby_files}" | grep -v 'mibs/conf/snmp-mibs-downloader') + go_files=$(echo "${go_files}" | grep -v 'mibs/conf/snmp-mibs-downloader') + + if [[ -n "${ruby_files}" ]] && [[ "${ruby_files}" != "${go_files}" ]]; then + # R artifact file lists are non-deterministic: the R package ecosystem installs + # transitive dependencies based on CRAN availability at build time. Two builds + # of the same R version will routinely differ in their transitive package set. + # Treat as WARN-only so the parity test still passes. + if [[ "${DEP}" == "r" ]]; then + echo "WARN: artifact file list differs (non-deterministic R package deps — expected)" + diff <(echo "${ruby_files}") <(echo "${go_files}") || true + else + echo "MISMATCH: artifact file list" + diff <(echo "${ruby_files}") <(echo "${go_files}") || true + mismatches=$((mismatches + 1)) + fi + else + echo " OK: artifact file list" + fi + fi + + # --- builds JSON (field by field) --- + ruby_json=$(ls "${RUBY_OUT}/builds/"*.json 2>/dev/null | head -1) + go_json=$(ls "${GO_OUT}/builds/"*.json 2>/dev/null | head -1) + + if [[ -z "${ruby_json}" || -z "${go_json}" ]]; then + echo "WARN: builds JSON missing (Ruby: ${ruby_json:-none}, Go: ${go_json:-none})" + else + for field in version "source.url" "source.sha256" "source.sha512" "source.md5" url sha256; do + ruby_val=$(jq -r ".${field} // empty" "${ruby_json}" 2>/dev/null) + go_val=$(jq -r ".${field} // empty" "${go_json}" 2>/dev/null) + if [[ "${ruby_val}" != "${go_val}" ]]; then + echo "MISMATCH: builds JSON field .${field}" + echo " Ruby: ${ruby_val}" + echo " Go: ${go_val}" + mismatches=$((mismatches + 1)) + fi + done + + ruby_subdeps=$(jq -r '.sub_dependencies // {} | to_entries[] | "\(.key)=\(.value.version)"' "${ruby_json}" 2>/dev/null | sort) + go_subdeps=$(jq -r '.sub_dependencies // {} | to_entries[] | "\(.key)=\(.value.version)"' "${go_json}" 2>/dev/null | sort) + if [[ "${ruby_subdeps}" != "${go_subdeps}" ]]; then + echo "MISMATCH: sub_dependencies" + diff <(echo "${ruby_subdeps}") <(echo "${go_subdeps}") || true + mismatches=$((mismatches + 1)) + else + echo " OK: builds JSON fields + sub_dependencies" + fi + fi + + # --- dep-metadata JSON --- + ruby_meta=$(ls "${RUBY_OUT}/dep-metadata/"*.json 2>/dev/null | head -1) + go_meta=$(ls "${GO_OUT}/dep-metadata/"*.json 2>/dev/null | head -1) + + if [[ -n "${ruby_meta}" && -n "${go_meta}" ]]; then + # Compare dep-metadata in two passes: + # + # 1. Structural fields (version, source.*) — must match exactly. + # 2. sha256 / url fields — these embed the artifact hash, which differs + # between independent runs for non-reproducible builds (e.g. bundler, + # where `gem install` records the current wall-clock time as file mtime). + # We only WARN on sha256/url mismatches so the parity test still passes + # as long as the artifact file list and structural metadata are identical. + # + # Also exclude sub_dependencies[].source.sha256: the Ruby builder computes + # these via sha_from_url (sha256 of the HTTP redirect response body, not the + # actual tarball), which is a Ruby builder bug. The Go builder uses the + # correct sha256 from data.json. Treat sub-dep source sha256 as WARN-only. + structural_fields='del(.sha256, .url) | if .sub_dependencies then .sub_dependencies |= with_entries(.value.source.sha256 = null) else . end' + if ! diff <(jq -S "${structural_fields}" "${ruby_meta}") \ + <(jq -S "${structural_fields}" "${go_meta}") > /dev/null 2>&1; then + echo "MISMATCH: dep-metadata JSON (structural fields)" + diff <(jq -S "${structural_fields}" "${ruby_meta}") \ + <(jq -S "${structural_fields}" "${go_meta}") || true + mismatches=$((mismatches + 1)) + else + # Check sha256/url (artifact hash) — WARN only, not a hard failure. + if ! diff <(jq -S . "${ruby_meta}") <(jq -S . "${go_meta}") > /dev/null 2>&1; then + echo "WARN: dep-metadata JSON sha256/url differ (non-reproducible build — expected for gem-install deps)" + diff <(jq -S . "${ruby_meta}") <(jq -S . "${go_meta}") || true + else + echo " OK: dep-metadata JSON" + fi + fi + else + echo "WARN: dep-metadata JSON missing (Ruby: ${ruby_meta:-none}, Go: ${go_meta:-none})" + fi + + return "${mismatches}" +} + +# ── Main ───────────────────────────────────────────────────────────────────── + +prepare_source + +run_ruby_builder + +run_go_builder + +mismatches=0 +compare_outputs || mismatches=$? + +if [[ "${mismatches}" -gt 0 ]]; then + echo "" + echo "FAIL: Parity test FAILED for ${DEP} ${VERSION} on ${STACK} (${mismatches} mismatch(es))" + exit 1 +fi + +echo "" +echo "PASS: Parity test PASSED for ${DEP} ${VERSION} on ${STACK}" diff --git a/test/parity/run-all.sh b/test/parity/run-all.sh new file mode 100755 index 00000000..7bed39d8 --- /dev/null +++ b/test/parity/run-all.sh @@ -0,0 +1,441 @@ +#!/usr/bin/env bash +# binary-builder/test/parity/run-all.sh +# Usage: run-all.sh [] [DEP=] +# +# Runs compare-builds.sh for every dep in the parity test matrix. +# For each dep we generate a real depwatcher-format data.json on-the-fly, +# then call compare-builds.sh --dep --data-json [--stack ]. +# +# To run a single dep only, set the DEP env var: +# DEP=httpd ./test/parity/run-all.sh +# DEP=httpd make parity-test +# +# Deps that require vendor credentials (appdynamics, appdynamics-java) are +# skipped with a SKIP notice; they can be tested manually when credentials +# are available. +# +# All checksums were verified against the upstream sources. +# +# Real depwatcher "modern" data.json format: +# { +# "source": {"name": "", "type": "", "repo": ""}, +# "version": {"url": "", "ref": "", +# "sha256": "", "sha512": ""} +# } + +set -euo pipefail + +STACK="${1:-cflinuxfs4}" +# Optional: filter to a single dep (set via env, e.g. DEP=httpd make parity-test) +FILTER_DEP="${DEP:-}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TMPDIR="$(mktemp -d)" + +cleanup() { rm -rf "${TMPDIR}"; } +trap cleanup EXIT + +declare -a PASSED=() +declare -a FAILED=() +declare -a FAILED_LOGS=() +declare -a SKIPPED=() + +# ── Helper ──────────────────────────────────────────────────────────────────── + +# write_data_json [] [] [] +write_data_json() { + local dep="$1" + local version="$2" + local url="$3" + local sha256="$4" + local sha512="${5:-}" + local source_type="${6:-url}" + local repo="${7:-}" + local path="${TMPDIR}/${dep}-data.json" + + jq -n \ + --arg name "${dep}" \ + --arg stype "${source_type}" \ + --arg repo "${repo}" \ + --arg url "${url}" \ + --arg ref "${version}" \ + --arg sha256 "${sha256}" \ + --arg sha512 "${sha512}" \ + '{ + "source": { "name": $name, "type": $stype, "repo": $repo }, + "version": { "url": $url, "ref": $ref, "sha256": $sha256, "sha512": $sha512 } + }' > "${path}" + + echo "${path}" +} + +run_dep() { + local dep="$1" + local data_json="$2" + + # Skip if a DEP filter is set and this dep doesn't match. + if [[ -n "${FILTER_DEP}" && "${dep}" != "${FILTER_DEP}" ]]; then return 0; fi + + echo "" + echo "════════════════════════════════════════════════════════════════" + echo " ${dep} on ${STACK}" + echo "════════════════════════════════════════════════════════════════" + + local version + version=$(jq -r '.version.ref // "unknown"' "${data_json}") + local log_file="/tmp/parity-logs/${dep}-${version}-${STACK}.log" + + if "${SCRIPT_DIR}/compare-builds.sh" --dep "${dep}" --data-json "${data_json}" --stack "${STACK}"; then + PASSED+=("${dep} ${version}") + else + FAILED+=("${dep} ${version}") + FAILED_LOGS+=("${log_file}") + echo "FAILED: ${dep} ${version} — log: ${log_file}" + fi +} + +# run_dep_r +# Wraps run_dep for `r`, creating the 4 sub-dep data.json files that the +# Go recipe reads from source-*-latest/ directories inside the container. +run_dep_r() { + local data_json="$1" + local sub_deps_dir="${TMPDIR}/r-sub-deps" + + # forecast 8.24.0 + mkdir -p "${sub_deps_dir}/source-forecast-latest" + jq -n \ + --arg name "forecast" \ + --arg stype "github_releases" \ + --arg repo "robjhyndman/forecast" \ + --arg url "https://cran.r-project.org/src/contrib/forecast_8.24.0.tar.gz" \ + --arg ref "8.24.0" \ + --arg sha256 "" \ + --arg sha512 "" \ + '{"source":{"name":$name,"type":$stype,"repo":$repo},"version":{"url":$url,"ref":$ref,"sha256":$sha256,"sha512":$sha512}}' \ + > "${sub_deps_dir}/source-forecast-latest/data.json" + + # plumber 1.3.0 + mkdir -p "${sub_deps_dir}/source-plumber-latest" + jq -n \ + --arg name "plumber" \ + --arg stype "github_releases" \ + --arg repo "rstudio/plumber" \ + --arg url "https://cran.r-project.org/src/contrib/plumber_1.3.0.tar.gz" \ + --arg ref "1.3.0" \ + --arg sha256 "" \ + --arg sha512 "" \ + '{"source":{"name":$name,"type":$stype,"repo":$repo},"version":{"url":$url,"ref":$ref,"sha256":$sha256,"sha512":$sha512}}' \ + > "${sub_deps_dir}/source-plumber-latest/data.json" + + # rserve 1.8.15 + mkdir -p "${sub_deps_dir}/source-rserve-latest" + jq -n \ + --arg name "Rserve" \ + --arg stype "github_releases" \ + --arg repo "s-u/Rserve" \ + --arg url "https://cran.r-project.org/src/contrib/Rserve_1.8-15.tar.gz" \ + --arg ref "1.8.15" \ + --arg sha256 "" \ + --arg sha512 "" \ + '{"source":{"name":$name,"type":$stype,"repo":$repo},"version":{"url":$url,"ref":$ref,"sha256":$sha256,"sha512":$sha512}}' \ + > "${sub_deps_dir}/source-rserve-latest/data.json" + + # shiny 1.10.0 + mkdir -p "${sub_deps_dir}/source-shiny-latest" + jq -n \ + --arg name "shiny" \ + --arg stype "github_releases" \ + --arg repo "rstudio/shiny" \ + --arg url "https://cran.r-project.org/src/contrib/shiny_1.10.0.tar.gz" \ + --arg ref "1.10.0" \ + --arg sha256 "" \ + --arg sha512 "" \ + '{"source":{"name":$name,"type":$stype,"repo":$repo},"version":{"url":$url,"ref":$ref,"sha256":$sha256,"sha512":$sha512}}' \ + > "${sub_deps_dir}/source-shiny-latest/data.json" + + local dep="r" + + # Skip if a DEP filter is set and this dep doesn't match. + if [[ -n "${FILTER_DEP}" && "${dep}" != "${FILTER_DEP}" ]]; then return 0; fi + + local version + version=$(jq -r '.version.ref // "unknown"' "${data_json}") + local log_file="/tmp/parity-logs/${dep}-${version}-${STACK}.log" + + echo "" + echo "════════════════════════════════════════════════════════════════" + echo " ${dep} on ${STACK}" + echo "════════════════════════════════════════════════════════════════" + + if "${SCRIPT_DIR}/compare-builds.sh" --dep "${dep}" --data-json "${data_json}" \ + --stack "${STACK}" --sub-deps-dir "${sub_deps_dir}"; then + PASSED+=("${dep} ${version}") + else + FAILED+=("${dep} ${version}") + FAILED_LOGS+=("${log_file}") + echo "FAILED: ${dep} ${version} — log: ${log_file}" + fi +} + +skip_dep() { + local dep="$1" + local reason="$2" + + # Skip silently if a DEP filter is set and this dep doesn't match. + if [[ -n "${FILTER_DEP}" && "${dep}" != "${FILTER_DEP}" ]]; then return 0; fi + + echo "" + echo "════════════════════════════════════════════════════════════════" + echo " ${dep} — SKIPPED: ${reason}" + echo "════════════════════════════════════════════════════════════════" + SKIPPED+=("${dep} (${reason})") +} + +# ── Test matrix ─────────────────────────────────────────────────────────────── +# All source URLs and SHA256 checksums verified against upstream. + +# ruby 3.3.6 — https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.6.tar.gz +run_dep ruby "$(write_data_json ruby 3.3.6 \ + "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.6.tar.gz" \ + "8dc48fffaf270f86f1019053f28e51e4da4cce32a36760a0603a9aee67d7fd8d" \ + "" github_releases "ruby/ruby")" + +# jruby 9.4.14.0 — https://repo1.maven.org/maven2/org/jruby/jruby-dist/9.4.14.0/ +# NOTE: Maven only publishes a .zip (no .tar.gz); the Go builder downloads .zip. +run_dep jruby "$(write_data_json jruby 9.4.14.0 \ + "https://repo1.maven.org/maven2/org/jruby/jruby-dist/9.4.14.0/jruby-dist-9.4.14.0-src.zip" \ + "400086b33f701a47dc28c5965d5a408bc2740301a5fb3b545e37abaa002ccdf8" \ + "" maven "")" + +# python 3.12.0 — https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tgz +run_dep python "$(write_data_json python 3.12.0 \ + "https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tgz" \ + "51412956d24a1ef7c97f1cb5f70e185c13e3de1f50d131c0aac6338080687afb" \ + "" url "")" + +# node 20.11.0 — https://nodejs.org/dist/v20.11.0/node-v20.11.0.tar.gz +run_dep node "$(write_data_json node 20.11.0 \ + "https://nodejs.org/dist/v20.11.0/node-v20.11.0.tar.gz" \ + "9884b22d88554d65025352ba7e4cb20f5d17a939231bea41a7894c0344fab1bf" \ + "" url "")" + +# go 1.22.0 — https://go.dev/dl/go1.22.0.src.tar.gz +run_dep go "$(write_data_json go 1.22.0 \ + "https://go.dev/dl/go1.22.0.src.tar.gz" \ + "4d196c3d41a0d6c1dfc64d04e3cc1f608b0c436bd87b7060ce3e23234e1f4d5c" \ + "" url "")" + +# nginx 1.25.3 — https://nginx.org/download/nginx-1.25.3.tar.gz +run_dep nginx "$(write_data_json nginx 1.25.3 \ + "https://nginx.org/download/nginx-1.25.3.tar.gz" \ + "64c5b975ca287939e828303fa857d22f142b251f17808dfe41733512d9cded86" \ + "" url "")" + +# nginx-static 1.25.3 — same source as nginx +run_dep nginx-static "$(write_data_json nginx-static 1.25.3 \ + "https://nginx.org/download/nginx-1.25.3.tar.gz" \ + "64c5b975ca287939e828303fa857d22f142b251f17808dfe41733512d9cded86" \ + "" url "")" + +# openresty 1.25.3.1 — https://openresty.org/download/openresty-1.25.3.1.tar.gz +run_dep openresty "$(write_data_json openresty 1.25.3.1 \ + "https://openresty.org/download/openresty-1.25.3.1.tar.gz" \ + "32ec1a253a5a13250355a075fe65b7d63ec45c560bbe213350f0992a57cd79df" \ + "" url "")" + +# httpd 2.4.58 — Go recipe downloads .tar.bz2 (not .tar.gz); sha256 is for .tar.bz2 +run_dep httpd "$(write_data_json httpd 2.4.58 \ + "https://archive.apache.org/dist/httpd/httpd-2.4.58.tar.gz" \ + "fa16d72a078210a54c47dd5bef2f8b9b8a01d94909a51453956b3ec6442ea4c5" \ + "" url "")" + +# bundler 2.5.6 — https://rubygems.org/gems/bundler-2.5.6.gem +run_dep bundler "$(write_data_json bundler 2.5.6 \ + "https://rubygems.org/gems/bundler-2.5.6.gem" \ + "1a1f21d1456e16dd2fee93461d9640348047aa2dcaf5d776874a60ddd4df5c64" \ + "" url "")" + +# rubygems 3.5.6 — https://rubygems.org/rubygems/rubygems-3.5.6.tgz +run_dep rubygems "$(write_data_json rubygems 3.5.6 \ + "https://rubygems.org/rubygems/rubygems-3.5.6.tgz" \ + "f3fcc0327cee0b7ebbee2ef014a42ba05b4032d7e1834dbcd3165dde700c99c2" \ + "" url "")" + +# r 4.4.2 — https://cran.r-project.org/src/base/R-4/R-4.4.2.tar.gz +# Uses run_dep_r to supply the 4 sub-dep data.json files required by the Go recipe. +run_dep_r "$(write_data_json r 4.4.2 \ + "https://cran.r-project.org/src/base/R-4/R-4.4.2.tar.gz" \ + "1578cd603e8d866b58743e49d8bf99c569e81079b6a60cf33cdf7bdffeb817ec" \ + "" url "")" + +# libunwind 1.6.2 — use the release download URL so both builders agree on the directory name. +# The refs/tags archive URL extracts to libunwind-1.6.2/ but Ruby builder derives "v1.6.2" from +# the filename v1.6.2.tar.gz and tries Dir.chdir("v1.6.2"), which fails. +# The release tarball libunwind-1.6.2.tar.gz extracts to libunwind-1.6.2/ and Ruby builder +# correctly derives "libunwind-1.6.2" from the filename. +run_dep libunwind "$(write_data_json libunwind 1.6.2 \ + "https://github.com/libunwind/libunwind/releases/download/v1.6.2/libunwind-1.6.2.tar.gz" \ + "4a6aec666991fb45d0889c44aede8ad6eb108071c3554fcdff671f9c94794976" \ + "" github_releases "libunwind/libunwind")" + +# libgdiplus 6.1 — https://github.com/mono/libgdiplus/archive/refs/tags/6.1.tar.gz +run_dep libgdiplus "$(write_data_json libgdiplus 6.1 \ + "https://github.com/mono/libgdiplus/archive/refs/tags/6.1.tar.gz" \ + "6ba47acef48ffa2a75d71f8958e0de7f8f52ea066ed97409b33e7a32f31835fd" \ + "" github_releases "mono/libgdiplus")" + +# hwc 106.0.0 — https://github.com/cloudfoundry/hwc/archive/refs/tags/106.0.0.tar.gz +run_dep hwc "$(write_data_json hwc 106.0.0 \ + "https://github.com/cloudfoundry/hwc/archive/refs/tags/106.0.0.tar.gz" \ + "87fe14594a5d51f43680a84a669ff1ae7b1ec64630608726beeca172ab0d4163" \ + "" github_releases "cloudfoundry/hwc")" + +# pip 24.0 — https://files.pythonhosted.org/... +run_dep pip "$(write_data_json pip 24.0 \ + "https://files.pythonhosted.org/packages/94/59/6638090c25e9bc4ce0c42817b5a234e183872a1129735a9330c472cc2056/pip-24.0.tar.gz" \ + "ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2" \ + "" url "")" + +# pipenv 2023.12.1 — https://files.pythonhosted.org/... +run_dep pipenv "$(write_data_json pipenv 2023.12.1 \ + "https://files.pythonhosted.org/packages/a6/26/5cdf9f0c6eb835074c3e43dde2880bfa739daa23fa534a5dd65848af5913/pipenv-2023.12.1.tar.gz" \ + "4aea73e23944e464ad2b849328e780ad121c5336e1c24a7ac15aa493c41c2341" \ + "" url "")" + +# setuptools 69.0.3 — https://files.pythonhosted.org/... +run_dep setuptools "$(write_data_json setuptools 69.0.3 \ + "https://files.pythonhosted.org/packages/fc/c9/b146ca195403e0182a374e0ea4dbc69136bad3cd55bc293df496d625d0f7/setuptools-69.0.3.tar.gz" \ + "be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78" \ + "" url "")" + +# yarn 1.22.21 — https://registry.npmjs.org/yarn/-/yarn-1.22.21.tgz +run_dep yarn "$(write_data_json yarn 1.22.21 \ + "https://registry.npmjs.org/yarn/-/yarn-1.22.21.tgz" \ + "dbed5b7e10c552ba0e1a545c948d5473bc6c5a28ce22a8fd27e493e3e5eb6370" \ + "" url "")" + +# bower 1.8.14 — https://registry.npmjs.org/bower/-/bower-1.8.14.tgz +run_dep bower "$(write_data_json bower 1.8.14 \ + "https://registry.npmjs.org/bower/-/bower-1.8.14.tgz" \ + "00df3dcc6e8b3a4dd7668934a20e60e6fc0c4269790192179388c928553a3f7e" \ + "" url "")" + +# composer 2.7.1 — https://github.com/composer/composer/releases/download/2.7.1/composer.phar +run_dep composer "$(write_data_json composer 2.7.1 \ + "https://github.com/composer/composer/releases/download/2.7.1/composer.phar" \ + "1ffd0be3f27e237b1ae47f9e8f29f96ac7f50a0bd9eef4f88cdbe94dd04bfff0" \ + "" github_releases "composer/composer")" + +# tomcat 10.1.18 — https://archive.apache.org/dist/tomcat/tomcat-10/v10.1.18/bin/ +run_dep tomcat "$(write_data_json tomcat 10.1.18 \ + "https://archive.apache.org/dist/tomcat/tomcat-10/v10.1.18/bin/apache-tomcat-10.1.18.tar.gz" \ + "6da0b4cbd3140e64a8719a2de19c20bf3902d264a142a816ac552ae216ade311" \ + "" url "")" + +# openjdk 11.0.22_7 — Adoptium Temurin JDK 11 +run_dep openjdk "$(write_data_json openjdk 11.0.22_7 \ + "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.22%2B7/OpenJDK11U-jdk_x64_linux_hotspot_11.0.22_7.tar.gz" \ + "25cf602cac350ef36067560a4e8042919f3be973d419eac4d839e2e0000b2cc8" \ + "" github_releases "adoptium/temurin11-binaries")" + +# zulu 21.32.17 — Azul Zulu JDK 21.0.2 +run_dep zulu "$(write_data_json zulu 21.32.17 \ + "https://cdn.azul.com/zulu/bin/zulu21.32.17-ca-jdk21.0.2-linux_x64.tar.gz" \ + "5ad730fbee6bb49bfff10bf39e84392e728d89103d3474a7e5def0fd134b300a" \ + "" zulu "")" + +# sapmachine 21.0.2 — SAP Machine JDK 21 +run_dep sapmachine "$(write_data_json sapmachine 21.0.2 \ + "https://github.com/SAP/SapMachine/releases/download/sapmachine-21.0.2/sapmachine-jdk-21.0.2_linux-x64_bin.tar.gz" \ + "3123189ec5b99eed78de0328e2fd49d7c13cc7d4524c341f1fe8fbd5165be31f" \ + "" github_releases "SAP/SapMachine")" + +# skywalking-agent 9.5.0 — Apache SkyWalking Java Agent (SHA512 checksum) +run_dep skywalking-agent "$(write_data_json skywalking-agent 9.5.0 \ + "https://archive.apache.org/dist/skywalking/java-agent/9.5.0/apache-skywalking-java-agent-9.5.0.tgz" \ + "" \ + "deb782b41e6cde1e4eae94f806bb73bccb0f6bd0362c6b9f90e387a6d84bad672c34b70ca204f9e5f74899726542c76c36b2e2af05ecbcab8fff73a661a3de21" \ + "url" "")" + +# jprofiler-profiler 15.0.4 — ej-technologies JProfiler +# URL: https://download.ej-technologies.com/jprofiler/jprofiler_linux_15_0_4.tar.gz +run_dep jprofiler-profiler "$(write_data_json jprofiler-profiler 15.0.4 \ + "https://download.ej-technologies.com/jprofiler/jprofiler_linux_15_0_4.tar.gz" \ + "fec741718854a11b2383bb278ca7103984e0ae659268ed53ea5a8b32077b86c9" \ + "" jprofiler "")" + +# your-kit-profiler 2025.9.185 — YourKit Java Profiler (latest publicly available) +# Version format: .. → URL uses year.minor/YourKit-JavaProfiler-year.minor-b-x64.zip +run_dep your-kit-profiler "$(write_data_json your-kit-profiler 2025.9.185 \ + "https://download.yourkit.com/yjp/2025.9/YourKit-JavaProfiler-2025.9-b185-x64.zip" \ + "1818a6f74ef231e53876c66ba9e7e4f0952f57cb1af40c2d410e21a6da8c33b7" \ + "" yourkit "")" + +# php 8.1.32 — https://www.php.net/distributions/php-8.1.32.tar.gz +run_dep php "$(write_data_json php 8.1.32 \ + "https://www.php.net/distributions/php-8.1.32.tar.gz" \ + "4846836d1de27dbd28e89180f073531087029a77e98e8e019b7b2eddbdb1baff" \ + "" url "")" + +# dotnet-sdk 8.0.101 — Microsoft .NET SDK (SHA512 checksum, not SHA256) +run_dep dotnet-sdk "$(write_data_json dotnet-sdk 8.0.101 \ + "https://builds.dotnet.microsoft.com/dotnet/Sdk/8.0.101/dotnet-sdk-8.0.101-linux-x64.tar.gz" \ + "" \ + "26df0151a3a59c4403b52ba0f0df61eaa904110d897be604f19dcaa27d50860c82296733329cb4a3cf20a2c2e518e8f5d5f36dfb7931bf714a45e46b11487c9a" \ + "url" "")" + +# dotnet-runtime 8.0.1 — Microsoft .NET Runtime (SHA512 checksum) +run_dep dotnet-runtime "$(write_data_json dotnet-runtime 8.0.1 \ + "https://builds.dotnet.microsoft.com/dotnet/Runtime/8.0.1/dotnet-runtime-8.0.1-linux-x64.tar.gz" \ + "" \ + "cbd03325280ff93cd0edab71c5564a50bb2423980f63d04602914db917c9c811a0068d848cab07d82e3260bff6684ad7cffacc2f449c06fc0b0aa8f845c399b6" \ + "url" "")" + +# dotnet-aspnetcore 8.0.1 — Microsoft ASP.NET Core Runtime +run_dep dotnet-aspnetcore "$(write_data_json dotnet-aspnetcore 8.0.1 \ + "https://builds.dotnet.microsoft.com/dotnet/aspnetcore/Runtime/8.0.1/aspnetcore-runtime-8.0.1-linux-x64.tar.gz" \ + "cd825a5bd7b40e5706840d7b22650b787f71db5e2e496c80e16571bf5003f8fe" \ + "" url "")" + +# appdynamics — requires vendor credentials; skip +skip_dep appdynamics "requires AppDynamics vendor credentials (appdynamics-credentials)" +skip_dep appdynamics-java "requires AppDynamics vendor credentials (appdynamics-credentials)" + +# ── Summary ─────────────────────────────────────────────────────────────────── + +echo "" +echo "════════════════════════════════════════════════════════════════" +echo " Parity test summary — stack: ${STACK}" +echo "════════════════════════════════════════════════════════════════" +echo " Passed: ${#PASSED[@]}" +for p in "${PASSED[@]}"; do echo " ✓ ${p}"; done +echo " Failed: ${#FAILED[@]}" +for i in "${!FAILED[@]}"; do + echo " ✗ ${FAILED[$i]}" + echo " log: ${FAILED_LOGS[$i]}" +done +echo " Skipped: ${#SKIPPED[@]}" +for s in "${SKIPPED[@]}"; do echo " - ${s}"; done + +if [[ "${#FAILED[@]}" -gt 0 ]]; then + echo "" + echo "════════════════════════════════════════════════════════════════" + echo " Failure details (last 20 lines of each log)" + echo "════════════════════════════════════════════════════════════════" + for i in "${!FAILED[@]}"; do + echo "" + echo "── ${FAILED[$i]} ──────────────────────────────────────────────" + echo " ${FAILED_LOGS[$i]}" + echo "────────────────────────────────────────────────────────────────" + if [[ -f "${FAILED_LOGS[$i]}" ]]; then + tail -20 "${FAILED_LOGS[$i]}" + else + echo " (log file not found)" + fi + done + echo "" + echo "FAIL: ${#FAILED[@]} dep(s) failed parity test" + exit 1 +fi + +echo "" +echo "PASS: All ${#PASSED[@]} deps passed parity test on ${STACK} (${#SKIPPED[@]} skipped)"