itest: add itest setup with two lnd nodes and a single faraday instance

This commit adds an itest setup which can be used to test faraday's rpc.
The setup is largely copied from our existing loop server itests.

.gitignore

@@ -1,3 +1,9 @@
 /faraday
 /frcli
 node_report.csv
+
+*.exe
+
+itest/itest.test
+itest/*.log
+itest/faraday

Makefile

@@ -23,7 +23,7 @@ GOTEST := GO111MODULE=on go test -v
 GOMOD := GO111MODULE=on go mod
 
 GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
-GOLIST := go list -deps $(PKG)/... | grep '$(PKG)'| grep -v '/vendor/'
+GOLIST := go list -deps $(PKG)/... | grep '$(PKG)'| grep -v '/vendor/' | grep -v '/itest'
 GOLISTCOVER := $(shell go list -deps -f '{{.ImportPath}}' ./... | grep '$(PKG)' | sed -e 's/^$(ESCPKG)/./')
 
 RM := rm -f

itest/Dockerfile

@@ -0,0 +1,50 @@
+# Faraday integration test dockerfile
+
+FROM golang:1.13-alpine
+
+ARG BITCOIND_VERSION=0.19.1
+ARG GLIBC_VERSION=2.29-r0
+ARG LND_VERSION=v0.11.0-beta.rc2
+
+WORKDIR /root
+
+RUN apk add --no-cache git gcc musl-dev make curl bash jq
+
+# Install glibc (for bitcoind)
+RUN wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \
+  && wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk \
+  && wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-bin-${GLIBC_VERSION}.apk \
+  && apk --no-cache add glibc-${GLIBC_VERSION}.apk \
+  && apk --no-cache add glibc-bin-${GLIBC_VERSION}.apk
+
+# Install bitcoind
+RUN wget https://bitcoin.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz && \
+  tar xvfz bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz
+
+RUN mkdir .bitcoin \
+  && mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin/
+
+# Get lnd sources and install. Can't use go get here, because tags aren't passed
+# as a compiler variable, leading to lnd not reporting the tags that it is built
+# with.
+RUN git clone https://github.com/lightningnetwork/lnd.git && \
+  cd lnd && git checkout ${LND_VERSION} && \
+  make install tags="signrpc walletrpc chainrpc invoicesrpc"
+
+# Copy the integration test executable into the image.
+COPY itest.test .
+RUN chmod +x itest.test
+
+# Copy the scripts and make them executable.
+COPY *.sh ./
+RUN chmod +x *.sh
+
+# Copy compiled faraday into the image. Assumption here is that the binary is
+# up to date.
+COPY faraday .
+RUN chmod +x faraday
+
+# Run the test setup.
+RUN ./itest_setup.sh
+
+ENTRYPOINT [ "./itest.sh" ]

itest/fail_test.go

@@ -0,0 +1,9 @@
+package itest
+
+import "testing"
+
+// TestFail only serves to detect bugs that prevent itest errors from surfacing.
+// It always fails and is caught at a higher level in run_itest.sh.
+func TestFail(t *testing.T) {
+	t.Fatal("failure canary")
+}

itest/itest.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Exit from script if error was raised.
+set -e
+
+# Set exit code of this script to the exit code of the last program to exit
+# non-zero. Otherwise failures in programs that are piped into another program
+# are ignored.
+set -o pipefail
+
+source util.sh
+
+echo "Running integration test $@"
+
+start_bitcoind
+start_lnds
+
+cd $WORKDIR
+./itest.test -test.run=$@
+
+stop_all
+
+echo "Integration test $@ passed"

itest/itest_setup.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Exit from script if error was raised.
+set -e
+
+# Set exit code of this script to the exit code of the last program to exit
+# non-zero. Otherwise failures in programs that are piped into another program
+# are ignored.
+set -o pipefail
+
+# Load common script code.
+source util.sh
+
+echo "Running integration test setup"
+
+start_bitcoind
+
+echo "Mining initial blocks"
+$BTCCTL generatetoaddress 400 2N9kBLwWmJjoPxBddwR8G9hwLMrQyHum44K
+
+start_lnds
+
+# Run test setup code. The test TestSetup is a special test that is ran during
+# the build of the docker image.
+./itest.test -test.run=TestSetup
+
+stop_all
+
+echo "Set up complete"

itest/log.go

@@ -0,0 +1,10 @@
+package itest
+
+import (
+	"github.com/btcsuite/btclog"
+)
+
+var (
+	backend = btclog.NewBackend(newPrefixStdout("itest"))
+	log     = backend.Logger("")
+)

itest/prefix_writer.go

@@ -0,0 +1,61 @@
+package itest
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+)
+
+// newPrefixStdout returns a new io.Writer instance that prefixes every line and
+// writes it to stdout.
+func newPrefixStdout(prefix string) *prefixWriter {
+	return &prefixWriter{
+		writer: os.Stdout,
+		prefix: prefix,
+	}
+}
+
+// prefixWriter is a pass-through writer that prefixes every line with the given
+// prefix.
+type prefixWriter struct {
+	writer io.Writer
+	prefix string
+
+	prefixWritten bool
+}
+
+// Write writes a slice of bytes to the underlying writer, after inserting
+// prefixes.
+func (w *prefixWriter) Write(p []byte) (int, error) {
+	var b bytes.Buffer
+	for _, c := range p {
+		// Only write a prefix is there are more characters coming.
+		// Otherwise the final output may get skewed if there are
+		// multiple processes writing logs.
+		if !w.prefixWritten {
+			b.WriteString(fmt.Sprintf("[%v] ", w.prefix))
+			w.prefixWritten = true
+		}
+
+		b.WriteByte(c)
+		if c == '\n' {
+			w.prefixWritten = false
+		}
+	}
+
+	_, err := w.writer.Write(b.Bytes())
+	if err != nil {
+		return 0, err
+	}
+
+	return len(p), nil
+}
+
+// attachPrefixStdout attaches a prefixed stdout writer to the command's stdout
+// and stderr output.
+func attachPrefixStdout(cmd *exec.Cmd, prefix string) {
+	cmd.Stdout = newPrefixStdout(prefix)
+	cmd.Stderr = newPrefixStdout(prefix + "-err")
+}

itest/rpc.go

@@ -0,0 +1,39 @@
+package itest
+
+import (
+	"fmt"
+
+	"github.com/btcsuite/btcd/rpcclient"
+	"github.com/lightninglabs/faraday/frdrpc"
+	"google.golang.org/grpc"
+)
+
+// getBitcoindClient returns an rpc client connection to the running bitcoind
+// daemon.
+func getBitcoindClient() (*rpcclient.Client, error) {
+	connCfg := &rpcclient.ConnConfig{
+		Host:         "localhost:18443",
+		User:         "devuser",
+		Pass:         "devpass",
+		HTTPPostMode: true,
+		DisableTLS:   true,
+	}
+
+	return rpcclient.New(connCfg, nil)
+}
+
+// getFaradayClient returns an rpc client connection to the running faraday
+// instance.
+func getFaradayClient(address string) (frdrpc.FaradayServerClient, error) {
+	opts := []grpc.DialOption{
+		grpc.WithInsecure(),
+	}
+
+	conn, err := grpc.Dial(address, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("unable to connect to RPC server: %v",
+			err)
+	}
+
+	return frdrpc.NewFaradayServerClient(conn), nil
+}

itest/setup_test.go

@@ -0,0 +1,41 @@
+package itest
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestSetup sets up a shared test environment.
+func TestSetup(t *testing.T) {
+	log.Infof("Running golang test setup")
+
+	c := newTestContext(t)
+
+	// Supply client and server with coins.
+	aliceAddr, err := c.aliceClient.WalletKit.NextAddr(
+		context.Background(),
+	)
+	require.NoError(c.t, err)
+
+	_, err = c.bitcoindClient.GenerateToAddress(1, aliceAddr, nil)
+	require.NoError(c.t, err)
+
+	bobAddr, err := c.bobClient.WalletKit.NextAddr(
+		context.Background(),
+	)
+	require.NoError(c.t, err)
+
+	_, err = c.bitcoindClient.GenerateToAddress(1, bobAddr, nil)
+	require.NoError(c.t, err)
+
+	// Mine 100 blocks to allow spending of the coinbase txes.
+	c.mineBlocks(100)
+
+	c.waitBalance(func(b balances) bool {
+		return b.aliceWallet > 0 && b.bobWallet > 0
+	})
+
+	log.Infof("Test setup complete")
+}

itest/test_context.go

@@ -0,0 +1,352 @@
+package itest
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/rpcclient"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/lightninglabs/faraday/frdrpc"
+	"github.com/lightninglabs/lndclient"
+	"github.com/lightningnetwork/lnd/routing/route"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	waitDuration = 20 * time.Second
+	waitTick     = 500 * time.Millisecond
+
+	processKillTimeout = 5 * time.Second
+
+	faradayCmd = "./faraday"
+
+	faradayArgs = []string{
+		"--rpclisten=localhost:8465",
+		"--regtest",
+		"--macaroondir=lnd-alice/data/chain/bitcoin/regtest",
+		"--tlscertpath=lnd-alice/tls.cert",
+		"--debuglevel=debug",
+	}
+)
+
+// testContext provides a set up test environment for itests to run in.
+type testContext struct {
+	bitcoindClient *rpcclient.Client
+
+	aliceClient *lndclient.GrpcLndServices
+	bobClient   *lndclient.GrpcLndServices
+
+	faradayClient frdrpc.FaradayServerClient
+
+	alicePubkey, bobPubkey route.Vertex
+
+	dummyAddress btcutil.Address
+
+	faradayCmd *exec.Cmd
+	faradayErr chan error
+
+	t *testing.T
+}
+
+// newTestContext returns a new context instance.
+func newTestContext(t *testing.T) *testContext {
+	var err error
+
+	ctx := &testContext{
+		t: t,
+	}
+
+	// Setup rpc client for bitcoind.
+	ctx.bitcoindClient, err = getBitcoindClient()
+	require.NoError(t, err)
+
+	// Set a dummy mining address.
+	ctx.dummyAddress, err = btcutil.DecodeAddress(
+		"2N9kBLwWmJjoPxBddwR8G9hwLMrQyHum44K",
+		&chaincfg.RegressionNetParams,
+	)
+	require.NoError(t, err)
+
+	// Setup rpc client for lnd instances.
+	ctx.aliceClient, err = lndclient.NewLndServices(
+		&lndclient.LndServicesConfig{
+			LndAddress:  "localhost:10009",
+			Network:     lndclient.NetworkRegtest,
+			MacaroonDir: "lnd-alice/data/chain/bitcoin/regtest",
+			TLSPath:     "lnd-alice/tls.cert",
+		},
+	)
+	require.NoError(t, err)
+
+	ctx.bobClient, err = lndclient.NewLndServices(
+		&lndclient.LndServicesConfig{
+			LndAddress:  "localhost:10002",
+			Network:     lndclient.NetworkRegtest,
+			MacaroonDir: "lnd-bob/data/chain/bitcoin/regtest",
+			TLSPath:     "lnd-bob/tls.cert",
+		},
+	)
+	require.NoError(t, err)
+
+	// Get lnd instance info.
+	aliceInfo, err := ctx.aliceClient.Client.GetInfo(context.Background())
+	require.NoError(t, err)
+
+	ctx.alicePubkey, err = route.NewVertexFromBytes(aliceInfo.IdentityPubkey[:])
+	require.NoError(t, err)
+
+	bobInfo, err := ctx.bobClient.Client.GetInfo(context.Background())
+	require.NoError(t, err)
+
+	ctx.bobPubkey, err = route.NewVertexFromBytes(bobInfo.IdentityPubkey[:])
+	require.NoError(t, err)
+
+	// Start faraday.
+	ctx.startFaraday()
+
+	return ctx
+}
+
+// mine signals btcd to mine the given number of blocks.
+func (c *testContext) mineBlocks(blocks uint32) {
+	_, err := c.bitcoindClient.GenerateToAddress(
+		int64(blocks), c.dummyAddress, nil,
+	)
+	require.NoError(c.t, err)
+}
+
+// mine mines a block and returns the number of included txes.
+func (c *testContext) mine() int {
+	c.t.Helper()
+
+	blockHashes, err := c.bitcoindClient.GenerateToAddress(
+		1, c.dummyAddress, nil,
+	)
+	require.NoError(c.t, err)
+
+	hash := blockHashes[0]
+	block, err := c.bitcoindClient.GetBlock(hash)
+	require.NoError(c.t, err)
+
+	// Subtract coinbase tx.
+	return len(block.Transactions) - 1
+}
+
+// mine mines a block and verifies that the expected number of transactions is
+// present (excluding the coinbase tx).
+func (c *testContext) mineExactly(expectedTxCount int) {
+	c.t.Helper()
+
+	txCount := c.mine()
+	require.Equal(c.t, expectedTxCount, txCount)
+}
+
+// mempoolTxCount returns the number of txes currently in the mempool.
+func (c *testContext) mempoolTxCount() int {
+	txes, err := c.bitcoindClient.GetRawMempool()
+	require.NoError(c.t, err)
+
+	return len(txes)
+}
+
+// balances stores the wallet and channel balances for alice and bob.
+type balances struct {
+	aliceWallet, aliceChannel btcutil.Amount
+	bobWallet, bobChannel     btcutil.Amount
+}
+
+// String returns human-readable balances
+func (b balances) String() string {
+	return fmt.Sprintf(
+		"alice: wallet=%v,channel=%v, bob: wallet=%v,channel=%v",
+		b.aliceWallet, b.aliceChannel,
+		b.bobWallet, b.bobChannel,
+	)
+}
+
+// getBalances returns the balances for the client and server lnd instances.
+func (c *testContext) getBalances() balances {
+	get := func(lnd lndclient.LightningClient) (btcutil.Amount,
+		btcutil.Amount) {
+
+		walletResp, err := lnd.ConfirmedWalletBalance(
+			context.Background(),
+		)
+		require.NoError(c.t, err)
+
+		channelResp, err := lnd.ConfirmedWalletBalance(
+			context.Background(),
+		)
+		require.NoError(c.t, err)
+
+		return walletResp, channelResp
+	}
+
+	var b balances
+	b.aliceWallet, b.aliceChannel = get(c.aliceClient.Client)
+	b.bobWallet, b.bobChannel = get(c.bobClient.Client)
+
+	return b
+}
+
+// waitBalance keeps querying the lnd balances until the given condition is met.
+func (c *testContext) waitBalance(condition func(balances) bool) {
+	c.t.Helper()
+
+	c.eventuallyf(
+		func() bool {
+			return condition(c.getBalances())
+		},
+		"timeout waiting for balance",
+	)
+}
+
+// eventuallyf wraps testify's Eventuallyf method with default time parameters.
+func (c *testContext) eventuallyf(condition func() bool, msg string,
+	args ...interface{}) { // nolint:unparam
+
+	c.t.Helper()
+
+	require.Eventuallyf(
+		c.t, condition, waitDuration, waitTick, msg, args,
+	)
+}
+
+// waitForChannelOpen waits for a channel between alice and bob to become
+// active.
+func (c *testContext) waitForChannelOpen(targetChannel *wire.OutPoint) {
+	c.t.Helper()
+
+	c.eventuallyf(
+		func() bool {
+			c.mine()
+
+			aliceChans, err := c.aliceClient.Client.ListChannels(
+				context.Background(),
+			)
+			require.NoError(c.t, err)
+
+			// If we did not find our target channel, we fail.
+			if findChannel(aliceChans, targetChannel) == nil {
+				return false
+			}
+
+			bobChans, err := c.bobClient.Client.ListChannels(
+				context.Background(),
+			)
+			require.NoError(c.t, err)
+
+			// Succeed if we found our channel in in bob's channels.
+			return findChannel(bobChans, targetChannel) != nil
+		},
+		"channel not open",
+	)
+}
+
+// findChannel finds a channel in a set of open channels, returning nil if it
+// is not found.
+// nolint:interfacer
+func findChannel(channels []lndclient.ChannelInfo,
+	target *wire.OutPoint) *lndclient.ChannelInfo {
+
+	for _, channel := range channels {
+		if channel.ChannelPoint == target.String() {
+			// Declare a variable in our scope so we don't return
+			// a pointer to a range variable.
+			foundChannel := channel
+			return &foundChannel
+		}
+	}
+
+	return nil
+}
+
+// waitForMempoolTxCount waits until the specified number of txes are present in
+// the mempool
+func (c *testContext) waitForMempoolTxCount(txCount int, msg string) {
+	c.t.Helper()
+
+	c.eventuallyf(
+		func() bool {
+			return c.mempoolTxCount() == txCount
+		},
+		msg,
+	)
+}
+
+// waitForTxesAndMine waits for a specified number of txes to arrive in the
+// mempool and then mines a block.
+func (c *testContext) waitForTxesAndMine(txCount int, msg string) {
+	c.t.Helper()
+
+	c.waitForMempoolTxCount(txCount, msg)
+	c.mineExactly(txCount)
+}
+
+// mempoolEmpty asserts that the mempool is empty.
+func (c *testContext) mempoolEmpty() {
+	c.t.Helper()
+
+	require.Equal(c.t, 0, c.mempoolTxCount(), "mempool not empty")
+}
+
+// startFaraday starts faraday, connecting to our test context's alice lnd node.
+// It returns process start errors and an error channel for errors that occur
+// after the start.
+func (c *testContext) startFaraday() {
+	// Start loop client daemon.
+	c.faradayCmd = exec.Command(
+		faradayCmd, faradayArgs...,
+	)
+
+	attachPrefixStdout(c.faradayCmd, "faraday")
+
+	log.Info("Starting Faraday")
+	require.NoError(c.t, c.faradayCmd.Start())
+
+	c.faradayErr = make(chan error, 1)
+	go func() {
+		c.faradayErr <- c.faradayCmd.Wait()
+	}()
+
+	// Setup connection to faraday.
+	var err error
+	c.faradayClient, err = getFaradayClient("localhost:8465")
+	require.NoError(c.t, err)
+
+	// Wait for connectivity.
+	c.eventuallyf(func() bool {
+		_, err = c.faradayClient.ChannelInsights(
+			context.Background(), &frdrpc.ChannelInsightsRequest{},
+		)
+		return err == nil
+	}, "could not connect to faraday process: %v", err)
+}
+
+// stopFaraday stops the faraday process.
+func (c *testContext) stopFaraday() {
+	if c.faradayCmd == nil {
+		return
+	}
+
+	// Kill the faraday process.
+	require.NoError(c.t, c.faradayCmd.Process.Kill())
+
+	select {
+	case <-c.faradayErr:
+	case <-time.After(processKillTimeout):
+		require.FailNow(c.t, "cannot kill faraday process")
+	}
+
+	c.faradayCmd = nil
+}
+
+// stop stops the faraday process.
+func (c *testContext) stop() {
+	c.stopFaraday()
+}

itest/util.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+function waitnoerror() {
+        for i in {1..30}; do $@ && return; sleep 1; done
+        echo "timeout"
+        exit 1
+}
+
+function start_bitcoind() {
+        echo "Starting bitcoind"
+        bitcoind -regtest -txindex -rpcuser=devuser -rpcpassword=devpass \
+        -zmqpubrawblock=tcp://0.0.0.0:29332 -zmqpubrawtx=tcp://0.0.0.0:29333 &
+
+        BTCD_PID=$!
+
+        BTCCTL="bitcoin-cli -regtest -rpcuser=devuser -rpcpassword=devpass"
+
+        # Wait for btcd startup
+        waitnoerror $BTCCTL getblockchaininfo
+}
+
+function start_lnds() {
+        echo "Starting lnd"
+        lnd --bitcoin.active --bitcoin.node=bitcoind --bitcoin.regtest --bitcoind.rpcuser=devuser \
+        --bitcoind.zmqpubrawblock=tcp://localhost:29332 --bitcoind.zmqpubrawtx=tcp://localhost:29333 \
+        --bitcoind.rpcpass=devpass --noseedbackup --nobootstrap --lnddir=lnd-alice \
+        -d trace | awk '{ print "[lnd-alice] " $0; }' &
+
+        LND_SERVER_PID=$!
+
+        lnd --bitcoin.active --bitcoin.node=bitcoind --bitcoin.regtest --bitcoind.rpcuser=devuser \
+        --bitcoind.rpcpass=devpass --noseedbackup --nobootstrap  --rpclisten=localhost:10002 \
+        --bitcoind.zmqpubrawblock=tcp://localhost:29332 --bitcoind.zmqpubrawtx=tcp://localhost:29333 \
+        --listen=localhost:10012 --restlisten=localhost:8002 \
+        --lnddir=lnd-bob | awk '{ print "[lnd-bob] " $0; }' &
+
+        LND_CLIENT_PID=$!
+
+        LNCLI_SERVER="lncli --network regtest --lnddir lnd-alice"
+        LNCLI_CLIENT="lncli --network regtest --lnddir lnd-bob --rpcserver=localhost:10002"
+
+        waitnoerror $LNCLI_SERVER getinfo
+        waitnoerror $LNCLI_CLIENT getinfo
+}
+
+function stop_all() {
+        $LNCLI_CLIENT stop
+        $LNCLI_SERVER stop
+        $BTCCTL stop
+
+        wait $BTCD_PID
+        wait $LND_CLIENT_PID
+        wait $LND_SERVER_PID
+}

run_itest.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -e
+
+TESTCASE=$1
+
+# Build faraday for transfer to docker container. Without CGO_ENABLED=0,
+# and the correct OS/ARCH set, the binary will not run in an alpine environment.
+# Copy the binary to the itest directory which serves as the docker build
+# context.
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make build
+cp faraday itest
+
+# Build itest executable for transfer to docker container.
+cd itest
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -tags itest
+
+# Build itest image.
+docker build -t faraday .
+
+# If a command line parameter is present, only execute that test and output
+# directly to stdout.
+if [[ $TESTCASE ]]; then
+        docker run --rm faraday $TESTCASE
+        exit
+fi
+
+# Verify that the image correctly bubbles up a failed test.
+if docker run --rm faraday TestFail; then
+        echo "Always failing test not failed"
+        exit 1
+fi
+
+# Clear log files from previous run.
+rm *.log || true
+
+run_test() {
+        echo "$1 started"
+        LOG_FILE="$1.log"
+        
+        # Start a new container to only run this specific test case. Direct all
+        # output to a test-specific log file.
+        docker run --rm faraday $1 > $LOG_FILE 2>&1
+        TEST_EXIT_CODE=$?
+
+        if [[ $TEST_EXIT_CODE -eq 0 ]]; then
+                echo "$1 passed"
+        else
+                echo "$1 failed"
+        fi
+
+        return $TEST_EXIT_CODE
+}
+
+# Export the function so that it is available with xargs below.
+export -f run_test
+
+# Query list of tests. Exclude special setup and fail test cases which ran
+# already.
+TESTS=$(./itest.test -test.list . | grep -vxE "TestSetup|TestFail")
+
+# Run test cases in parallel with a maximum degree.
+MAX_PARALLEL_TESTS=4
+echo "$TESTS" | xargs -P $MAX_PARALLEL_TESTS -I {} bash -c "run_test {}"
+
+# Dump log files so that they become visible in CI.
+echo "$TESTS" | xargs -I {} bash -c "echo ------ {} ------ && cat {}.log && echo"