itest: copy integration test framework from lnd

.github/workflows/main.yml

@@ -171,3 +171,51 @@ jobs:
 
       - name: compile rpc for golang
         run: make rpc
+
+  itest:
+    name: integration test
+    runs-on: ubuntu-latest
+    steps:
+      - name: set git config
+        run: |
+          git config --global core.eol lf
+          git config --global core.autocrlf false
+
+      - name: git checkout
+        uses: actions/checkout@v2
+
+      - name: setup nodejs v${{ matrix.node_version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+
+      - name: download cache
+        uses: actions/cache@v1
+        with:
+          path: /home/runner/work/download_cache
+          key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_protoc.sh') }}
+          restore-keys: |
+            lnd-${{ runner.os }}-download-${{ hashFiles('**/install_protoc.sh') }}
+            lnd-${{ runner.os }}-download-
+
+      - name: get yarn cache dir
+        id: yarn-cache-dir
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - name: yarn cache
+        uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ matrix.node_version }}-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-${{ matrix.node_version }}-${{ hashFiles('**/yarn.lock') }}
+            ${{ runner.os }}-yarn-${{ matrix.node_version }}-
+            ${{ runner.os }}-yarn-
+
+      - name: install dependencies
+        working-directory: ./app
+        run: yarn
+
+      - name: run check
+        run: make itest

.gitignore

@@ -6,4 +6,10 @@ litcli-debug
 /lightning-terminal-*
 
 # MacOS junk
-.DS_Store
+.DS_Store
+
+itest/btcd-itest
+itest/litd-itest
+itest/itest.test
+itest/.logs
+itest/*.log

Makefile

@@ -3,6 +3,7 @@ ESCPKG := github.com\/lightninglabs\/lightning-terminal
 LND_PKG := github.com/lightningnetwork/lnd
 LOOP_PKG := github.com/lightninglabs/loop
 POOL_PKG := github.com/lightninglabs/pool
+BTCD_PKG := github.com/btcsuite/btcd
 
 LINT_PKG := github.com/golangci/golangci-lint/cmd/golangci-lint
 GOVERALLS_PKG := github.com/mattn/goveralls
@@ -68,6 +69,9 @@ LDFLAGS := $(call make_ldflags, $(LND_RELEASE_TAGS))
 # and omit the DWARF symbol table (-w). Also we clear the build ID.
 RELEASE_LDFLAGS := $(call make_ldflags, $(LND_RELEASE_TAGS), -s -w -buildid=)
 
+ITEST_TAGS := rpctest itest $(LND_RELEASE_TAGS)
+ITEST_LDFLAGS := $(call make_ldflags, $(ITEST_TAGS))
+
 GREEN := "\\033[0;32m"
 NC := "\\033[0m"
 define print
@@ -160,6 +164,20 @@ travis-cover: lint unit-cover goveralls
 
 travis-itest: lint
 
+build-itest: app-build
+	@$(call print, "Building itest btcd and litd.")
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o itest/btcd-itest -ldflags "$(ITEST_LDFLAGS)" $(BTCD_PKG)
+
+itest-only:
+	@$(call print, "Building itest binary.")
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o itest/litd-itest -ldflags "$(ITEST_LDFLAGS)" $(PKG)/cmd/litd
+	CGO_ENABLED=0 $(GOTEST) -v ./itest -tags="$(DEV_TAGS) $(ITEST_TAGS)" -c -o itest/itest.test
+
+	@$(call print, "Running integration tests.")
+	rm -rf itest/*.log itest/.logs*; date
+	scripts/itest_part.sh $(ITEST_FLAGS)
+
+itest: build-itest itest-only
 
 # =============
 # FLAKE HUNTING

go.mod

@@ -5,6 +5,7 @@ require (
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
 	github.com/btcsuite/btcutil v1.0.3-0.20210527170813-e2ba6805a890
 	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
+	github.com/go-errors/errors v1.0.1
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.5.0
 	github.com/improbable-eng/grpc-web v0.12.0
 	github.com/jessevdk/go-flags v1.4.0
@@ -23,6 +24,7 @@ require (
 	github.com/urfave/cli v1.20.0
 	go.etcd.io/bbolt v1.3.6
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1
 	google.golang.org/grpc v1.39.0
 	google.golang.org/protobuf v1.27.1

go.sum

@@ -426,6 +426,7 @@ github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570=
 github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=

itest/litd_test.go

@@ -0,0 +1,166 @@
+package itest
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcd/integration/rpctest"
+	"github.com/lightningnetwork/lnd/lntest"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLightningTerminal performs a series of integration tests amongst a
+// programmatically driven network of lnd nodes.
+func TestLightningTerminal(t *testing.T) {
+	// If no tests are registered, then we can exit early.
+	if len(allTestCases) == 0 {
+		t.Skip("integration tests not selected with flag 'itest'")
+	}
+
+	// Parse testing flags that influence our test execution.
+	logDir := "."
+	require.NoError(t, os.MkdirAll(logDir, 0700))
+
+	// Before we start any node, we need to make sure that any btcd node
+	// that is started through the RPC harness uses a unique port as well to
+	// avoid any port collisions.
+	rpctest.ListenAddressGenerator = lntest.GenerateBtcdListenerAddresses
+
+	// Declare the network harness here to gain access to its
+	// 'OnTxAccepted' call back.
+	var litdHarness *NetworkHarness
+
+	// Create an instance of the btcd's rpctest.Harness that will act as
+	// the miner for all tests. This will be used to fund the wallets of
+	// the nodes within the test network and to drive blockchain related
+	// events within the network. Revert the default setting of accepting
+	// non-standard transactions on simnet to reject them. Transactions on
+	// the lightning network should always be standard to get better
+	// guarantees of getting included in to blocks.
+	//
+	// We will also connect it to our chain backend.
+	miner, err := lntest.NewMiner()
+	require.NoError(t, err, "failed to create new miner")
+	defer func() {
+		require.NoError(t, miner.Stop(), "failed to stop miner")
+	}()
+
+	// Start a chain backend.
+	chainBackend, cleanUp, err := lntest.NewBackend(
+		miner.P2PAddress(), harnessNetParams,
+	)
+	require.NoError(t, err, "new backend")
+	defer func() {
+		require.NoError(t, cleanUp(), "cleanup")
+	}()
+
+	// Before we start anything, we want to overwrite some of the connection
+	// settings to make the tests more robust. We might need to restart the
+	// miner while there are already blocks present, which will take a bit
+	// longer than the 1 second the default settings amount to. Doubling
+	// both values will give us retries up to 4 seconds.
+	miner.MaxConnRetries = rpctest.DefaultMaxConnectionRetries * 2
+	miner.ConnectionRetryTimeout = rpctest.DefaultConnectionRetryTimeout * 2
+
+	// Set up miner and connect chain backend to it.
+	require.NoError(t, miner.SetUp(true, 50))
+	require.NoError(t, miner.Client.NotifyNewTransactions(false))
+	require.NoError(t, chainBackend.ConnectMiner(), "connect miner")
+
+	// Now we can set up our test harness (LND instance), with the chain
+	// backend we just created.
+	ht := newHarnessTest(t, nil)
+	binary := ht.getLitdBinary()
+	litdHarness, err = NewNetworkHarness(miner, chainBackend, binary)
+	if err != nil {
+		ht.Fatalf("unable to create lightning network harness: %v", err)
+	}
+	defer litdHarness.Stop()
+
+	// Spawn a new goroutine to watch for any fatal errors that any of the
+	// running lnd processes encounter. If an error occurs, then the test
+	// case should naturally as a result and we log the server error here to
+	// help debug.
+	go func() {
+		for {
+			select {
+			case err, more := <-litdHarness.ProcessErrors():
+				if !more {
+					return
+				}
+				ht.Logf("litd finished with error (stderr):\n%v",
+					err)
+			}
+		}
+	}()
+
+	// Next mine enough blocks in order for segwit and the CSV package
+	// soft-fork to activate on SimNet.
+	numBlocks := harnessNetParams.MinerConfirmationWindow * 2
+	if _, err := miner.Client.Generate(numBlocks); err != nil {
+		ht.Fatalf("unable to generate blocks: %v", err)
+	}
+
+	// With the btcd harness created, we can now complete the
+	// initialization of the network. args - list of lnd arguments,
+	// example: "--debuglevel=debug"
+	// TODO(roasbeef): create master balanced channel with all the monies?
+	aliceBobArgs := []string{
+		"--default-remote-max-htlcs=483",
+		"--dust-threshold=5000000",
+	}
+
+	// Run the subset of the test cases selected in this tranche.
+	for _, testCase := range allTestCases {
+		testCase := testCase
+		name := fmt.Sprintf("%s/%s", chainBackend.Name(), testCase.name)
+
+		success := t.Run(name, func(t1 *testing.T) {
+			cleanTestCaseName := strings.ReplaceAll(
+				testCase.name, " ", "_",
+			)
+
+			err = litdHarness.SetUp(
+				t1, cleanTestCaseName, aliceBobArgs,
+			)
+			require.NoError(t1,
+				err, "unable to set up test lightning network",
+			)
+			defer func() {
+				require.NoError(t1, litdHarness.TearDown())
+			}()
+
+			litdHarness.EnsureConnected(
+				t1, litdHarness.Alice, litdHarness.Bob,
+			)
+
+			logLine := fmt.Sprintf(
+				"STARTING ============ %v ============\n",
+				testCase.name,
+			)
+
+			litdHarness.Alice.AddToLog(logLine)
+			litdHarness.Bob.AddToLog(logLine)
+
+			// Create a separate harness test for the testcase to
+			// avoid overwriting the external harness test that is
+			// tied to the parent test.
+			ht := newHarnessTest(t1, litdHarness)
+			ht.RunTestCase(testCase)
+		})
+
+		// Stop at the first failure. Mimic behavior of original test
+		// framework.
+		if !success {
+			// Log failure time to help relate the lnd logs to the
+			// failure.
+			t.Logf("Failure time: %v", time.Now().Format(
+				"2006-01-02 15:04:05.000",
+			))
+			break
+		}
+	}
+}

itest/litd_test_list_off_test.go

@@ -0,0 +1,6 @@
+//go:build !itest
+// +build !itest
+
+package itest
+
+var allTestCases = []*testCase{}

itest/litd_test_list_on_test.go

@@ -0,0 +1,8 @@
+//go:build itest
+// +build itest
+
+package itest
+
+var allTestCases = []*testCase{
+	// TODO(guggero): Add tests.
+}

itest/test_harness.go

@@ -0,0 +1,224 @@
+package itest
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
+	"github.com/btcsuite/btcd/rpcclient"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/go-errors/errors"
+	"github.com/lightningnetwork/lnd/lntest"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	harnessNetParams = &chaincfg.RegressionNetParams
+
+	// litdExecutable is the full path to the litd binary.
+	litdExecutable = flag.String(
+		"litdexec", itestLitdBinary, "full path to litd binary",
+	)
+
+	slowMineDelay = 20 * time.Millisecond
+)
+
+const (
+	defaultTimeout      = lntest.DefaultTimeout
+	minerMempoolTimeout = lntest.MinerMempoolTimeout
+	itestLitdBinary     = "litd-itest"
+)
+
+// harnessTest wraps a regular testing.T providing enhanced error detection
+// and propagation. All error will be augmented with a full stack-trace in
+// order to aid in debugging. Additionally, any panics caused by active
+// test cases will also be handled and represented as fatals.
+type harnessTest struct {
+	t *testing.T
+
+	// testCase is populated during test execution and represents the
+	// current test case.
+	testCase *testCase
+
+	// lndHarness is a reference to the current network harness. Will be
+	// nil if not yet set up.
+	lndHarness *NetworkHarness
+}
+
+// newHarnessTest creates a new instance of a harnessTest from a regular
+// testing.T instance.
+func newHarnessTest(t *testing.T, net *NetworkHarness) *harnessTest {
+	return &harnessTest{t, nil, net}
+}
+
+// Skipf calls the underlying testing.T's Skip method, causing the current test
+// to be skipped.
+func (h *harnessTest) Skipf(format string, args ...interface{}) {
+	h.t.Skipf(format, args...)
+}
+
+// Fatalf causes the current active test case to fail with a fatal error. All
+// integration tests should mark test failures solely with this method due to
+// the error stack traces it produces.
+func (h *harnessTest) Fatalf(format string, a ...interface{}) {
+	stacktrace := errors.Wrap(fmt.Sprintf(format, a...), 1).ErrorStack()
+
+	if h.testCase != nil {
+		h.t.Fatalf("Failed: (%v): exited with error: \n"+
+			"%v", h.testCase.name, stacktrace)
+	} else {
+		h.t.Fatalf("Error outside of test: %v", stacktrace)
+	}
+}
+
+// RunTestCase executes a harness test case. Any errors or panics will be
+// represented as fatal.
+func (h *harnessTest) RunTestCase(testCase *testCase) {
+
+	h.testCase = testCase
+	defer func() {
+		h.testCase = nil
+	}()
+
+	defer func() {
+		if err := recover(); err != nil {
+			description := errors.Wrap(err, 2).ErrorStack()
+			h.t.Fatalf("Failed: (%v) panicked with: \n%v",
+				h.testCase.name, description)
+		}
+	}()
+
+	testCase.test(h.lndHarness, h)
+}
+
+func (h *harnessTest) Logf(format string, args ...interface{}) {
+	h.t.Logf(format, args...)
+}
+
+func (h *harnessTest) Log(args ...interface{}) {
+	h.t.Log(args...)
+}
+
+func (h *harnessTest) getLitdBinary() string {
+	binary := itestLitdBinary
+	litdExec := ""
+	if litdExecutable != nil && *litdExecutable != "" {
+		litdExec = *litdExecutable
+	}
+	if litdExec != "" {
+		binary = litdExec
+	}
+
+	return binary
+}
+
+type testCase struct {
+	name string
+	test func(net *NetworkHarness, t *harnessTest)
+}
+
+// waitForNTxsInMempool polls until finding the desired number of transactions
+// in the provided miner's mempool. An error is returned if this number is not
+// met after the given timeout.
+func waitForNTxsInMempool(miner *rpcclient.Client, n int,
+	timeout time.Duration) ([]*chainhash.Hash, error) {
+
+	breakTimeout := time.After(timeout)
+	ticker := time.NewTicker(50 * time.Millisecond)
+	defer ticker.Stop()
+
+	var err error
+	var mempool []*chainhash.Hash
+	for {
+		select {
+		case <-breakTimeout:
+			return nil, fmt.Errorf("wanted %v, found %v txs "+
+				"in mempool: %v", n, len(mempool), mempool)
+		case <-ticker.C:
+			mempool, err = miner.GetRawMempool()
+			if err != nil {
+				return nil, err
+			}
+
+			if len(mempool) == n {
+				return mempool, nil
+			}
+		}
+	}
+}
+
+// mineBlocksSlow mines 'num' of blocks and checks that blocks are present in
+// the mining node's blockchain. numTxs should be set to the number of
+// transactions (excluding the coinbase) we expect to be included in the first
+// mined block. Between each mined block an artificial delay is introduced to
+// give all network participants time to catch up.
+//
+// NOTE: This function currently is just an alias for mineBlocksSlow.
+func mineBlocks(t *harnessTest, net *lntest.NetworkHarness,
+	num uint32, numTxs int) []*wire.MsgBlock {
+
+	return mineBlocksSlow(t, net, num, numTxs)
+}
+
+// mineBlocksSlow mines 'num' of blocks and checks that blocks are present in
+// the mining node's blockchain. numTxs should be set to the number of
+// transactions (excluding the coinbase) we expect to be included in the first
+// mined block. Between each mined block an artificial delay is introduced to
+// give all network participants time to catch up.
+func mineBlocksSlow(t *harnessTest, net *lntest.NetworkHarness,
+	num uint32, numTxs int) []*wire.MsgBlock {
+
+	t.t.Helper()
+
+	// If we expect transactions to be included in the blocks we'll mine,
+	// we wait here until they are seen in the miner's mempool.
+	var txids []*chainhash.Hash
+	var err error
+	if numTxs > 0 {
+		txids, err = waitForNTxsInMempool(
+			net.Miner.Client, numTxs, minerMempoolTimeout,
+		)
+		require.NoError(t.t, err, "unable to find txns in mempool")
+	}
+
+	blocks := make([]*wire.MsgBlock, num)
+	blockHashes := make([]*chainhash.Hash, 0, num)
+
+	for i := uint32(0); i < num; i++ {
+		generatedHashes, err := net.Miner.Client.Generate(1)
+		require.NoError(t.t, err, "generate blocks")
+		blockHashes = append(blockHashes, generatedHashes...)
+
+		time.Sleep(slowMineDelay)
+	}
+
+	for i, blockHash := range blockHashes {
+		block, err := net.Miner.Client.GetBlock(blockHash)
+		require.NoError(t.t, err, "get blocks")
+
+		blocks[i] = block
+	}
+
+	// Finally, assert that all the transactions were included in the first
+	// block.
+	for _, txid := range txids {
+		assertTxInBlock(t, blocks[0], txid)
+	}
+
+	return blocks
+}
+
+func assertTxInBlock(t *harnessTest, block *wire.MsgBlock, txid *chainhash.Hash) {
+	for _, tx := range block.Transactions {
+		sha := tx.TxHash()
+		if bytes.Equal(txid[:], sha[:]) {
+			return
+		}
+	}
+
+	t.Fatalf("tx was not included in block")
+}

scripts/itest_part.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Let's work with absolute paths only, we run in the itest directory itself.
+WORKDIR=$(pwd)/itest
+
+# Windows insists on having the .exe suffix for an executable, we need to add
+# that here if necessary.
+EXEC="$WORKDIR"/itest.test
+LITD_EXEC="$WORKDIR"/litd-itest
+BTCD_EXEC="$WORKDIR"/btcd-itest
+echo $EXEC -test.v "$@" -logoutput -goroutinedump -logdir=.logs -litdexec=$LITD_EXEC -btcdexec=$BTCD_EXEC
+
+# Exit code 255 causes the parallel jobs to abort, so if one part fails the
+# other is aborted too.
+cd "$WORKDIR" || exit 255
+$EXEC -test.v "$@" -logoutput -goroutinedump -logdir=.logs -litdexec=$LITD_EXEC -btcdexec=$BTCD_EXEC || exit 255