new e2e framework (#1835)

Makefile

@@ -37,7 +37,10 @@ gotest:
 ci:
 	go test -count=1 -p=1 -v ./tests/...
 
-alltest: gotest ci
+e2e:
+	./hack/run-e2e.sh
+
+alltest: gotest ci e2e
 	
 clean:
 	rm -f ./bin/frpc

cmd/frpc/sub/root.go

@@ -212,7 +212,13 @@ func runClient(cfgFilePath string) (err error) {
 	return
 }
 
-func startService(cfg config.ClientCommonConf, pxyCfgs map[string]config.ProxyConf, visitorCfgs map[string]config.VisitorConf, cfgFile string) (err error) {
+func startService(
+	cfg config.ClientCommonConf,
+	pxyCfgs map[string]config.ProxyConf,
+	visitorCfgs map[string]config.VisitorConf,
+	cfgFile string,
+) (err error) {
+
 	log.InitLog(cfg.LogWay, cfg.LogFile, cfg.LogLevel,
 		cfg.LogMaxDays, cfg.DisableLogColor)
 

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/fatedier/golib v0.0.0-20181107124048-ff8cd814b049
 	github.com/fatedier/kcp-go v2.0.4-0.20190803094908-fe8645b0a904+incompatible
 	github.com/golang/snappy v0.0.0-20170215233205-553a64147049 // indirect
+	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.0
 	github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d
@@ -16,22 +17,23 @@ require (
 	github.com/klauspost/cpuid v1.2.0 // indirect
 	github.com/klauspost/reedsolomon v1.9.1 // indirect
 	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/onsi/ginkgo v1.12.2
+	github.com/onsi/gomega v1.10.1
 	github.com/pires/go-proxyproto v0.0.0-20190111085350-4d51b51e3bfc
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/prometheus/client_golang v1.4.1
 	github.com/rakyll/statik v0.1.1
 	github.com/rodaine/table v1.0.0
 	github.com/spf13/cobra v0.0.3
-	github.com/spf13/pflag v1.0.1 // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/templexxx/cpufeat v0.0.0-20170927014610-3794dfbfb047 // indirect
 	github.com/templexxx/xor v0.0.0-20170926022130-0af8e873c554 // indirect
 	github.com/tjfoc/gmsm v0.0.0-20171124023159-98aa888b79d8 // indirect
 	github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec
 	github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae // indirect
-	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
+	golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/text v0.3.2 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	gopkg.in/square/go-jose.v2 v2.4.1 // indirect
+	k8s.io/apimachinery v0.18.3
 )

go.sum

@@ -1,4 +1,7 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -16,39 +19,73 @@ github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHo
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatedier/beego v0.0.0-20171024143340-6c6a4f5bd5eb h1:wCrNShQidLmvVWn/0PikGmpdP0vtQmnvyRg3ZBEhczw=
 github.com/fatedier/beego v0.0.0-20171024143340-6c6a4f5bd5eb/go.mod h1:wx3gB6dbIfBRcucp94PI9Bt3I0F2c/MyNEWuhzpWiwk=
 github.com/fatedier/golib v0.0.0-20181107124048-ff8cd814b049 h1:teH578mf2ii42NHhIp3PhgvjU5bv+NFMq9fSQR8NaG8=
 github.com/fatedier/golib v0.0.0-20181107124048-ff8cd814b049/go.mod h1:DqIrnl0rp3Zybg9zbJmozTy1n8fYJoX+QoAj9slIkKM=
 github.com/fatedier/kcp-go v2.0.4-0.20190803094908-fe8645b0a904+incompatible h1:ssXat9YXFvigNge/IkkZvFMn8yeYKFX+uI6wn2mLJ74=
 github.com/fatedier/kcp-go v2.0.4-0.20190803094908-fe8645b0a904+incompatible/go.mod h1:YpCOaxj7vvMThhIQ9AfTOPW2sfztQR5WDfs7AflSy4s=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049 h1:K9KHZbXKpGydfDN0aZrsoHpLJlZsBrGMFWbgLDGnPZk=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/reedsolomon v1.9.1 h1:kYrT1MlR4JH6PqOpC+okdb9CDTcwEC/BqpzK4WFyXL8=
@@ -60,6 +97,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@@ -68,7 +106,22 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.12.2 h1:Ke9m3h2Hu0wsZ45yewCqhYr3Z+emcNTuLY2nMWCkrSI=
+github.com/onsi/ginkgo v1.12.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/pires/go-proxyproto v0.0.0-20190111085350-4d51b51e3bfc h1:lNOt1SMsgHXTdpuGw+RpnJtzUcCb/oRKZP65pBy9pr8=
 github.com/pires/go-proxyproto v0.0.0-20190111085350-4d51b51e3bfc/go.mod h1:6/gX3+E/IYGa0wMORlSMla999awQFdbaeQCHjSMKIzY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -101,8 +154,9 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -122,41 +176,82 @@ github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/square/go-jose.v2 v2.4.1 h1:H0TmLt7/KmzlrDOpa1F+zr0Tk90PbJYBfsVUmRLrf9Y=
 gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+k8s.io/apimachinery v0.18.3 h1:pOGcbVAhxADgUYnjS08EFXs9QMl8qaH5U4fr5LGUrSk=
+k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
+k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
+sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

hack/run-e2e.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
+
+which ginkgo &> /dev/null
+if [ $? -ne 0 ]; then
+    echo "ginkgo not found, try to install..."
+    go get -u github.com/onsi/ginkgo/ginkgo
+fi
+
+ginkgo -nodes=1 ${ROOT}/test/e2e -- -frpc-path=${ROOT}/bin/frpc -frps-path=${ROOT}/bin/frps -log-level=debug

test/e2e/e2e.go

@@ -0,0 +1,61 @@
+package e2e
+
+import (
+	"testing"
+
+	"github.com/fatedier/frp/test/e2e/framework"
+	"github.com/fatedier/frp/utils/log"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+	"github.com/onsi/gomega"
+)
+
+var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
+	setupSuite()
+	return nil
+}, func(data []byte) {
+	// Run on all Ginkgo nodes
+	setupSuitePerGinkgoNode()
+})
+
+var _ = ginkgo.SynchronizedAfterSuite(func() {
+	CleanupSuite()
+}, func() {
+	AfterSuiteActions()
+})
+
+// RunE2ETests checks configuration parameters (specified through flags) and then runs
+// E2E tests using the Ginkgo runner.
+// If a "report directory" is specified, one or more JUnit test reports will be
+// generated in this directory, and cluster logs will also be saved.
+// This function is called on each Ginkgo node in parallel mode.
+func RunE2ETests(t *testing.T) {
+	gomega.RegisterFailHandler(framework.Fail)
+
+	log.Info("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
+	ginkgo.RunSpecs(t, "frp e2e suite")
+}
+
+// setupSuite is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step.
+// There are certain operations we only want to run once per overall test invocation
+// (such as deleting old namespaces, or verifying that all system pods are running.
+// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
+// to ensure that these operations only run on the first parallel Ginkgo node.
+//
+// This function takes two parameters: one function which runs on only the first Ginkgo node,
+// returning an opaque byte array, and then a second function which runs on all Ginkgo nodes,
+// accepting the byte array.
+func setupSuite() {
+	// Run only on Ginkgo node 1
+	// TODO
+}
+
+// setupSuitePerGinkgoNode is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step.
+// There are certain operations we only want to run once per overall test invocation on each Ginkgo node
+// such as making some global variables accessible to all parallel executions
+// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
+// Ref: https://onsi.github.io/ginkgo/#parallel-specs
+func setupSuitePerGinkgoNode() {
+	// config.GinkgoConfig.ParallelNode
+}

test/e2e/e2e_test.go

@@ -0,0 +1,34 @@
+package e2e
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/fatedier/frp/test/e2e/framework"
+	"github.com/fatedier/frp/utils/log"
+)
+
+// handleFlags sets up all flags and parses the command line.
+func handleFlags() {
+	framework.RegisterCommonFlags(flag.CommandLine)
+	flag.Parse()
+}
+
+func TestMain(m *testing.M) {
+	// Register test flags, then parse flags.
+	handleFlags()
+
+	if err := framework.ValidateTestContext(&framework.TestContext); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	log.InitLog("console", "", framework.TestContext.LogLevel, 0, true)
+	os.Exit(m.Run())
+}
+
+func TestE2E(t *testing.T) {
+	RunE2ETests(t)
+}

test/e2e/examples.go

@@ -0,0 +1,40 @@
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/fatedier/frp/test/e2e/framework"
+	"github.com/fatedier/frp/test/e2e/framework/consts"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var connTimeout = 5 * time.Second
+
+var _ = Describe("[Feature: Example]", func() {
+	f := framework.NewDefaultFramework()
+
+	Describe("TCP", func() {
+		It("Expose a TCP echo server", func() {
+			serverConf := `
+			[common]
+			bind_port = {{ .PortServer }}
+			`
+
+			clientConf := fmt.Sprintf(`
+			[common]
+			server_port = {{ .PortServer }}
+
+			[tcp]
+			type = tcp
+			local_port = {{ .%s }}
+			remote_port = {{ .PortTCP }}
+			`, framework.TCPEchoServerPort)
+
+			f.RunProcesses([]string{serverConf}, []string{clientConf})
+
+			framework.ExpectTCPReuqest(f.UsedPorts["PortTCP"], []byte(consts.TestString), []byte(consts.TestString), connTimeout)
+		})
+	})
+})

test/e2e/framework/cleanup.go

@@ -0,0 +1,59 @@
+package framework
+
+import (
+	"sync"
+)
+
+// CleanupActionHandle is an integer pointer type for handling cleanup action
+type CleanupActionHandle *int
+type cleanupFuncHandle struct {
+	actionHandle CleanupActionHandle
+	actionHook   func()
+}
+
+var cleanupActionsLock sync.Mutex
+var cleanupHookList = []cleanupFuncHandle{}
+
+// AddCleanupAction installs a function that will be called in the event of the
+// whole test being terminated.  This allows arbitrary pieces of the overall
+// test to hook into SynchronizedAfterSuite().
+// The hooks are called in last-in-first-out order.
+func AddCleanupAction(fn func()) CleanupActionHandle {
+	p := CleanupActionHandle(new(int))
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	c := cleanupFuncHandle{actionHandle: p, actionHook: fn}
+	cleanupHookList = append([]cleanupFuncHandle{c}, cleanupHookList...)
+	return p
+}
+
+// RemoveCleanupAction removes a function that was installed by
+// AddCleanupAction.
+func RemoveCleanupAction(p CleanupActionHandle) {
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	for i, item := range cleanupHookList {
+		if item.actionHandle == p {
+			cleanupHookList = append(cleanupHookList[:i], cleanupHookList[i+1:]...)
+			break
+		}
+	}
+}
+
+// RunCleanupActions runs all functions installed by AddCleanupAction.  It does
+// not remove them (see RemoveCleanupAction) but it does run unlocked, so they
+// may remove themselves.
+func RunCleanupActions() {
+	list := []func(){}
+	func() {
+		cleanupActionsLock.Lock()
+		defer cleanupActionsLock.Unlock()
+		for _, p := range cleanupHookList {
+			list = append(list, p.actionHook)
+		}
+	}()
+	// Run unlocked.
+	for _, fn := range list {
+		fn()
+	}
+}

test/e2e/framework/client.go

@@ -0,0 +1,11 @@
+package framework
+
+type FRPClient struct {
+	port int
+}
+
+func (f *Framework) FRPClient(port int) *FRPClient {
+	return &FRPClient{
+		port: port,
+	}
+}

test/e2e/framework/consts/consts.go

@@ -0,0 +1,5 @@
+package consts
+
+const (
+	TestString = "frp is a fast reverse proxy to help you expose a local server behind a NAT or firewall to the internet."
+)

test/e2e/framework/expect.go

@@ -0,0 +1,55 @@
+package framework
+
+import (
+	"github.com/onsi/gomega"
+)
+
+// ExpectEqual expects the specified two are the same, otherwise an exception raises
+func ExpectEqual(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.Equal(extra), explain...)
+}
+
+// ExpectEqualValues expects the specified two are the same, it not strict about type
+func ExpectEqualValues(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.BeEquivalentTo(extra), explain...)
+}
+
+// ExpectNotEqual expects the specified two are not the same, otherwise an exception raises
+func ExpectNotEqual(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).NotTo(gomega.Equal(extra), explain...)
+}
+
+// ExpectError expects an error happens, otherwise an exception raises
+func ExpectError(err error, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, err).To(gomega.HaveOccurred(), explain...)
+}
+
+// ExpectNoError checks if "err" is set, and if so, fails assertion while logging the error.
+func ExpectNoError(err error, explain ...interface{}) {
+	ExpectNoErrorWithOffset(1, err, explain...)
+}
+
+// ExpectNoErrorWithOffset checks if "err" is set, and if so, fails assertion while logging the error at "offset" levels above its caller
+// (for example, for call chain f -> g -> ExpectNoErrorWithOffset(1, ...) error would be logged for "f").
+func ExpectNoErrorWithOffset(offset int, err error, explain ...interface{}) {
+	gomega.ExpectWithOffset(1+offset, err).NotTo(gomega.HaveOccurred(), explain...)
+}
+
+// ExpectConsistOf expects actual contains precisely the extra elements.  The ordering of the elements does not matter.
+func ExpectConsistOf(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.ConsistOf(extra), explain...)
+}
+
+// ExpectHaveKey expects the actual map has the key in the keyset
+func ExpectHaveKey(actual interface{}, key interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.HaveKey(key), explain...)
+}
+
+// ExpectEmpty expects actual is empty
+func ExpectEmpty(actual interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.BeEmpty(), explain...)
+}
+
+func ExpectTrue(actual interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).Should(gomega.BeTrue(), explain...)
+}

test/e2e/framework/framework.go

@@ -0,0 +1,167 @@
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strings"
+	"text/template"
+
+	"github.com/fatedier/frp/test/e2e/pkg/port"
+	"github.com/fatedier/frp/test/e2e/pkg/process"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+)
+
+type Options struct {
+	TotalParallelNode int
+	CurrentNodeIndex  int
+	FromPortIndex     int
+	ToPortIndex       int
+}
+
+type Framework struct {
+	TempDirectory string
+	UsedPorts     map[string]int
+
+	portAllocator *port.Allocator
+
+	// Multiple mock servers used for e2e testing.
+	mockServers *MockServers
+
+	// To make sure that this framework cleans up after itself, no matter what,
+	// we install a Cleanup action before each test and clear it after.  If we
+	// should abort, the AfterSuite hook should run all Cleanup actions.
+	cleanupHandle CleanupActionHandle
+
+	// beforeEachStarted indicates that BeforeEach has started
+	beforeEachStarted bool
+
+	serverConfPaths []string
+	serverProcesses []*process.Process
+	clientConfPaths []string
+	clientProcesses []*process.Process
+}
+
+func NewDefaultFramework() *Framework {
+	options := Options{
+		TotalParallelNode: config.GinkgoConfig.ParallelTotal,
+		CurrentNodeIndex:  config.GinkgoConfig.ParallelNode,
+		FromPortIndex:     20000,
+		ToPortIndex:       50000,
+	}
+	return NewFramework(options)
+}
+
+func NewFramework(opt Options) *Framework {
+	f := &Framework{
+		portAllocator: port.NewAllocator(opt.FromPortIndex, opt.ToPortIndex, opt.TotalParallelNode, opt.CurrentNodeIndex-1),
+	}
+	f.mockServers = NewMockServers(f.portAllocator)
+	if err := f.mockServers.Run(); err != nil {
+		Failf("%v", err)
+	}
+
+	ginkgo.BeforeEach(f.BeforeEach)
+	ginkgo.AfterEach(f.AfterEach)
+	return f
+}
+
+// BeforeEach create a temp directory.
+func (f *Framework) BeforeEach() {
+	f.beforeEachStarted = true
+
+	f.cleanupHandle = AddCleanupAction(f.AfterEach)
+
+	dir, err := ioutil.TempDir(os.TempDir(), "frpe2e-test-*")
+	ExpectNoError(err)
+	f.TempDirectory = dir
+}
+
+func (f *Framework) AfterEach() {
+	if !f.beforeEachStarted {
+		return
+	}
+
+	RemoveCleanupAction(f.cleanupHandle)
+
+	os.RemoveAll(f.TempDirectory)
+	f.TempDirectory = ""
+	f.UsedPorts = nil
+	f.serverConfPaths = nil
+	f.clientConfPaths = nil
+	for _, p := range f.serverProcesses {
+		p.Stop()
+	}
+	for _, p := range f.clientProcesses {
+		p.Stop()
+	}
+	f.serverProcesses = nil
+	f.clientProcesses = nil
+}
+
+var portRegex = regexp.MustCompile(`{{ \.Port.*? }}`)
+
+// RenderPortsTemplate render templates with ports.
+//
+// Local: {{ .Port1 }}
+// Target: {{ .Port2 }}
+//
+// return rendered content and all allocated ports.
+func (f *Framework) genPortsFromTemplates(templates []string) (ports map[string]int, err error) {
+	ports = make(map[string]int)
+	for _, t := range templates {
+		arrs := portRegex.FindAllString(t, -1)
+		for _, str := range arrs {
+			str = strings.TrimPrefix(str, "{{ .")
+			str = strings.TrimSuffix(str, " }}")
+			str = strings.TrimSpace(str)
+			ports[str] = 0
+		}
+	}
+	defer func() {
+		if err != nil {
+			for _, port := range ports {
+				f.portAllocator.Release(port)
+			}
+		}
+	}()
+
+	for name := range ports {
+		port := f.portAllocator.Get()
+		if port <= 0 {
+			return nil, fmt.Errorf("can't allocate port")
+		}
+		ports[name] = port
+	}
+	return
+
+}
+
+func (f *Framework) RenderTemplates(templates []string) (outs []string, ports map[string]int, err error) {
+	ports, err = f.genPortsFromTemplates(templates)
+	if err != nil {
+		return
+	}
+
+	params := f.mockServers.GetTemplateParams()
+	for name, port := range ports {
+		params[name] = port
+	}
+
+	for _, t := range templates {
+		tmpl, err := template.New("").Parse(t)
+		if err != nil {
+			return nil, nil, err
+		}
+		buffer := bytes.NewBuffer(nil)
+		if err = tmpl.Execute(buffer, params); err != nil {
+			return nil, nil, err
+		}
+		outs = append(outs, buffer.String())
+	}
+	return
+}

test/e2e/framework/ginkgowrapper/wrapper.go

@@ -0,0 +1,80 @@
+// Package ginkgowrapper wraps Ginkgo Fail and Skip functions to panic
+// with structured data instead of a constant string.
+package ginkgowrapper
+
+import (
+	"bufio"
+	"bytes"
+	"regexp"
+	"runtime"
+	"runtime/debug"
+	"strings"
+
+	"github.com/onsi/ginkgo"
+)
+
+// FailurePanic is the value that will be panicked from Fail.
+type FailurePanic struct {
+	Message        string // The failure message passed to Fail
+	Filename       string // The filename that is the source of the failure
+	Line           int    // The line number of the filename that is the source of the failure
+	FullStackTrace string // A full stack trace starting at the source of the failure
+}
+
+// String makes FailurePanic look like the old Ginkgo panic when printed.
+func (FailurePanic) String() string { return ginkgo.GINKGO_PANIC }
+
+// Fail wraps ginkgo.Fail so that it panics with more useful
+// information about the failure. This function will panic with a
+// FailurePanic.
+func Fail(message string, callerSkip ...int) {
+	skip := 1
+	if len(callerSkip) > 0 {
+		skip += callerSkip[0]
+	}
+
+	_, file, line, _ := runtime.Caller(skip)
+	fp := FailurePanic{
+		Message:        message,
+		Filename:       file,
+		Line:           line,
+		FullStackTrace: pruneStack(skip),
+	}
+
+	defer func() {
+		e := recover()
+		if e != nil {
+			panic(fp)
+		}
+	}()
+
+	ginkgo.Fail(message, skip)
+}
+
+// ginkgo adds a lot of test running infrastructure to the stack, so
+// we filter those out
+var stackSkipPattern = regexp.MustCompile(`onsi/ginkgo`)
+
+func pruneStack(skip int) string {
+	skip += 2 // one for pruneStack and one for debug.Stack
+	stack := debug.Stack()
+	scanner := bufio.NewScanner(bytes.NewBuffer(stack))
+	var prunedStack []string
+
+	// skip the top of the stack
+	for i := 0; i < 2*skip+1; i++ {
+		scanner.Scan()
+	}
+
+	for scanner.Scan() {
+		if stackSkipPattern.Match(scanner.Bytes()) {
+			scanner.Scan() // these come in pairs
+		} else {
+			prunedStack = append(prunedStack, scanner.Text())
+			scanner.Scan() // these come in pairs
+			prunedStack = append(prunedStack, scanner.Text())
+		}
+	}
+
+	return strings.Join(prunedStack, "\n")
+}

test/e2e/framework/log.go

@@ -0,0 +1,94 @@
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"regexp"
+	"runtime/debug"
+	"time"
+
+	"github.com/onsi/ginkgo"
+
+	e2eginkgowrapper "github.com/fatedier/frp/test/e2e/framework/ginkgowrapper"
+)
+
+func nowStamp() string {
+	return time.Now().Format(time.StampMilli)
+}
+
+func log(level string, format string, args ...interface{}) {
+	fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...)
+}
+
+// Logf logs the info.
+func Logf(format string, args ...interface{}) {
+	log("INFO", format, args...)
+}
+
+// Failf logs the fail info, including a stack trace.
+func Failf(format string, args ...interface{}) {
+	FailfWithOffset(1, format, args...)
+}
+
+// FailfWithOffset calls "Fail" and logs the error with a stack trace that starts at "offset" levels above its caller
+// (for example, for call chain f -> g -> FailfWithOffset(1, ...) error would be logged for "f").
+func FailfWithOffset(offset int, format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	skip := offset + 1
+	log("FAIL", "%s\n\nFull Stack Trace\n%s", msg, PrunedStack(skip))
+	e2eginkgowrapper.Fail(nowStamp()+": "+msg, skip)
+}
+
+// Fail is a replacement for ginkgo.Fail which logs the problem as it occurs
+// together with a stack trace and then calls ginkgowrapper.Fail.
+func Fail(msg string, callerSkip ...int) {
+	skip := 1
+	if len(callerSkip) > 0 {
+		skip += callerSkip[0]
+	}
+	log("FAIL", "%s\n\nFull Stack Trace\n%s", msg, PrunedStack(skip))
+	e2eginkgowrapper.Fail(nowStamp()+": "+msg, skip)
+}
+
+var codeFilterRE = regexp.MustCompile(`/github.com/onsi/ginkgo/`)
+
+// PrunedStack is a wrapper around debug.Stack() that removes information
+// about the current goroutine and optionally skips some of the initial stack entries.
+// With skip == 0, the returned stack will start with the caller of PruneStack.
+// From the remaining entries it automatically filters out useless ones like
+// entries coming from Ginkgo.
+//
+// This is a modified copy of PruneStack in https://github.com/onsi/ginkgo/blob/f90f37d87fa6b1dd9625e2b1e83c23ffae3de228/internal/codelocation/code_location.go#L25:
+// - simplified API and thus renamed (calls debug.Stack() instead of taking a parameter)
+// - source code filtering updated to be specific to Kubernetes
+// - optimized to use bytes and in-place slice filtering from
+//   https://github.com/golang/go/wiki/SliceTricks#filter-in-place
+func PrunedStack(skip int) []byte {
+	fullStackTrace := debug.Stack()
+	stack := bytes.Split(fullStackTrace, []byte("\n"))
+	// Ensure that the even entries are the method names and the
+	// the odd entries the source code information.
+	if len(stack) > 0 && bytes.HasPrefix(stack[0], []byte("goroutine ")) {
+		// Ignore "goroutine 29 [running]:" line.
+		stack = stack[1:]
+	}
+	// The "+2" is for skipping over:
+	// - runtime/debug.Stack()
+	// - PrunedStack()
+	skip += 2
+	if len(stack) > 2*skip {
+		stack = stack[2*skip:]
+	}
+	n := 0
+	for i := 0; i < len(stack)/2; i++ {
+		// We filter out based on the source code file name.
+		if !codeFilterRE.Match([]byte(stack[i*2+1])) {
+			stack[n] = stack[i*2]
+			stack[n+1] = stack[i*2+1]
+			n += 2
+		}
+	}
+	stack = stack[:n]
+
+	return bytes.Join(stack, []byte("\n"))
+}

test/e2e/framework/mockservers.go

@@ -0,0 +1,60 @@
+package framework
+
+import (
+	"github.com/fatedier/frp/test/e2e/mock/echoserver"
+	"github.com/fatedier/frp/test/e2e/pkg/port"
+)
+
+const (
+	TCPEchoServerPort = "TCPEchoServerPort"
+	UDPEchoServerPort = "UDPEchoServerPort"
+)
+
+type MockServers struct {
+	tcpEchoServer *echoserver.Server
+	udpEchoServer *echoserver.Server
+}
+
+func NewMockServers(portAllocator *port.Allocator) *MockServers {
+	s := &MockServers{}
+	tcpPort := portAllocator.Get()
+	udpPort := portAllocator.Get()
+	s.tcpEchoServer = echoserver.New(echoserver.Options{
+		Type:      echoserver.TCP,
+		BindAddr:  "127.0.0.1",
+		BindPort:  int32(tcpPort),
+		RepeatNum: 1,
+	})
+	s.udpEchoServer = echoserver.New(echoserver.Options{
+		Type:      echoserver.UDP,
+		BindAddr:  "127.0.0.1",
+		BindPort:  int32(udpPort),
+		RepeatNum: 1,
+	})
+	return s
+}
+
+func (m *MockServers) Run() error {
+	if err := m.tcpEchoServer.Run(); err != nil {
+		return err
+	}
+	if err := m.udpEchoServer.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *MockServers) GetTemplateParams() map[string]interface{} {
+	ret := make(map[string]interface{})
+	ret[TCPEchoServerPort] = m.tcpEchoServer.GetOptions().BindPort
+	ret[UDPEchoServerPort] = m.udpEchoServer.GetOptions().BindPort
+	return ret
+}
+
+func (m *MockServers) GetParam(key string) interface{} {
+	params := m.GetTemplateParams()
+	if v, ok := params[key]; ok {
+		return v
+	}
+	return nil
+}

test/e2e/framework/process.go

@@ -0,0 +1,59 @@
+package framework
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"time"
+
+	"github.com/fatedier/frp/test/e2e/pkg/process"
+	flog "github.com/fatedier/frp/utils/log"
+)
+
+func GenerateConfigFile(path string, content string) error {
+	return ioutil.WriteFile(path, []byte(content), 0666)
+}
+
+// RunProcesses run multiple processes from templates.
+// The first template should always be frps.
+func (f *Framework) RunProcesses(serverTemplates []string, clientTemplates []string) {
+	templates := make([]string, 0, len(serverTemplates)+len(clientTemplates))
+	for _, t := range serverTemplates {
+		templates = append(templates, t)
+	}
+	for _, t := range clientTemplates {
+		templates = append(templates, t)
+	}
+	outs, ports, err := f.RenderTemplates(templates)
+	ExpectNoError(err)
+	ExpectTrue(len(templates) > 0)
+
+	f.UsedPorts = ports
+
+	for i := range serverTemplates {
+		path := filepath.Join(f.TempDirectory, fmt.Sprintf("frp-e2e-server-%d", i))
+		err = ioutil.WriteFile(path, []byte(outs[i]), 0666)
+		ExpectNoError(err)
+		flog.Trace("[%s] %s", path, outs[i])
+		p := process.New(TestContext.FRPServerPath, []string{"-c", path})
+		f.serverConfPaths = append(f.serverConfPaths, path)
+		f.serverProcesses = append(f.serverProcesses, p)
+		err = p.Start()
+		ExpectNoError(err)
+		time.Sleep(500 * time.Millisecond)
+	}
+
+	for i := range clientTemplates {
+		index := i + len(serverTemplates)
+		path := filepath.Join(f.TempDirectory, fmt.Sprintf("frp-e2e-client-%d", i))
+		err = ioutil.WriteFile(path, []byte(outs[index]), 0666)
+		ExpectNoError(err)
+		flog.Trace("[%s] %s", path, outs[index])
+		p := process.New(TestContext.FRPClientPath, []string{"-c", path})
+		f.clientConfPaths = append(f.clientConfPaths, path)
+		f.clientProcesses = append(f.clientProcesses, p)
+		err = p.Start()
+		ExpectNoError(err)
+		time.Sleep(500 * time.Millisecond)
+	}
+}

test/e2e/framework/request.go

@@ -0,0 +1,13 @@
+package framework
+
+import (
+	"time"
+
+	"github.com/fatedier/frp/test/e2e/pkg/request"
+)
+
+func ExpectTCPReuqest(port int, in, out []byte, timeout time.Duration) {
+	res, err := request.SendTCPRequest(port, in, timeout)
+	ExpectNoError(err)
+	ExpectEqual(string(out), res)
+}

test/e2e/framework/test_context.go

@@ -0,0 +1,42 @@
+package framework
+
+import (
+	"flag"
+	"fmt"
+	"os"
+)
+
+type TestContextType struct {
+	FRPClientPath string
+	FRPServerPath string
+	LogLevel      string
+}
+
+var TestContext TestContextType
+
+// RegisterCommonFlags registers flags common to all e2e test suites.
+// The flag set can be flag.CommandLine (if desired) or a custom
+// flag set that then gets passed to viperconfig.ViperizeFlags.
+//
+// The other Register*Flags methods below can be used to add more
+// test-specific flags. However, those settings then get added
+// regardless whether the test is actually in the test suite.
+//
+func RegisterCommonFlags(flags *flag.FlagSet) {
+	flags.StringVar(&TestContext.FRPClientPath, "frpc-path", "../../bin/frpc", "The frp client binary to use.")
+	flags.StringVar(&TestContext.FRPServerPath, "frps-path", "../../bin/frps", "The frp server binary to use.")
+	flags.StringVar(&TestContext.LogLevel, "log-level", "debug", "Log level.")
+}
+
+func ValidateTestContext(t *TestContextType) error {
+	if t.FRPClientPath == "" || t.FRPServerPath == "" {
+		return fmt.Errorf("frpc and frps binary path can't be empty")
+	}
+	if _, err := os.Stat(t.FRPClientPath); err != nil {
+		return fmt.Errorf("load frpc-path error: %v", err)
+	}
+	if _, err := os.Stat(t.FRPServerPath); err != nil {
+		return fmt.Errorf("load frps-path error: %v", err)
+	}
+	return nil
+}

test/e2e/framework/util.go

@@ -0,0 +1,14 @@
+package framework
+
+import (
+	"github.com/google/uuid"
+)
+
+// RunID is a unique identifier of the e2e run.
+// Beware that this ID is not the same for all tests in the e2e run, because each Ginkgo node creates it separately.
+var RunID string
+
+func init() {
+	uuid, _ := uuid.NewUUID()
+	RunID = uuid.String()
+}

test/e2e/mock/echoserver/echoserver.go

@@ -0,0 +1,111 @@
+package echoserver
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	fnet "github.com/fatedier/frp/utils/net"
+)
+
+type ServerType string
+
+const (
+	TCP  ServerType = "tcp"
+	UDP  ServerType = "udp"
+	Unix ServerType = "unix"
+)
+
+type Options struct {
+	Type              ServerType
+	BindAddr          string
+	BindPort          int32
+	RepeatNum         int
+	SpecifiedResponse string
+}
+
+type Server struct {
+	opt Options
+
+	l net.Listener
+}
+
+func New(opt Options) *Server {
+	if opt.Type == "" {
+		opt.Type = TCP
+	}
+	if opt.BindAddr == "" {
+		opt.BindAddr = "127.0.0.1"
+	}
+	if opt.RepeatNum <= 0 {
+		opt.RepeatNum = 1
+	}
+	return &Server{
+		opt: opt,
+	}
+}
+
+func (s *Server) GetOptions() Options {
+	return s.opt
+}
+
+func (s *Server) Run() error {
+	if err := s.initListener(); err != nil {
+		return err
+	}
+
+	go func() {
+		for {
+			c, err := s.l.Accept()
+			if err != nil {
+				return
+			}
+			go s.handle(c)
+		}
+	}()
+	return nil
+}
+
+func (s *Server) Close() error {
+	if s.l != nil {
+		return s.l.Close()
+	}
+	return nil
+}
+
+func (s *Server) initListener() (err error) {
+	switch s.opt.Type {
+	case TCP:
+		s.l, err = net.Listen("tcp", fmt.Sprintf("%s:%d", s.opt.BindAddr, s.opt.BindPort))
+	case UDP:
+		s.l, err = fnet.ListenUDP(s.opt.BindAddr, int(s.opt.BindPort))
+	case Unix:
+		s.l, err = net.Listen("unix", s.opt.BindAddr)
+	default:
+		return fmt.Errorf("unknown server type: %s", s.opt.Type)
+	}
+	if err != nil {
+		return
+	}
+	return nil
+}
+
+func (s *Server) handle(c net.Conn) {
+	defer c.Close()
+
+	buf := make([]byte, 2048)
+	for {
+		n, err := c.Read(buf)
+		if err != nil {
+			return
+		}
+
+		var response string
+		if len(s.opt.SpecifiedResponse) > 0 {
+			response = s.opt.SpecifiedResponse
+		} else {
+			response = strings.Repeat(string(buf[:n]), s.opt.RepeatNum)
+		}
+		c.Write([]byte(response))
+	}
+}

test/e2e/pkg/port/port.go

@@ -0,0 +1,57 @@
+package port
+
+import (
+	"fmt"
+	"net"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+type Allocator struct {
+	reserved sets.Int
+	used     sets.Int
+}
+
+// NewAllocator return a port allocator for testing.
+// Example: from: 10, to: 20, mod 4, index 1
+// Reserved ports: 13, 17
+func NewAllocator(from int, to int, mod int, index int) *Allocator {
+	pa := &Allocator{
+		reserved: sets.NewInt(),
+		used:     sets.NewInt(),
+	}
+	for i := from; i <= to; i++ {
+		if i%mod == index {
+			pa.reserved.Insert(i)
+		}
+	}
+	return pa
+}
+
+func (pa *Allocator) Get() int {
+	for i := 0; i < 10; i++ {
+		port, _ := pa.reserved.PopAny()
+		if port == 0 {
+			return 0
+		}
+
+		// TODO: Distinguish between TCP and UDP
+		l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+		if err != nil {
+			// Maybe not controlled by us, mark it used.
+			pa.used.Insert(port)
+			continue
+		}
+		l.Close()
+		pa.used.Insert(port)
+		return port
+	}
+	return 0
+}
+
+func (pa *Allocator) Release(port int) {
+	if pa.used.Has(port) {
+		pa.used.Delete(port)
+		pa.reserved.Insert(port)
+	}
+}

test/e2e/pkg/process/process.go

@@ -0,0 +1,47 @@
+package process
+
+import (
+	"bytes"
+	"context"
+	"os/exec"
+)
+
+type Process struct {
+	cmd         *exec.Cmd
+	cancel      context.CancelFunc
+	errorOutput *bytes.Buffer
+
+	beforeStopHandler func()
+}
+
+func New(path string, params []string) *Process {
+	ctx, cancel := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(ctx, path, params...)
+	p := &Process{
+		cmd:    cmd,
+		cancel: cancel,
+	}
+	p.errorOutput = bytes.NewBufferString("")
+	cmd.Stderr = p.errorOutput
+	return p
+}
+
+func (p *Process) Start() error {
+	return p.cmd.Start()
+}
+
+func (p *Process) Stop() error {
+	if p.beforeStopHandler != nil {
+		p.beforeStopHandler()
+	}
+	p.cancel()
+	return p.cmd.Wait()
+}
+
+func (p *Process) ErrorOutput() string {
+	return p.errorOutput.String()
+}
+
+func (p *Process) SetBeforeStopHandler(fn func()) {
+	p.beforeStopHandler = fn
+}

test/e2e/pkg/request/request.go

@@ -0,0 +1,31 @@
+package request
+
+import (
+	"fmt"
+	"net"
+	"time"
+)
+
+func SendTCPRequest(port int, content []byte, timeout time.Duration) (res string, err error) {
+	c, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+	if err != nil {
+		err = fmt.Errorf("connect to tcp server error: %v", err)
+		return
+	}
+	defer c.Close()
+
+	c.SetDeadline(time.Now().Add(timeout))
+	return sendTCPRequestByConn(c, content)
+}
+
+func sendTCPRequestByConn(c net.Conn, content []byte) (res string, err error) {
+	c.Write(content)
+
+	buf := make([]byte, 2048)
+	n, errRet := c.Read(buf)
+	if errRet != nil {
+		err = fmt.Errorf("read from tcp error: %v", errRet)
+		return
+	}
+	return string(buf[:n]), nil
+}

test/e2e/suites.go

@@ -0,0 +1,16 @@
+package e2e
+
+// CleanupSuite is the boilerplate that can be used after tests on ginkgo were run, on the SynchronizedAfterSuite step.
+// Similar to SynchronizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs).
+// Here, the order of functions is reversed; first, the function which runs everywhere,
+// and then the function that only runs on the first Ginkgo node.
+func CleanupSuite() {
+	// Run on all Ginkgo nodes
+	// TODO
+}
+
+// AfterSuiteActions are actions that are run on ginkgo's SynchronizedAfterSuite
+func AfterSuiteActions() {
+	// Run only Ginkgo on node 1
+	// TODO
+}