diff --git a/router-tests/freeport/LICENSE b/router-tests/freeport/LICENSE new file mode 100644 index 0000000000..7c5baa45e1 --- /dev/null +++ b/router-tests/freeport/LICENSE @@ -0,0 +1,365 @@ +Copyright (c) 2020 HashiCorp, Inc. + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/router-tests/freeport/ephemeral_darwin.go b/router-tests/freeport/ephemeral_darwin.go new file mode 100644 index 0000000000..18e6b012f3 --- /dev/null +++ b/router-tests/freeport/ephemeral_darwin.go @@ -0,0 +1,40 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build darwin + +package freeport + +import ( + "fmt" + "os/exec" + "regexp" + "strconv" +) + +const ephemeralPortRangeSysctlFirst = "net.inet.ip.portrange.first" +const ephemeralPortRangeSysctlLast = "net.inet.ip.portrange.last" + +var ephemeralPortRangePatt = regexp.MustCompile(`^\s*(\d+)\s+(\d+)\s*$`) + +func getEphemeralPortRange() (int, int, error) { + cmd := exec.Command("/usr/sbin/sysctl", "-n", ephemeralPortRangeSysctlFirst, ephemeralPortRangeSysctlLast) + out, err := cmd.Output() + if err != nil { + return 0, 0, err + } + + val := string(out) + + m := ephemeralPortRangePatt.FindStringSubmatch(val) + if m != nil { + min, err1 := strconv.Atoi(m[1]) + max, err2 := strconv.Atoi(m[2]) + + if err1 == nil && err2 == nil { + return min, max, nil + } + } + + return 0, 0, fmt.Errorf("unexpected sysctl value %q for keys %q, %q", val, ephemeralPortRangeSysctlFirst, ephemeralPortRangeSysctlLast) +} diff --git a/router-tests/freeport/ephemeral_darwin_test.go b/router-tests/freeport/ephemeral_darwin_test.go new file mode 100644 index 0000000000..5f68890d6b --- /dev/null +++ b/router-tests/freeport/ephemeral_darwin_test.go @@ -0,0 +1,21 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build darwin + +package freeport + +import ( + "testing" +) + +func TestGetEphemeralPortRange(t *testing.T) { + min, max, err := getEphemeralPortRange() + if err != nil { + t.Fatalf("err: %v", err) + } + if min <= 0 || max <= 0 || min > max { + t.Fatalf("unexpected values: min=%d, max=%d", min, max) + } + t.Logf("min=%d, max=%d", min, max) +} diff --git a/router-tests/freeport/ephemeral_fallback.go b/router-tests/freeport/ephemeral_fallback.go new file mode 100644 index 0000000000..94ba3fb3c0 --- /dev/null +++ b/router-tests/freeport/ephemeral_fallback.go @@ -0,0 +1,10 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build !linux && !darwin + +package freeport + +func getEphemeralPortRange() (int, int, error) { + return 0, 0, nil +} diff --git a/router-tests/freeport/ephemeral_linux.go b/router-tests/freeport/ephemeral_linux.go new file mode 100644 index 0000000000..ea7c133b5d --- /dev/null +++ b/router-tests/freeport/ephemeral_linux.go @@ -0,0 +1,38 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build linux + +package freeport + +import ( + "fmt" + "os" + "regexp" + "strconv" +) + +const ephemeralPortRangeProcFile = "/proc/sys/net/ipv4/ip_local_port_range" + +var ephemeralPortRangePatt = regexp.MustCompile(`^\s*(\d+)\s+(\d+)\s*$`) + +func getEphemeralPortRange() (int, int, error) { + out, err := os.ReadFile(ephemeralPortRangeProcFile) + if err != nil { + return 0, 0, err + } + + val := string(out) + + m := ephemeralPortRangePatt.FindStringSubmatch(val) + if m != nil { + min, err1 := strconv.Atoi(m[1]) + max, err2 := strconv.Atoi(m[2]) + + if err1 == nil && err2 == nil { + return min, max, nil + } + } + + return 0, 0, fmt.Errorf("unexpected sysctl value %q for key %q", val, ephemeralPortRangeProcFile) +} diff --git a/router-tests/freeport/ephemeral_linux_test.go b/router-tests/freeport/ephemeral_linux_test.go new file mode 100644 index 0000000000..aa3e2de53a --- /dev/null +++ b/router-tests/freeport/ephemeral_linux_test.go @@ -0,0 +1,21 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build linux + +package freeport + +import ( + "testing" +) + +func TestGetEphemeralPortRange(t *testing.T) { + min, max, err := getEphemeralPortRange() + if err != nil { + t.Fatalf("err: %v", err) + } + if min <= 0 || max <= 0 || min > max { + t.Fatalf("unexpected values: min=%d, max=%d", min, max) + } + t.Logf("min=%d, max=%d", min, max) +} diff --git a/router-tests/freeport/freeport.go b/router-tests/freeport/freeport.go new file mode 100644 index 0000000000..de008c7104 --- /dev/null +++ b/router-tests/freeport/freeport.go @@ -0,0 +1,487 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +// Package freeport provides a helper for reserving free TCP ports across multiple +// processes on the same machine. Each process reserves a block of ports outside +// the ephemeral port range. Tests can request one of these reserved ports +// and freeport will ensure that no other test uses that port until it is returned +// to freeport. +// +// Freeport is particularly useful when the code being tested does not accept +// a net.Listener. Any code that accepts a net.Listener (or uses net/http/httptest.Server) +// can use port 0 (ex: 127.0.0.1:0) to find an unused ephemeral port that will +// not conflict. +// +// Any code that does not accept a net.Listener or can not bind directly to port +// zero should use freeport to find an unused port. +package freeport + +import ( + "container/list" + "fmt" + "math/rand" + "net" + "os" + "runtime" + "sync" + "time" +) + +const ( + // maxBlocks is the number of available port blocks before exclusions. + maxBlocks = 30 + + // lowPort is the lowest port number that should be used. + lowPort = 10000 + + // attempts is how often we try to allocate a port block + // before giving up. + attempts = 10 +) + +var ( + // blockSize is the size of the allocated port block. ports are given out + // consecutively from that block and after that point in a LRU fashion. + blockSize int + + // effectiveMaxBlocks is the number of available port blocks. + // lowPort + effectiveMaxBlocks * blockSize must be less than 65535. + effectiveMaxBlocks int + + // firstPort is the first port of the allocated block. + firstPort int + + // lockLn is the system-wide mutex for the port block. + lockLn net.Listener + + // mu guards: + // - pendingPorts + // - freePorts + // - total + mu sync.Mutex + + // once is used to do the initialization on the first call to retrieve free + // ports + once sync.Once + + // condNotEmpty is a condition variable to wait for freePorts to be not + // empty. Linked to 'mu' + condNotEmpty *sync.Cond + + // freePorts is a FIFO of all currently free ports. Take from the front, + // and return to the back. + freePorts *list.List + + // pendingPorts is a FIFO of recently freed ports that have not yet passed + // the not-in-use check. + pendingPorts *list.List + + // total is the total number of available ports in the block for use. + total int + + // seededRand is a random generator that is pre-seeded from the current time. + seededRand *rand.Rand + + // stopCh is used to signal to background goroutines to terminate. Only + // really exists for the safety of reset() during unit tests. + stopCh chan struct{} + + // stopWg is used to keep track of background goroutines that are still + // alive. Only really exists for the safety of reset() during unit tests. + stopWg sync.WaitGroup + + // portLastUser associates ports with a test name in order to debug + // which test may be leaking unclosed TCP connections. + portLastUser map[int]string + + // logLevel is the verbosity of the logging output. + // The default log level is DEBUG. + logLevel LogLevel +) + +// initialize is used to initialize freeport. +func initialize() { + var err error + + blockSize = 1500 + limit, err := systemLimit() + if err != nil { + panic("freeport: error getting system limit: " + err.Error()) + } + if limit > 0 && limit < blockSize { + logf(INFO, "blockSize %d too big for system limit %d. Adjusting...", blockSize, limit) + blockSize = limit - 3 + } + + effectiveMaxBlocks, err = adjustMaxBlocks() + if err != nil { + panic("freeport: ephemeral port range detection failed: " + err.Error()) + } + if effectiveMaxBlocks < 0 { + panic("freeport: no blocks of ports available outside of ephemeral range") + } + if lowPort+effectiveMaxBlocks*blockSize > 65535 { + panic("freeport: block size too big or too many blocks requested") + } + + seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) // This is compatible with go 1.19 but unnecessary in >= go1.20 + firstPort, lockLn = alloc() + + condNotEmpty = sync.NewCond(&mu) + freePorts = list.New() + pendingPorts = list.New() + + // fill with all available free ports + for port := firstPort + 1; port < firstPort+blockSize; port++ { + if used := isPortInUse(port); !used { + freePorts.PushBack(port) + } + } + total = freePorts.Len() + + stopWg.Add(1) + stopCh = make(chan struct{}) + + portLastUser = make(map[int]string) + // Note: we pass this param explicitly to the goroutine so that we can + // freely recreate the underlying stop channel during reset() after closing + // the original. + go checkFreedPorts(stopCh) +} + +func shutdownGoroutine() { + mu.Lock() + if stopCh == nil { + mu.Unlock() + return + } + + close(stopCh) + stopCh = nil + mu.Unlock() + + stopWg.Wait() +} + +// reset will reverse the setup from initialize() and then redo it (for tests) +func reset() { + logf(INFO, "resetting the freeport package state") + shutdownGoroutine() + + mu.Lock() + defer mu.Unlock() + + effectiveMaxBlocks = 0 + firstPort = 0 + if lockLn != nil { + lockLn.Close() + lockLn = nil + } + + once = sync.Once{} + + freePorts = nil + pendingPorts = nil + portLastUser = nil + total = 0 +} + +func checkFreedPorts(stopCh <-chan struct{}) { + defer stopWg.Done() + + ticker := time.NewTicker(250 * time.Millisecond) + for { + select { + case <-stopCh: + logf(INFO, "Closing checkFreedPorts()") + return + case <-ticker.C: + checkFreedPortsOnce() + } + } +} + +func checkFreedPortsOnce() { + mu.Lock() + defer mu.Unlock() + + pending := pendingPorts.Len() + remove := make([]*list.Element, 0, pending) + for elem := pendingPorts.Front(); elem != nil; elem = elem.Next() { + port := elem.Value.(int) + if used := isPortInUse(port); !used { + freePorts.PushBack(port) + remove = append(remove, elem) + } else { + logf(WARN, "port %d still being used by %q", port, portLastUser[port]) + } + } + + retained := pending - len(remove) + + if retained > 0 { + logf(WARN, "%d out of %d pending ports are still in use; something probably didn't wait around for the port to be closed!", retained, pending) + } + + if len(remove) == 0 { + return + } + + for _, elem := range remove { + pendingPorts.Remove(elem) + } + + condNotEmpty.Broadcast() +} + +// adjustMaxBlocks avoids having the allocation ranges overlap the ephemeral +// port range. +func adjustMaxBlocks() (int, error) { + ephemeralPortMin, ephemeralPortMax, err := getEphemeralPortRange() + if err != nil { + return 0, err + } + + if ephemeralPortMin <= 0 || ephemeralPortMax <= 0 { + logf(INFO, "ephemeral port range detection not configured for GOOS=%q", runtime.GOOS) + return maxBlocks, nil + } + + logf(INFO, "detected ephemeral port range of [%d, %d]", ephemeralPortMin, ephemeralPortMax) + for block := 0; block < maxBlocks; block++ { + min := lowPort + block*blockSize + max := min + blockSize + overlap := intervalOverlap(min, max-1, ephemeralPortMin, ephemeralPortMax) + if overlap { + logf(INFO, "reducing max blocks from %d to %d to avoid the ephemeral port range", maxBlocks, block) + return block, nil + } + } + return maxBlocks, nil +} + +// alloc reserves a port block for exclusive use for the lifetime of the +// application. lockLn serves as a system-wide mutex for the port block and is +// implemented as a TCP listener which is bound to the firstPort and which will +// be automatically released when the application terminates. +func alloc() (int, net.Listener) { + for i := 0; i < attempts; i++ { + block := int(seededRand.Int31n(int32(effectiveMaxBlocks))) + firstPort := lowPort + block*blockSize + ln, err := net.ListenTCP("tcp", tcpAddr("127.0.0.1", firstPort)) + if err != nil { + continue + } + // logf("DEBUG", "allocated port block %d (%d-%d)", block, firstPort, firstPort+blockSize-1) + return firstPort, ln + } + panic("freeport: cannot allocate port block") +} + +type LogLevel uint8 + +const ( + DEBUG LogLevel = iota + INFO + WARN + ERROR + DISABLED +) + +func (l LogLevel) String() string { + switch l { + case DEBUG: + return "DEBUG" + case INFO: + return "INFO" + case WARN: + return "WARN" + case ERROR: + return "ERROR" + default: + return "DISABLED" + } +} + +// SetLogLevel sets the verbosity of the logging output. +// The default log level is DEBUG. +func SetLogLevel(level LogLevel) { + mu.Lock() + defer mu.Unlock() + + if level > DISABLED { + level = DISABLED + } + + logLevel = level +} + +// Take returns a list of free ports from the reserved port block. It is safe +// to call this method concurrently. Ports have been tested to be available on +// 127.0.0.1 TCP but there is no guarantee that they will remain free in the +// future. +// +// Most callers should prefer GetN or GetOne. +func Take(n int) (ports []int, err error) { + if n <= 0 { + return nil, fmt.Errorf("freeport: cannot take %d ports", n) + } + + mu.Lock() + defer mu.Unlock() + + // Reserve a port block + once.Do(initialize) + + if n > total { + return nil, fmt.Errorf("freeport: block size too small") + } + + for len(ports) < n { + for freePorts.Len() == 0 { + if total == 0 { + return nil, fmt.Errorf("freeport: impossible to satisfy request; there are no actual free ports in the block anymore") + } + condNotEmpty.Wait() + } + + elem := freePorts.Front() + freePorts.Remove(elem) + port := elem.Value.(int) + + if used := isPortInUse(port); used { + // Something outside of the test suite has stolen this port, possibly + // due to assignment to an ephemeral port, remove it completely. + logf(WARN, "leaked port %d due to theft; removing from circulation", port) + total-- + continue + } + + ports = append(ports, port) + } + + return ports, nil +} + +// peekFree returns the next port that will be returned by Take to aid in testing. +func peekFree() int { + mu.Lock() + defer mu.Unlock() + return freePorts.Front().Value.(int) +} + +// peekAllFree returns all free ports that could be returned by Take to aid in testing. +func peekAllFree() []int { + mu.Lock() + defer mu.Unlock() + + var out []int + for elem := freePorts.Front(); elem != nil; elem = elem.Next() { + port := elem.Value.(int) + out = append(out, port) + } + + return out +} + +// stats returns diagnostic data to aid in testing +func stats() (numTotal, numPending, numFree int) { + mu.Lock() + defer mu.Unlock() + return total, pendingPorts.Len(), freePorts.Len() +} + +// Return returns a block of ports back to the general pool. These ports should +// have been returned from a call to Take(). +func Return(ports []int) { + if len(ports) == 0 { + return // convenience short circuit for test ergonomics + } + + mu.Lock() + defer mu.Unlock() + + for _, port := range ports { + if port > firstPort && port < firstPort+blockSize { + pendingPorts.PushBack(port) + } + } +} + +func isPortInUse(port int) bool { + ln, err := net.ListenTCP("tcp", tcpAddr("127.0.0.1", port)) + if err != nil { + return true + } + ln.Close() + return false +} + +func tcpAddr(ip string, port int) *net.TCPAddr { + return &net.TCPAddr{IP: net.ParseIP(ip), Port: port} +} + +// intervalOverlap returns true if the doubly-inclusive integer intervals +// represented by [min1, max1] and [min2, max2] overlap. +func intervalOverlap(min1, max1, min2, max2 int) bool { + if min1 > max1 { + logf(WARN, "interval1 is not ordered [%d, %d]", min1, max1) + return false + } + if min2 > max2 { + logf(WARN, "interval2 is not ordered [%d, %d]", min2, max2) + return false + } + return min1 <= max2 && min2 <= max1 +} + +func logf(severity LogLevel, format string, a ...interface{}) { + if severity >= DISABLED { + return + } + + if severity >= logLevel { + fmt.Fprintf(os.Stderr, "["+severity.String()+"] freeport: "+format+"\n", a...) + } +} + +// TestingT is the minimal set of methods implemented by *testing.T that are +// used by functions in freelist. +// +// In the future new methods may be added to this interface, but those methods +// should always be implemented by *testing.T +type TestingT interface { + Cleanup(func()) + Helper() + Fatalf(format string, args ...interface{}) + Name() string +} + +// GetN returns n free ports from the reserved port block, and returns the +// ports to the pool when the test ends. See Take for more details. +func GetN(t TestingT, n int) []int { + t.Helper() + ports, err := Take(n) + if err != nil { + t.Fatalf("failed to take %v ports: %v", n, err) + } + logf(DEBUG, "Test %q took ports %v", t.Name(), ports) + mu.Lock() + for _, p := range ports { + portLastUser[p] = t.Name() + } + mu.Unlock() + t.Cleanup(func() { + Return(ports) + logf(DEBUG, "Test %q returned ports %v", t.Name(), ports) + }) + return ports +} + +// GetOne returns a single free port from the reserved port block, and returns the +// port to the pool when the test ends. See Take for more details. +// Use GetN if more than a single port is required. +func GetOne(t TestingT) int { + t.Helper() + return GetN(t, 1)[0] +} diff --git a/router-tests/freeport/freeport_test.go b/router-tests/freeport/freeport_test.go new file mode 100644 index 0000000000..28d30d3f34 --- /dev/null +++ b/router-tests/freeport/freeport_test.go @@ -0,0 +1,362 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package freeport + +import ( + "bytes" + "fmt" + "io" + "net" + "os" + "testing" + + "github.com/wundergraph/cosmo/router-tests/freeport/retry" +) + +func TestTakeReturn(t *testing.T) { + // NOTE: for global var reasons this cannot execute in parallel + // t.Parallel() + + // Since this test is destructive (i.e. it leaks all ports) it means that + // any other test cases in this package will not function after it runs. To + // help out we reset the global state after we run this test. + defer reset() + + // OK: do a simple take/return cycle to trigger the package initialization + func() { + ports, err := Take(1) + if err != nil { + t.Fatalf("err: %v", err) + } + defer Return(ports) + + if len(ports) != 1 { + t.Fatalf("expected %d but got %d ports", 1, len(ports)) + } + }() + + waitForStatsReset := func() (numTotal int) { + t.Helper() + numTotal, numPending, numFree := stats() + if numTotal != numFree+numPending { + t.Fatalf("expected total (%d) and free+pending (%d) ports to match", numTotal, numFree+numPending) + } + retry.Run(t, func(r *retry.R) { + numTotal, numPending, numFree = stats() + if numPending != 0 { + r.Fatalf("pending is still non zero: %d", numPending) + } + if numTotal != numFree { + r.Fatalf("total (%d) does not equal free (%d)", numTotal, numFree) + } + }) + return numTotal + } + + // Reset + numTotal := waitForStatsReset() + + // -------------------- + // OK: take the max + func() { + ports, err := Take(numTotal) + if err != nil { + t.Fatalf("err: %v", err) + } + defer Return(ports) + + if len(ports) != numTotal { + t.Fatalf("expected %d but got %d ports", numTotal, len(ports)) + } + }() + + // Reset + numTotal = waitForStatsReset() + + expectError := func(expected string, got error) { + t.Helper() + if got == nil { + t.Fatalf("expected error but was nil") + } + if got.Error() != expected { + t.Fatalf("expected error %q but got %q", expected, got.Error()) + } + } + + // -------------------- + // ERROR: take too many ports + func() { + ports, err := Take(numTotal + 1) + defer Return(ports) + expectError("freeport: block size too small", err) + }() + + // -------------------- + // ERROR: invalid ports request (negative) + func() { + _, err := Take(-1) + expectError("freeport: cannot take -1 ports", err) + }() + + // -------------------- + // ERROR: invalid ports request (zero) + func() { + _, err := Take(0) + expectError("freeport: cannot take 0 ports", err) + }() + + // -------------------- + // OK: Steal a port under the covers and let freeport detect the theft and compensate + leakedPort := peekFree() + func() { + leakyListener, err := net.ListenTCP("tcp", tcpAddr("127.0.0.1", leakedPort)) + if err != nil { + t.Fatalf("err: %v", err) + } + defer leakyListener.Close() + + func() { + ports, err := Take(3) + if err != nil { + t.Fatalf("err: %v", err) + } + defer Return(ports) + + if len(ports) != 3 { + t.Fatalf("expected %d but got %d ports", 3, len(ports)) + } + + for _, port := range ports { + if port == leakedPort { + t.Fatalf("did not expect for Take to return the leaked port") + } + } + }() + + newNumTotal := waitForStatsReset() + if newNumTotal != numTotal-1 { + t.Fatalf("expected total to drop to %d but got %d", numTotal-1, newNumTotal) + } + numTotal = newNumTotal // update outer variable for later tests + }() + + // -------------------- + // OK: sequence it so that one Take must wait on another Take to Return. + func() { + mostPorts, err := Take(numTotal - 5) + if err != nil { + t.Fatalf("err: %v", err) + } + + type reply struct { + ports []int + err error + } + ch := make(chan reply, 1) + go func() { + ports, err := Take(10) + ch <- reply{ports: ports, err: err} + }() + + Return(mostPorts) + + r := <-ch + if r.err != nil { + t.Fatalf("err: %v", r.err) + } + defer Return(r.ports) + + if len(r.ports) != 10 { + t.Fatalf("expected %d ports but got %d", 10, len(r.ports)) + } + }() + + // Reset + numTotal = waitForStatsReset() + + // -------------------- + // ERROR: Now we end on the crazy "Ocean's 11" level port theft where we + // orchestrate a situation where all ports are stolen and we don't find out + // until Take. + func() { + // 1. Grab all of the ports. + allPorts := peekAllFree() + + // 2. Leak all of the ports + leaked := make([]io.Closer, 0, len(allPorts)) + defer func() { + for _, c := range leaked { + c.Close() + } + }() + for i, port := range allPorts { + ln, err := net.ListenTCP("tcp", tcpAddr("127.0.0.1", port)) + if err != nil { + t.Fatalf("%d err: %v", i, err) + } + leaked = append(leaked, ln) + } + + // 3. Request 1 port which will detect the leaked ports and fail. + _, err := Take(1) + expectError("freeport: impossible to satisfy request; there are no actual free ports in the block anymore", err) + + // 4. Wait for the block to zero out. + newNumTotal := waitForStatsReset() + if newNumTotal != 0 { + t.Fatalf("expected total to drop to %d but got %d", 0, newNumTotal) + } + }() +} + +func TestIntervalOverlap(t *testing.T) { + cases := []struct { + min1, max1, min2, max2 int + overlap bool + }{ + {0, 0, 0, 0, true}, + {1, 1, 1, 1, true}, + {1, 3, 1, 3, true}, // same + {1, 3, 4, 6, false}, // serial + {1, 4, 3, 6, true}, // inner overlap + {1, 6, 3, 4, true}, // nest + } + + for _, tc := range cases { + t.Run(fmt.Sprintf("%d:%d vs %d:%d", tc.min1, tc.max1, tc.min2, tc.max2), func(t *testing.T) { + if tc.overlap != intervalOverlap(tc.min1, tc.max1, tc.min2, tc.max2) { // 1 vs 2 + t.Fatalf("expected %v but got %v", tc.overlap, !tc.overlap) + } + if tc.overlap != intervalOverlap(tc.min2, tc.max2, tc.min1, tc.max1) { // 2 vs 1 + t.Fatalf("expected %v but got %v", tc.overlap, !tc.overlap) + } + }) + } +} + +func TestAllowSettingLogLevel(t *testing.T) { + cases := []struct { + level LogLevel + expected LogLevel + }{ + {DEBUG, DEBUG}, + {INFO, INFO}, + {WARN, WARN}, + {ERROR, ERROR}, + {DISABLED, DISABLED}, + {LogLevel(99), DISABLED}, + } + + for _, tc := range cases { + t.Run(fmt.Sprintf("Setting log level to %v", tc.level), func(t *testing.T) { + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + + t.Cleanup(func() { + _ = r.Close() + }) + + origStderr := os.Stderr + os.Stderr = w + t.Cleanup(func() { + os.Stderr = origStderr + }) + SetLogLevel(tc.level) + if logLevel != tc.expected { + t.Fatalf("expected log level to be %v but got %v", tc.level, logLevel) + } + + logf(tc.level, "This is a test log message") + _ = w.Close() + + var b bytes.Buffer + if _, err := io.Copy(&b, r); err != nil { + t.Fatalf("failed to read from pipe: %v", err) + } + + if tc.level >= DISABLED { + if b.String() != "" { + t.Fatalf("expected no log output but got %q", b.String()) + } + + return + } + + expectedPrefix := []byte("[" + tc.expected.String() + "]") + + if !bytes.HasPrefix(b.Bytes(), expectedPrefix) { + t.Fatalf("expected log output to start with %q but got %q", expectedPrefix, b.Bytes()) + } + }) + } +} + +func TestLogLevelMessages(t *testing.T) { + tests := []struct { + level LogLevel + inputLogLevel LogLevel + expectMessageInStdErr bool + }{ + { + level: DEBUG, + inputLogLevel: INFO, + expectMessageInStdErr: true, + }, + { + level: INFO, + inputLogLevel: INFO, + expectMessageInStdErr: true, + }, + { + level: WARN, + inputLogLevel: INFO, + expectMessageInStdErr: false, + }, + { + level: ERROR, + inputLogLevel: INFO, + expectMessageInStdErr: false, + }, + } + + for _, tc := range tests { + t.Run(fmt.Sprintf("Setting log level to %v", tc.level), func(t *testing.T) { + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + + t.Cleanup(func() { + _ = r.Close() + }) + + origStderr := os.Stderr + os.Stderr = w + t.Cleanup(func() { + os.Stderr = origStderr + }) + SetLogLevel(tc.level) + + logf(tc.inputLogLevel, "This is a test log message") + _ = w.Close() + + var b bytes.Buffer + if _, err := io.Copy(&b, r); err != nil { + t.Fatalf("failed to read from pipe: %v", err) + } + + if tc.expectMessageInStdErr { + if b.String() == "" { + t.Fatalf("expected log output but got none") + } + } else { + if b.String() != "" { + t.Fatalf("expected no log output but got %q", b.String()) + } + } + }) + } +} diff --git a/router-tests/freeport/retry/counter.go b/router-tests/freeport/retry/counter.go new file mode 100644 index 0000000000..ffd509f1a4 --- /dev/null +++ b/router-tests/freeport/retry/counter.go @@ -0,0 +1,26 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package retry + +import "time" + +// Counter repeats an operation a given number of +// times and waits between subsequent operations. +type Counter struct { + Count int + Wait time.Duration + + count int +} + +func (r *Counter) Continue() bool { + if r.count == r.Count { + return false + } + if r.count > 0 { + time.Sleep(r.Wait) + } + r.count++ + return true +} diff --git a/router-tests/freeport/retry/doc.go b/router-tests/freeport/retry/doc.go new file mode 100644 index 0000000000..4afaac062b --- /dev/null +++ b/router-tests/freeport/retry/doc.go @@ -0,0 +1,22 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +// Package retry provides support for repeating operations in tests. +// +// A sample retry operation looks like this: +// +// func TestX(t *testing.T) { +// retry.Run(t, func(r *retry.R) { +// if err := foo(); err != nil { +// r.Errorf("foo: %s", err) +// return +// } +// }) +// } +// +// Run uses the DefaultFailer, which is a Timer with a Timeout of 7s, +// and a Wait of 25ms. To customize, use RunWith. +// +// WARNING: unlike *testing.T, *retry.R#Fatal and FailNow *do not* +// fail the test function entirely, only the current run the retry func +package retry diff --git a/router-tests/freeport/retry/interface.go b/router-tests/freeport/retry/interface.go new file mode 100644 index 0000000000..78d2d948ff --- /dev/null +++ b/router-tests/freeport/retry/interface.go @@ -0,0 +1,35 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package retry + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var nilInf TestingTB = nil + +// Assertion that our TestingTB can be passed to +var _ require.TestingT = nilInf +var _ assert.TestingT = nilInf + +// TestingTB is an interface that describes the implementation of the testing object. +// Using an interface that describes testing.TB instead of the actual implementation +// makes testutil usable in a wider variety of contexts (e.g. use with ginkgo : https://godoc.org/github.com/onsi/ginkgo#GinkgoT) +type TestingTB interface { + Cleanup(func()) + Error(args ...any) + Errorf(format string, args ...any) + Fail() + FailNow() + Failed() bool + Fatal(args ...any) + Fatalf(format string, args ...any) + Helper() + Log(args ...any) + Logf(format string, args ...any) + Name() string + Setenv(key, value string) + TempDir() string +} diff --git a/router-tests/freeport/retry/output.go b/router-tests/freeport/retry/output.go new file mode 100644 index 0000000000..6d4006f42d --- /dev/null +++ b/router-tests/freeport/retry/output.go @@ -0,0 +1,42 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package retry + +import ( + "bytes" + "fmt" + "runtime" + "strings" +) + +func dedup(a []string) string { + if len(a) == 0 { + return "" + } + seen := map[string]struct{}{} + var b bytes.Buffer + for _, s := range a { + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + b.WriteString(s) + b.WriteRune('\n') + } + return b.String() +} + +func decorate(s string) string { + _, file, line, ok := runtime.Caller(3) + if ok { + n := strings.LastIndex(file, "/") + if n >= 0 { + file = file[n+1:] + } + } else { + file = "???" + line = 1 + } + return fmt.Sprintf("%s:%d: %s", file, line, s) +} diff --git a/router-tests/freeport/retry/retry.go b/router-tests/freeport/retry/retry.go new file mode 100644 index 0000000000..c095b0291b --- /dev/null +++ b/router-tests/freeport/retry/retry.go @@ -0,0 +1,228 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package retry + +import ( + "fmt" + "os" +) + +var _ TestingTB = &R{} + +type R struct { + wrapped TestingTB + retryer Retryer + + done bool + fullOutput bool + immediateCleanup bool + + attempts []*attempt +} + +func (r *R) Cleanup(clean func()) { + if r.immediateCleanup { + a := r.getCurrentAttempt() + a.cleanups = append(a.cleanups, clean) + } else { + r.wrapped.Cleanup(clean) + } +} + +func (r *R) Error(args ...any) { + r.Log(args...) + r.Fail() +} + +func (r *R) Errorf(format string, args ...any) { + r.Logf(format, args...) + r.Fail() +} + +func (r *R) Fail() { + r.getCurrentAttempt().failed = true +} + +func (r *R) FailNow() { + r.Fail() + panic(attemptFailed{}) +} + +func (r *R) Failed() bool { + return r.getCurrentAttempt().failed +} + +func (r *R) Fatal(args ...any) { + r.Log(args...) + r.FailNow() +} + +func (r *R) Fatalf(format string, args ...any) { + r.Logf(format, args...) + r.FailNow() +} + +func (r *R) Helper() { + // *testing.T will just record which functions are helpers by their addresses and + // it doesn't much matter where where we record that they are helpers + r.wrapped.Helper() +} + +func (r *R) Log(args ...any) { + r.log(fmt.Sprintln(args...)) +} + +func (r *R) Logf(format string, args ...any) { + r.log(fmt.Sprintf(format, args...)) +} + +// Name will return the name of the underlying TestingT. +func (r *R) Name() string { + return r.wrapped.Name() +} + +// Setenv will save the current value of the specified env var, set it to the +// specified value and then restore it to the original value in a cleanup function +// once the retry attempt has finished. +func (r *R) Setenv(key, value string) { + prevValue, ok := os.LookupEnv(key) + + if err := os.Setenv(key, value); err != nil { + r.wrapped.Fatalf("cannot set environment variable: %v", err) + } + + if ok { + r.Cleanup(func() { + os.Setenv(key, prevValue) + }) + } else { + r.Cleanup(func() { + os.Unsetenv(key) + }) + } +} + +// TempDir will use the wrapped TestingT to create a temporary directory +// that will be cleaned up when ALL RETRYING has finished. +func (r *R) TempDir() string { + return r.wrapped.TempDir() +} + +// Check will call r.Fatal(err) if err is not nil +func (r *R) Check(err error) { + if err != nil { + r.Fatal(err) + } +} + +func (r *R) Stop(err error) { + r.log(err.Error()) + r.done = true +} + +func (r *R) log(s string) { + a := r.getCurrentAttempt() + a.output = append(a.output, decorate(s)) +} + +func (r *R) getCurrentAttempt() *attempt { + if len(r.attempts) == 0 { + panic("no retry attempts have been started yet") + } + + return r.attempts[len(r.attempts)-1] +} + +// cleanupAttempt will perform all the register cleanup operations recorded +// during execution of the single round of the test function. +func (r *R) cleanupAttempt(a *attempt) { + // Make sure that if a cleanup function panics, + // we still run the remaining cleanup functions. + defer func() { + err := recover() + if err != nil { + r.Stop(fmt.Errorf("error when performing test cleanup: %v", err)) + } + if len(a.cleanups) > 0 { + r.cleanupAttempt(a) + } + }() + + for len(a.cleanups) > 0 { + var cleanup func() + if len(a.cleanups) > 0 { + last := len(a.cleanups) - 1 + cleanup = a.cleanups[last] + a.cleanups = a.cleanups[:last] + } + if cleanup != nil { + cleanup() + } + } +} + +// runAttempt will execute one round of the test function and handle cleanups and panic recovery +// of a failed attempt that should not stop retrying. +func (r *R) runAttempt(f func(r *R)) { + r.Helper() + + a := &attempt{} + r.attempts = append(r.attempts, a) + + defer r.cleanupAttempt(a) + defer func() { + if p := recover(); p != nil && p != (attemptFailed{}) { + panic(p) + } + }() + f(r) +} + +func (r *R) run(f func(r *R)) { + r.Helper() + + for r.retryer.Continue() { + r.runAttempt(f) + + switch { + case r.done: + r.recordRetryFailure() + return + case !r.Failed(): + // the current attempt did not fail so we can go ahead and return + return + } + } + + // We cannot retry any more and no attempt has succeeded yet. + r.recordRetryFailure() +} + +func (r *R) recordRetryFailure() { + r.Helper() + output := r.getCurrentAttempt().output + if r.fullOutput { + var combined []string + for _, attempt := range r.attempts { + combined = append(combined, attempt.output...) + } + output = combined + } + + out := dedup(output) + if out != "" { + r.wrapped.Log(out) + } + r.wrapped.FailNow() +} + +type attempt struct { + failed bool + output []string + cleanups []func() +} + +// attemptFailed is a sentinel value to indicate that the func itself +// didn't panic, rather that `FailNow` was called. +type attemptFailed struct{} diff --git a/router-tests/freeport/retry/retry_test.go b/router-tests/freeport/retry/retry_test.go new file mode 100644 index 0000000000..ecc8e56082 --- /dev/null +++ b/router-tests/freeport/retry/retry_test.go @@ -0,0 +1,263 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package retry + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// delta defines the time band a test run should complete in. +var delta = 25 * time.Millisecond + +func TestRetryer(t *testing.T) { + tests := []struct { + desc string + r Retryer + }{ + {"counter", &Counter{Count: 3, Wait: 100 * time.Millisecond}}, + {"timer", &Timer{Timeout: 200 * time.Millisecond, Wait: 100 * time.Millisecond}}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + var iters int + start := time.Now() + for tt.r.Continue() { + iters++ + } + dur := time.Since(start) + if got, want := iters, 3; got != want { + t.Fatalf("got %d retries want %d", got, want) + } + // since the first iteration happens immediately + // the retryer waits only twice for three iterations. + // order of events: (true, (wait) true, (wait) true, false) + if got, want := dur, 200*time.Millisecond; got < (want-delta) || got > (want+delta) { + t.Fatalf("loop took %v want %v (+/- %v)", got, want, delta) + } + }) + } +} + +func TestBasics(t *testing.T) { + t.Run("Error allows retry", func(t *testing.T) { + i := 0 + Run(t, func(r *R) { + i++ + t.Logf("i: %d; r: %#v", i, r) + if i == 1 { + r.Errorf("Errorf, i: %d", i) + return + } + }) + assert.Equal(t, i, 2) + }) + + t.Run("Fatal returns from func, but does not fail test", func(t *testing.T) { + i := 0 + gotHere := false + ft := &fakeT{T: t} + Run(ft, func(r *R) { + i++ + t.Logf("i: %d; r: %#v", i, r) + if i == 1 { + r.Fatalf("Fatalf, i: %d", i) + gotHere = true + } + }) + + assert.False(t, gotHere) + assert.Equal(t, i, 2) + // surprisingly, r.FailNow() *does not* trigger ft.FailNow()! + assert.Equal(t, ft.fails, 0) + }) + + t.Run("Func being run can panic with struct{}{}", func(t *testing.T) { + gotPanic := false + func() { + defer func() { + if p := recover(); p != nil { + gotPanic = true + } + }() + Run(t, func(r *R) { + panic(struct{}{}) + }) + }() + + assert.True(t, gotPanic) + }) +} + +func TestRunWith(t *testing.T) { + t.Run("calls FailNow after exceeding retries", func(t *testing.T) { + ft := &fakeT{T: t} + iter := 0 + RunWith(&Counter{Count: 3, Wait: time.Millisecond}, ft, func(r *R) { + iter++ + r.FailNow() + }) + + require.Equal(t, 3, iter) + require.Equal(t, 1, ft.fails) + }) + + t.Run("Stop ends the retrying", func(t *testing.T) { + ft := &fakeT{T: t} + iter := 0 + RunWith(&Counter{Count: 5, Wait: time.Millisecond}, ft, func(r *R) { + iter++ + if iter == 2 { + r.Stop(fmt.Errorf("do not proceed")) + } + r.Fatalf("not yet") + }) + + // TODO: these should all be assert + require.Equal(t, 2, iter) + require.Equal(t, 1, ft.fails) + require.Len(t, ft.out, 1) + require.Contains(t, ft.out[0], "not yet\n") + require.Contains(t, ft.out[0], "do not proceed\n") + }) +} + +func TestCleanup_Passthrough(t *testing.T) { + +} + +func TestCleanup(t *testing.T) { + + t.Run("basic", func(t *testing.T) { + ft := &fakeT{T: t} + cleanupsExecuted := 0 + + Run( + ft, + func(r *R) { + r.Cleanup(func() { + cleanupsExecuted += 1 + }) + }, + WithImmediateCleanup(), + WithRetryer(&Counter{Count: 2, Wait: time.Millisecond}), + ) + + require.Equal(t, 0, ft.fails) + require.Equal(t, 1, cleanupsExecuted) + }) + t.Run("cleanup-panic-recovery", func(t *testing.T) { + ft := &fakeT{T: t} + cleanupsExecuted := 0 + Run( + ft, + func(r *R) { + r.Cleanup(func() { + cleanupsExecuted += 1 + }) + + r.Cleanup(func() { + cleanupsExecuted += 1 + panic(fmt.Errorf("fake test error")) + }) + + r.Cleanup(func() { + cleanupsExecuted += 1 + }) + + // test is successful but should fail due to the cleanup panicing + }, + WithRetryer(&Counter{Count: 2, Wait: time.Millisecond}), + WithImmediateCleanup(), + ) + + require.Equal(t, 3, cleanupsExecuted) + require.Equal(t, 1, ft.fails) + require.Contains(t, ft.out[0], "fake test error") + }) + + t.Run("cleanup-per-retry", func(t *testing.T) { + ft := &fakeT{T: t} + iter := 0 + cleanupsExecuted := 0 + Run( + ft, + func(r *R) { + if cleanupsExecuted != iter { + r.Stop(fmt.Errorf("cleanups not executed between retries")) + return + } + iter += 1 + + r.Cleanup(func() { + cleanupsExecuted += 1 + }) + + r.FailNow() + }, + WithRetryer(&Counter{Count: 3, Wait: time.Millisecond}), + WithImmediateCleanup(), + ) + + require.Equal(t, 3, cleanupsExecuted) + // ensure that r.Stop hadn't been called. If it was then we would + // have log output + require.Len(t, ft.out, 0) + }) + + t.Run("passthrough-to-t", func(t *testing.T) { + cleanupsExecuted := 0 + + require.True(t, t.Run("internal", func(t *testing.T) { + iter := 0 + Run( + t, + func(r *R) { + iter++ + + r.Cleanup(func() { + cleanupsExecuted += 1 + }) + + // fail all but the last one to ensure the right number of cleanups + // are eventually executed + if iter < 3 { + r.FailNow() + } + }, + WithRetryer(&Counter{Count: 3, Wait: time.Millisecond}), + ) + + // at this point nothing should be cleaned up + require.Equal(t, 0, cleanupsExecuted) + })) + + // now since the subtest finished the test cleanup funcs + // should have been executed. + require.Equal(t, 3, cleanupsExecuted) + }) +} + +type fakeT struct { + fails int + out []string + *testing.T +} + +func (f *fakeT) Helper() {} + +func (f *fakeT) Log(args ...interface{}) { + f.out = append(f.out, fmt.Sprint(args...)) +} + +func (f *fakeT) FailNow() { + f.fails++ +} + +var _ TestingTB = &fakeT{} diff --git a/router-tests/freeport/retry/retryer.go b/router-tests/freeport/retry/retryer.go new file mode 100644 index 0000000000..45f41988f2 --- /dev/null +++ b/router-tests/freeport/retry/retryer.go @@ -0,0 +1,36 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package retry + +import "time" + +// Retryer provides an interface for repeating operations +// until they succeed or an exit condition is met. +type Retryer interface { + // Continue returns true if the operation should be repeated, otherwise it + // returns false to indicate retrying should stop. + Continue() bool +} + +// DefaultRetryer provides default retry.Run() behavior for unit tests, namely +// 7s timeout with a wait of 25ms +func DefaultRetryer() Retryer { + return &Timer{Timeout: 7 * time.Second, Wait: 25 * time.Millisecond} +} + +// ThirtySeconds repeats an operation for thirty seconds and waits 500ms in between. +// Best for known slower operations like waiting on eventually consistent state. +func ThirtySeconds() *Timer { + return &Timer{Timeout: 30 * time.Second, Wait: 500 * time.Millisecond} +} + +// TwoSeconds repeats an operation for two seconds and waits 25ms in between. +func TwoSeconds() *Timer { + return &Timer{Timeout: 2 * time.Second, Wait: 25 * time.Millisecond} +} + +// ThreeTimes repeats an operation three times and waits 25ms in between. +func ThreeTimes() *Counter { + return &Counter{Count: 3, Wait: 25 * time.Millisecond} +} diff --git a/router-tests/freeport/retry/run.go b/router-tests/freeport/retry/run.go new file mode 100644 index 0000000000..51dad9f4e4 --- /dev/null +++ b/router-tests/freeport/retry/run.go @@ -0,0 +1,48 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package retry + +type Option func(r *R) + +func WithRetryer(retryer Retryer) Option { + return func(r *R) { + r.retryer = retryer + } +} + +func WithFullOutput() Option { + return func(r *R) { + r.fullOutput = true + } +} + +// WithImmediateCleanup will cause all cleanup operations added +// by calling the Cleanup method on *R to be performed after +// the retry attempt completes (regardless of pass/fail status) +// Use this only if all resources created during the retry loop should +// not persist after the retry has finished. +func WithImmediateCleanup() Option { + return func(r *R) { + r.immediateCleanup = true + } +} + +func Run(t TestingTB, f func(r *R), opts ...Option) { + t.Helper() + r := &R{ + wrapped: t, + retryer: DefaultRetryer(), + } + + for _, opt := range opts { + opt(r) + } + + r.run(f) +} + +func RunWith(r Retryer, t TestingTB, f func(r *R)) { + t.Helper() + Run(t, f, WithRetryer(r)) +} diff --git a/router-tests/freeport/retry/timer.go b/router-tests/freeport/retry/timer.go new file mode 100644 index 0000000000..6c65d38ba0 --- /dev/null +++ b/router-tests/freeport/retry/timer.go @@ -0,0 +1,30 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package retry + +import "time" + +// Timer repeats an operation for a given amount +// of time and waits between subsequent operations. +type Timer struct { + Timeout time.Duration + Wait time.Duration + + // stop is the timeout deadline. + // TODO: Next()? + // Set on the first invocation of Next(). + stop time.Time +} + +func (r *Timer) Continue() bool { + if r.stop.IsZero() { + r.stop = time.Now().Add(r.Timeout) + return true + } + if time.Now().After(r.stop) { + return false + } + time.Sleep(r.Wait) + return true +} diff --git a/router-tests/freeport/systemlimit.go b/router-tests/freeport/systemlimit.go new file mode 100644 index 0000000000..8d71aa8acb --- /dev/null +++ b/router-tests/freeport/systemlimit.go @@ -0,0 +1,14 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build !windows + +package freeport + +import "golang.org/x/sys/unix" + +func systemLimit() (int, error) { + var limit unix.Rlimit + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &limit) + return int(limit.Cur), err +} diff --git a/router-tests/freeport/systemlimit_windows.go b/router-tests/freeport/systemlimit_windows.go new file mode 100644 index 0000000000..a01041ae3d --- /dev/null +++ b/router-tests/freeport/systemlimit_windows.go @@ -0,0 +1,10 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build windows + +package freeport + +func systemLimit() (int, error) { + return 0, nil +} diff --git a/router-tests/go.mod b/router-tests/go.mod index c03ace6fcc..09abf7cf33 100644 --- a/router-tests/go.mod +++ b/router-tests/go.mod @@ -9,7 +9,6 @@ require ( github.com/golang-jwt/jwt/v5 v5.2.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.1 - github.com/hashicorp/consul/sdk v0.16.1 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-retryablehttp v0.7.7 github.com/hasura/go-graphql-client v0.14.3 @@ -36,6 +35,7 @@ require ( go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 golang.org/x/net v0.46.0 + golang.org/x/sys v0.37.0 google.golang.org/grpc v1.71.0 google.golang.org/protobuf v1.36.9 gopkg.in/yaml.v3 v3.0.1 @@ -173,7 +173,6 @@ require ( golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 // indirect golang.org/x/mod v0.29.0 // indirect golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect golang.org/x/text v0.30.0 // indirect golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.38.0 // indirect @@ -211,5 +210,3 @@ replace ( github.com/wundergraph/cosmo/router-plugin => ../router-plugin // github.com/wundergraph/graphql-go-tools/v2 => ../../graphql-go-tools/v2 ) - -replace github.com/hashicorp/consul/sdk => github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 diff --git a/router-tests/go.sum b/router-tests/go.sum index a46cdb7d69..7360bf44c2 100644 --- a/router-tests/go.sum +++ b/router-tests/go.sum @@ -352,8 +352,6 @@ github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/ github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083 h1:8/D7f8gKxTBjW+SZK4mhxTTBVpxcqeBgWF1Rfmltbfk= github.com/wundergraph/astjson v0.0.0-20250106123708-be463c97e083/go.mod h1:eOTL6acwctsN4F3b7YE+eE2t8zcJ/doLm9sZzsxxxrE= -github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301 h1:EzfKHQoTjFDDcgaECCCR2aTePqMu9QBmPbyhqIYOhV0= -github.com/wundergraph/consul/sdk v0.0.0-20250204115147-ed842a8fd301/go.mod h1:wxI0Nak5dI5RvJuzGyiEK4nZj0O9X+Aw6U0tC1wPKq0= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.240 h1:xnYwsUrmDcQnrZQ4+WZ8oODktsKJyAJWiYplWfmiQuk= github.com/wundergraph/graphql-go-tools/v2 v2.0.0-rc.240/go.mod h1:mX25ASEQiKamxaFSK6NZihh0oDCigIuzro30up4mFH4= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= diff --git a/router-tests/jwks/jwks.go b/router-tests/jwks/jwks.go index 9e0cabcc8e..e5b4a9e204 100644 --- a/router-tests/jwks/jwks.go +++ b/router-tests/jwks/jwks.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "log" - "net" "net/http" "net/http/httptest" "testing" @@ -13,7 +12,6 @@ import ( "github.com/MicahParks/jwkset" "github.com/golang-jwt/jwt/v5" - "github.com/hashicorp/consul/sdk/freeport" ) const ( @@ -168,13 +166,6 @@ func NewServerWithCrypto(t *testing.T, providers ...Crypto) (*Server, error) { mux.HandleFunc(oidcHTTPPath, s.oidcJSON) httpServer := httptest.NewUnstartedServer(mux) - port := freeport.GetOne(t) - l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) - if err != nil { - t.Fatalf("could not listen on port: %s", err.Error()) - } - _ = httpServer.Listener.Close() - httpServer.Listener = l httpServer.Start() s.httpServer = httpServer diff --git a/router-tests/lifecycle/shutdown_test.go b/router-tests/lifecycle/shutdown_test.go index be4576e12d..9afc4af194 100644 --- a/router-tests/lifecycle/shutdown_test.go +++ b/router-tests/lifecycle/shutdown_test.go @@ -17,8 +17,10 @@ import ( func TestShutdownGoroutineLeaks(t *testing.T) { defer goleak.VerifyNone(t, - goleak.IgnoreTopFunction("github.com/hashicorp/consul/sdk/freeport.checkFreedPorts"), // Freeport, spawned by init - goleak.IgnoreAnyFunction("net/http.(*conn).serve"), // HTTPTest server I can't close if I want to keep the problematic goroutine open for the test + // Freeport, spawned by init + goleak.IgnoreTopFunction("github.com/wundergraph/cosmo/router-tests/freeport.checkFreedPorts"), + // HTTPTest server I can't close if I want to keep the problematic goroutine open for the test + goleak.IgnoreAnyFunction("net/http.(*conn).serve"), ) xEnv, err := testenv.CreateTestEnv(t, &testenv.Config{ diff --git a/router-tests/lifecycle/supervisor_test.go b/router-tests/lifecycle/supervisor_test.go index c3456da210..4835eb473c 100644 --- a/router-tests/lifecycle/supervisor_test.go +++ b/router-tests/lifecycle/supervisor_test.go @@ -11,8 +11,10 @@ import ( func TestRouterSupervisor(t *testing.T) { defer goleak.VerifyNone(t, - goleak.IgnoreTopFunction("github.com/hashicorp/consul/sdk/freeport.checkFreedPorts"), // Freeport, spawned by init - goleak.IgnoreAnyFunction("net/http.(*conn).serve"), // HTTPTest server I can't close if I want to keep the problematic goroutine open for the test + // Freeport, spawned by init + goleak.IgnoreTopFunction("github.com/wundergraph/cosmo/router-tests/freeport.checkFreedPorts"), + // HTTPTest server I can't close if I want to keep the problematic goroutine open for the test + goleak.IgnoreAnyFunction("net/http.(*conn).serve"), ) xEnv, err := testenv.CreateTestSupervisorEnv(t, &testenv.Config{}) diff --git a/router-tests/router_compatibility_version_config_poller_test.go b/router-tests/router_compatibility_version_config_poller_test.go index fb6bc64025..76a91632a8 100644 --- a/router-tests/router_compatibility_version_config_poller_test.go +++ b/router-tests/router_compatibility_version_config_poller_test.go @@ -3,12 +3,12 @@ package integration import ( "context" "fmt" - "github.com/hashicorp/consul/sdk/freeport" - "github.com/wundergraph/cosmo/router/pkg/routerconfig" - "github.com/wundergraph/cosmo/router/pkg/routerconfig/cdn" "testing" "time" + "github.com/wundergraph/cosmo/router/pkg/routerconfig" + "github.com/wundergraph/cosmo/router/pkg/routerconfig/cdn" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/wundergraph/cosmo/router-tests/testenv" @@ -21,8 +21,7 @@ func TestRouterCompatibilityVersionConfigPoller(t *testing.T) { t.Run("test that a v1 router compatibility version config is requested from the correct cdn path", func(t *testing.T) { t.Parallel() - cdnPort := freeport.GetOne(t) - cdnServer := testenv.SetupCDNServer(t, cdnPort) + cdnServer, cdnPort := testenv.SetupCDNServer(t) token, err := testenv.GenerateVersionedJwtToken() require.NoError(t, err) client, err := cdn.NewClient(fmt.Sprintf("http://127.0.0.1:%d", cdnPort), token, &cdn.Options{ @@ -54,8 +53,7 @@ func TestRouterCompatibilityVersionConfigPoller(t *testing.T) { t.Run("test that a v2 router compatibility version config is requested from the correct cdn path", func(t *testing.T) { t.Parallel() - cdnPort := freeport.GetOne(t) - cdnServer := testenv.SetupCDNServer(t, cdnPort) + cdnServer, cdnPort := testenv.SetupCDNServer(t) token, err := testenv.GenerateVersionedJwtToken() require.NoError(t, err) client, err := cdn.NewClient(fmt.Sprintf("http://127.0.0.1:%d", cdnPort), token, &cdn.Options{ diff --git a/router-tests/testenv/testenv.go b/router-tests/testenv/testenv.go index c3961a3250..2d1e5f64de 100644 --- a/router-tests/testenv/testenv.go +++ b/router-tests/testenv/testenv.go @@ -36,7 +36,6 @@ import ( "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" "github.com/gorilla/websocket" - "github.com/hashicorp/consul/sdk/freeport" "github.com/hashicorp/go-cleanhttp" "github.com/hashicorp/go-retryablehttp" "github.com/nats-io/nats.go" @@ -57,6 +56,7 @@ import ( "github.com/wundergraph/cosmo/demo/pkg/subgraphs" projects "github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects/generated" "github.com/wundergraph/cosmo/demo/pkg/subgraphs/projects/src/service" + "github.com/wundergraph/cosmo/router-tests/freeport" "github.com/wundergraph/cosmo/router/core" nodev1 "github.com/wundergraph/cosmo/router/gen/proto/wg/cosmo/node/v1" "github.com/wundergraph/cosmo/router/pkg/config" @@ -478,10 +478,6 @@ func CreateTestSupervisorEnv(t testing.TB, cfg *Config) (*Environment, error) { Countries: atomic.NewInt64(0), } - requiredPorts := 2 - - ports := freeport.GetN(t, requiredPorts) - getPubSubName := GetPubSubNameFn(pubSubPrefix) ctx, cancel := context.WithCancelCause(context.Background()) @@ -634,15 +630,13 @@ func CreateTestSupervisorEnv(t testing.TB, cfg *Config) (*Environment, error) { cdnServer := cfg.CdnSever if cfg.CdnSever == nil { - cdnServer = SetupCDNServer(t, freeport.GetOne(t)) + cdnServer, _ = SetupCDNServer(t) } if cfg.PrometheusRegistry != nil { - cfg.PrometheusPort = ports[0] + cfg.PrometheusPort = freeport.GetOne(t) } - listenerAddr := fmt.Sprintf("localhost:%d", ports[1]) - client := &http.Client{} if !cfg.NoRetryClient { @@ -658,6 +652,7 @@ func CreateTestSupervisorEnv(t testing.TB, cfg *Config) (*Environment, error) { cfg.MCP.Server.ListenAddr = fmt.Sprintf("localhost:%d", freeport.GetOne(t)) } + listenerAddr := fmt.Sprintf("localhost:%d", freeport.GetOne(t)) baseURL := fmt.Sprintf("http://%s", listenerAddr) rs, err := core.NewRouterSupervisor(&core.RouterSupervisorOpts{ @@ -905,10 +900,6 @@ func CreateTestEnv(t testing.TB, cfg *Config) (*Environment, error) { Countries: atomic.NewInt64(0), } - requiredPorts := 2 - - ports := freeport.GetN(t, requiredPorts) - getPubSubName := GetPubSubNameFn(pubSubPrefix) ctx, cancel := context.WithCancelCause(context.Background()) @@ -1061,15 +1052,13 @@ func CreateTestEnv(t testing.TB, cfg *Config) (*Environment, error) { cdnServer := cfg.CdnSever if cfg.CdnSever == nil { - cdnServer = SetupCDNServer(t, freeport.GetOne(t)) + cdnServer, _ = SetupCDNServer(t) } if cfg.PrometheusRegistry != nil { - cfg.PrometheusPort = ports[0] + cfg.PrometheusPort = freeport.GetOne(t) } - listenerAddr := fmt.Sprintf("localhost:%d", ports[1]) - client := &http.Client{} if !cfg.NoRetryClient { @@ -1085,6 +1074,7 @@ func CreateTestEnv(t testing.TB, cfg *Config) (*Environment, error) { cfg.MCP.Server.ListenAddr = fmt.Sprintf("localhost:%d", freeport.GetOne(t)) } + listenerAddr := fmt.Sprintf("localhost:%d", freeport.GetOne(t)) rr, err := configureRouter(listenerAddr, cfg, &routerConfig, cdnServer, natsSetup) if err != nil { cancel(err) @@ -1652,42 +1642,24 @@ func testVersionedTokenClaims() jwt.MapClaims { } } -func makeHttpTestServerWithPort(t testing.TB, handler http.Handler, port int) *httptest.Server { - s := httptest.NewUnstartedServer(handler) - l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) - if err != nil { - t.Fatalf("could not listen on port: %s", err.Error()) - } - _ = s.Listener.Close() - s.Listener = l - s.Start() - - return s -} - func makeSafeHttpTestServer(t testing.TB, handler http.Handler) *httptest.Server { + // NewUnstartedServer binds an ephemeral port. + // We want to avoid using freeport because it creates too much strain on the network stack: + // freeport checks if port is available by listening on it and then closing the listener. + // On Linux trying to listen on the just-closed port could lead to the "unable to bind" error. s := httptest.NewUnstartedServer(handler) - port := freeport.GetOne(t) - l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) - if err != nil { - t.Fatalf("could not listen on port: %s", err.Error()) - } - _ = s.Listener.Close() - s.Listener = l s.Start() - return s } func makeSafeGRPCServer(t testing.TB, sd *grpc.ServiceDesc, service any) (*grpc.Server, string) { t.Helper() - port := freeport.GetOne(t) - - endpoint := fmt.Sprintf("localhost:%d", port) - - lis, err := net.Listen("tcp", endpoint) + // We could use freeport here, but it is easy to use ephemeral port and get the endpoint + // easily. + lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) + endpoint := lis.Addr().String() require.NotNil(t, service) @@ -1695,20 +1667,21 @@ func makeSafeGRPCServer(t testing.TB, sd *grpc.ServiceDesc, service any) (*grpc. s.RegisterService(sd, service) go func() { - err := s.Serve(lis) - require.NoError(t, err) + if err := s.Serve(lis); err != nil && !errors.Is(err, grpc.ErrServerStopped) { + t.Errorf("gRPC test server Serve error: %v", err) + } }() return s, endpoint } -func SetupCDNServer(t testing.TB, port int) *httptest.Server { +func SetupCDNServer(t testing.TB) (cdnServer *httptest.Server, port int) { _, filePath, _, ok := runtime.Caller(0) require.True(t, ok) baseCdnFile := filepath.Join(path.Dir(filePath), "testdata", "cdn") cdnFileServer := http.FileServer(http.Dir(baseCdnFile)) var cdnRequestLog []string - cdnServer := makeHttpTestServerWithPort(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/" { requestLog, err := json.Marshal(cdnRequestLog) require.NoError(t, err) @@ -1729,9 +1702,10 @@ func SetupCDNServer(t testing.TB, port int) *httptest.Server { _, _, err := jwtParser.ParseUnverified(token, parsedClaims) require.NoError(t, err) cdnFileServer.ServeHTTP(w, r) - }), port) - - return cdnServer + }) + cdnServer = httptest.NewServer(handler) + port = cdnServer.Listener.Addr().(*net.TCPAddr).Port + return cdnServer, port } func gqlURL(srv *httptest.Server) string { diff --git a/router-tests/testenv/testexec.go b/router-tests/testenv/testexec.go index a02ccfbec2..4c391fb810 100644 --- a/router-tests/testenv/testexec.go +++ b/router-tests/testenv/testexec.go @@ -14,9 +14,10 @@ import ( "testing" "time" - "github.com/hashicorp/consul/sdk/freeport" "github.com/stretchr/testify/require" "go.uber.org/atomic" + + "github.com/wundergraph/cosmo/router-tests/freeport" ) const routerDir = "../router" @@ -118,13 +119,12 @@ func runRouterBin(t *testing.T, ctx context.Context, opts RunRouterBinConfigOpti return nil, err } - port := freeport.GetOne(t) - listenerAddr := fmt.Sprintf("localhost:%d", port) + listenerAddr := fmt.Sprintf("localhost:%d", freeport.GetOne(t)) token, err := generateJwtToken() if err != nil { return nil, err } - testCdn := SetupCDNServer(t, freeport.GetOne(t)) + testCdn, _ := SetupCDNServer(t) var envs []string envVars := map[string]string{