feat: implement simulated top req res middleware

packages/proxy/lib/http/index.ts

@@ -5,6 +5,8 @@ import type {
   CypressIncomingRequest,
   CypressOutgoingResponse,
   BrowserPreRequest,
+  AppliedCredentialByUrlAndResourceMap,
+  GetCredentialLevelOfRequest,
 } from '@packages/proxy'
 import Debug from 'debug'
 import chalk from 'chalk'
@@ -73,6 +75,8 @@ export type ServerCtx = Readonly<{
   getFileServerToken: () => string
   getCookieJar: () => CookieJar
   remoteStates: RemoteStates
+  appliedCredentialByUrlAndResourceMap: AppliedCredentialByUrlAndResourceMap
+  getCredentialLevelOfRequest: GetCredentialLevelOfRequest
   getRenderedHTMLOrigins: Http['getRenderedHTMLOrigins']
   netStubbingState: NetStubbingState
   middleware: HttpMiddlewareStacks
@@ -222,6 +226,8 @@ export class Http {
   request: any
   socket: CyServer.Socket
   serverBus: EventEmitter
+  appliedCredentialByUrlAndResourceMap: AppliedCredentialByUrlAndResourceMap
+  getCredentialLevelOfRequest: GetCredentialLevelOfRequest
   renderedHTMLOrigins: {[key: string]: boolean} = {}
   autUrl?: string
   getCookieJar: () => CookieJar
@@ -240,6 +246,8 @@ export class Http {
     this.socket = opts.socket
     this.request = opts.request
     this.serverBus = opts.serverBus
+    this.appliedCredentialByUrlAndResourceMap = opts.appliedCredentialByUrlAndResourceMap
+    this.getCredentialLevelOfRequest = opts.getCredentialLevelOfRequest
     this.getCookieJar = opts.getCookieJar
 
     if (typeof opts.middleware === 'undefined') {
@@ -267,6 +275,8 @@ export class Http {
       netStubbingState: this.netStubbingState,
       socket: this.socket,
       serverBus: this.serverBus,
+      appliedCredentialByUrlAndResourceMap: this.appliedCredentialByUrlAndResourceMap,
+      getCredentialLevelOfRequest: this.getCredentialLevelOfRequest,
       getCookieJar: this.getCookieJar,
       debug: (formatter, ...args) => {
         if (!debugVerbose.enabled) return

packages/proxy/lib/http/request-middleware.ts

@@ -2,7 +2,8 @@ import _ from 'lodash'
 import { blocked, cors } from '@packages/network'
 import { InterceptRequest } from '@packages/net-stubbing'
 import type { HttpMiddleware } from './'
-import { getSameSiteContext, addCookieJarCookiesToRequest } from './util/cookies'
+import { getSameSiteContext, addCookieJarCookiesToRequest, shouldAttachAndSetCookies } from './util/cookies'
+import { doesTopNeedToBeSimulated } from './util/top-simulation'
 
 // do not use a debug namespace in this file - use the per-request `this.debug` instead
 // available as cypress-verbose:proxy:http
@@ -23,6 +24,7 @@ const LogRequest: RequestMiddleware = function () {
 
 const ExtractCypressMetadataHeaders: RequestMiddleware = function () {
   this.req.isAUTFrame = !!this.req.headers['x-cypress-is-aut-frame']
+  const requestIsXhrOrFetch = this.req.headers['x-cypress-request']
 
   if (this.req.headers['x-cypress-is-aut-frame']) {
     delete this.req.headers['x-cypress-is-aut-frame']
@@ -33,6 +35,23 @@ const ExtractCypressMetadataHeaders: RequestMiddleware = function () {
     delete this.req.headers['x-cypress-request']
   }
 
+  if (!this.config.experimentalSessionAndOrigin ||
+    !doesTopNeedToBeSimulated(this) ||
+    // this should be unreachable, as the x-cypress-request header is only attached if the resource type is 'xhr'
+    // inside the extension or electron equivalent. This is only needed for defensive purposes.
+    (requestIsXhrOrFetch !== 'true' && requestIsXhrOrFetch !== 'xhr' && requestIsXhrOrFetch !== 'fetch')) {
+    this.next()
+
+    return
+  }
+
+  this.debug(`looking up credentials for ${this.req.proxiedUrl}`)
+  let { resourceType, credentialStatus } = this.getCredentialLevelOfRequest(this.req.proxiedUrl, requestIsXhrOrFetch !== 'true' ? requestIsXhrOrFetch : undefined)
+
+  this.debug(`credentials calculated for ${resourceType}:${credentialStatus}`)
+
+  this.req.requestedWith = resourceType
+  this.req.credentialsLevel = credentialStatus
   this.next()
 }
 
@@ -52,9 +71,16 @@ const MaybeSimulateSecHeaders: RequestMiddleware = function () {
 }
 
 const MaybeAttachCrossOriginCookies: RequestMiddleware = function () {
+  if (!this.config.experimentalSessionAndOrigin || !doesTopNeedToBeSimulated(this)) {
+    return this.next()
+  }
+
+  // Top needs to be simulated since the AUT is in a cross origin state. Get the requestedWith and credentials and see what cookies need to be attached
   const currentAUTUrl = this.getAUTUrl()
+  const shouldCookiesBeAttachedToRequest = shouldAttachAndSetCookies(this.req.proxiedUrl, currentAUTUrl, this.req.requestedWith, this.req.credentialsLevel, this.req.isAUTFrame)
 
-  if (!this.config.experimentalSessionAndOrigin || !currentAUTUrl) {
+  this.debug(`should cookies be attached to request?: ${shouldCookiesBeAttachedToRequest}`)
+  if (!shouldCookiesBeAttachedToRequest) {
     return this.next()
   }
 
@@ -70,7 +96,8 @@ const MaybeAttachCrossOriginCookies: RequestMiddleware = function () {
   this.debug('existing cookies on request from cookie jar: %s', applicableCookiesInCookieJar.join('; '))
   this.debug('add cookies to request from header: %s', cookiesOnRequest.join('; '))
 
-  this.req.headers['cookie'] = addCookieJarCookiesToRequest(applicableCookiesInCookieJar, cookiesOnRequest)
+  // if the cookie header is empty (i.e. ''), set it to undefined for expected behavior
+  this.req.headers['cookie'] = addCookieJarCookiesToRequest(applicableCookiesInCookieJar, cookiesOnRequest) || undefined
 
   this.debug('cookies being sent with request: %s', this.req.headers['cookie'])
   this.next()

packages/proxy/lib/http/response-middleware.ts

@@ -4,7 +4,7 @@ import type Debug from 'debug'
 import type { CookieOptions } from 'express'
 import { cors, concatStream, httpUtils } from '@packages/network'
 import type { CypressIncomingRequest, CypressOutgoingResponse } from '@packages/proxy'
-import type { HttpMiddleware, HttpMiddlewareThis } from '.'
+import type { HttpMiddleware } from '.'
 import iconv from 'iconv-lite'
 import type { IncomingMessage, IncomingHttpHeaders } from 'http'
 import { InterceptResponse } from '@packages/net-stubbing'
@@ -13,6 +13,7 @@ import * as rewriter from './util/rewriter'
 import zlib from 'zlib'
 import { URL } from 'url'
 import { CookiesHelper } from './util/cookies'
+import { doesTopNeedToBeSimulated } from './util/top-simulation'
 
 interface ResponseMiddlewareProps {
   /**
@@ -370,48 +371,33 @@ const MaybePreventCaching: ResponseMiddleware = function () {
   this.next()
 }
 
-const checkIfNeedsCrossOriginHandling = (ctx: HttpMiddlewareThis<ResponseMiddlewareProps>) => {
-  const currentAUTUrl = ctx.getAUTUrl()
-
-  // A cookie needs cross origin handling if the request itself is
-  // cross-origin or the origins between requests don't match,
-  // since the browser won't set them in that case and if it's
-  // secondary-origin -> primary-origin, we don't recognize the request as cross-origin
-  return (
-    ctx.config.experimentalSessionAndOrigin
-    && (
-      (currentAUTUrl && !cors.urlOriginsMatch(currentAUTUrl, ctx.req.proxiedUrl))
-      || !ctx.remoteStates.isPrimaryOrigin(ctx.req.proxiedUrl)
-    )
-  )
-}
-
-const CopyCookiesFromIncomingRes: ResponseMiddleware = async function () {
+const MaybeCopyCookiesFromIncomingRes: ResponseMiddleware = async function () {
   const cookies: string | string[] | undefined = this.incomingRes.headers['set-cookie']
 
   if (!cookies || !cookies.length) {
     return this.next()
   }
 
-  // Cross-origin Cookie Handling
+  // Simulated Top Cookie Handling
   // ---------------------------
   // - We capture cookies sent by responses and add them to our own server-side
   //   tough-cookie cookie jar. All request cookies are captured, since any
-  //   future request could be cross-origin even if the response that sets them
+  //   future request could be cross-origin in the context of top, even if the response that sets them
   //   is not.
   // - If we sent the cookie header, it may fail to be set by the browser
   //   (in most cases). However, we cannot determine all the cases in which Set-Cookie
-  //   will currently fail, and currently is best to set optimistically until #23551 is addressed.
+  //   will currently fail. We try to address this in our tough cookie jar
+  //   by only setting cookies that would otherwise work in the browser if the AUT url was top
   // - We also set the cookies through automation so they are available in the
   //   browser via document.cookie and via Cypress cookie APIs
-  //   (e.g. cy.getCookie). This is only done for cross-origin responses, since
-  //   non-cross-origin responses will be successfully set in the browser
-  //   automatically.
+  //   (e.g. cy.getCookie). This is only done when the AUT url and top do not match responses,
+  //   since AUT and Top being same origin will be successfully set in the browser
+  //   automatically as expected.
   // - In the request middleware, we retrieve the cookies for a given URL
   //   and attach them to the request, like the browser normally would.
   //   tough-cookie handles retrieving the correct cookies based on domain,
   //   path, etc. It also removes cookies from the cookie jar if they've expired.
-  const needsCrossOriginHandling = checkIfNeedsCrossOriginHandling(this)
+  const doesTopNeedSimulating = doesTopNeedToBeSimulated(this)
 
   const appendCookie = (cookie: string) => {
     // always call 'Set-Cookie' in the browser as cross origin or same site requests
@@ -425,7 +411,7 @@ const CopyCookiesFromIncomingRes: ResponseMiddleware = async function () {
     }
   }
 
-  if (!this.config.experimentalSessionAndOrigin) {
+  if (!this.config.experimentalSessionAndOrigin || !doesTopNeedSimulating) {
     ([] as string[]).concat(cookies).forEach((cookie) => {
       appendCookie(cookie)
     })
@@ -440,7 +426,9 @@ const CopyCookiesFromIncomingRes: ResponseMiddleware = async function () {
     request: {
       url: this.req.proxiedUrl,
       isAUTFrame: this.req.isAUTFrame,
-      needsCrossOriginHandling,
+      doesTopNeedSimulating,
+      resourceType: this.req.requestedWith,
+      credentialLevel: this.req.credentialsLevel,
     },
   })
 
@@ -454,7 +442,7 @@ const CopyCookiesFromIncomingRes: ResponseMiddleware = async function () {
 
   const addedCookies = await cookiesHelper.getAddedCookies()
 
-  if (!needsCrossOriginHandling || !addedCookies.length) {
+  if (!addedCookies.length) {
     return this.next()
   }
 
@@ -599,7 +587,7 @@ export default {
   OmitProblematicHeaders,
   MaybePreventCaching,
   MaybeStripDocumentDomainFeaturePolicy,
-  CopyCookiesFromIncomingRes,
+  MaybeCopyCookiesFromIncomingRes,
   MaybeSendRedirectToClient,
   CopyResponseStatusCode,
   ClearCyInitialCookie,

packages/proxy/lib/http/util/cookies.ts

@@ -11,7 +11,9 @@ type SiteContext = 'same-origin' | 'same-site' | 'cross-site'
 interface RequestDetails {
   url: string
   isAUTFrame: boolean
-  needsCrossOriginHandling: boolean
+  doesTopNeedSimulating: boolean
+  resourceType?: RequestResourceType
+  credentialLevel?: RequestCredentialLevel
 }
 
 /**
@@ -162,6 +164,7 @@ export class CookiesHelper {
   debug: Debug.Debugger
   defaultDomain: string
   sameSiteContext: 'strict' | 'lax' | 'none'
+  siteContext: SiteContext
   previousCookies: Cookie[] = []
 
   constructor ({ cookieJar, currentAUTUrl, request, debug }) {
@@ -170,6 +173,7 @@ export class CookiesHelper {
     this.request = request
     this.debug = debug
     this.sameSiteContext = getSameSiteContext(currentAUTUrl, request.url, request.isAUTFrame)
+    this.siteContext = calculateSiteContext(this.request.url, this.currentAUTUrl || '')
 
     const parsedRequestUrl = new URL(request.url)
 
@@ -180,7 +184,7 @@ export class CookiesHelper {
     // this plays a part in adding cross-origin cookies to the browser via
     // automation. if the request doesn't need cross-origin handling, this
     // is a noop
-    if (!this.request.needsCrossOriginHandling) return
+    if (!this.request.doesTopNeedSimulating) return
 
     this.previousCookies = this.cookieJar.getAllCookies()
   }
@@ -189,7 +193,7 @@ export class CookiesHelper {
     // this plays a part in adding cross-origin cookies to the browser via
     // automation. if the request doesn't need cross-origin handling, this
     // is a noop
-    if (!this.request.needsCrossOriginHandling) return []
+    if (!this.request.doesTopNeedSimulating) return []
 
     const afterCookies = this.cookieJar.getAllCookies()
 
@@ -208,10 +212,28 @@ export class CookiesHelper {
     // because Secure is required for SameSite=None. not all browsers currently
     // currently enforce this, but they're heading in that direction since
     // it's now the standard, so this is more future-proof
+    // TODO: in the future we may want to check for https, which might be tricky since localhost is considered a secure context
     if (!toughCookie || (toughCookie.sameSite === 'none' && !toughCookie.secure)) {
       return
     }
 
+    // cross site cookies cannot set lax/strict cookies in the browser for xhr/fetch requests (but ok with navigation/document requests)
+    if (this.request.resourceType && this.siteContext === 'cross-site' && toughCookie.sameSite !== 'none') {
+      this.debug(`cannot set cookie with SameSite=${toughCookie.sameSite} when site context is ${this.siteContext}`)
+
+      return
+    }
+
+    // don't set the cookie in our own cookie jar if the cookie would otherwise fail being set in the browser if the AUT Url
+    // was actually top. This prevents cookies from being applied to our cookie jar when they shouldn't, preventing possible security implications.
+    const shouldSetCookieGivenSiteContext = shouldAttachAndSetCookies(this.request.url, this.currentAUTUrl, this.request.resourceType, this.request.credentialLevel, this.request.isAUTFrame)
+
+    if (!shouldSetCookieGivenSiteContext) {
+      this.debug(`not setting cookie for ${this.request.url} with simulated top ${ this.currentAUTUrl} for ${ this.request.resourceType}:${this.request.credentialLevel}, cookie: ${toughCookie}`)
+
+      return
+    }
+
     try {
       this.cookieJar.setCookie(toughCookie, this.request.url, this.sameSiteContext)
     } catch (err) {

packages/proxy/lib/types.ts

@@ -13,6 +13,8 @@ export type CypressIncomingRequest = Request & {
   responseTimeout?: number
   followRedirect?: boolean
   isAUTFrame: boolean
+  requestedWith?: RequestResourceType
+  credentialsLevel?: RequestCredentialLevel
 }
 
 export type RequestResourceType = 'fetch' | 'xhr'
@@ -21,6 +23,16 @@ export type RequestCredentialLevel = 'same-origin' | 'include' | 'omit' | boolea
 
 export type CypressWantsInjection = 'full' | 'fullCrossOrigin' | 'partial' | false
 
+export type AppliedCredentialByUrlAndResourceMap = Map<string, Array<{
+  resourceType: RequestResourceType
+  credentialStatus: RequestCredentialLevel
+}>>
+
+export type GetCredentialLevelOfRequest = (url: string, optionalResourceType?: RequestResourceType) => {
+  resourceType: RequestResourceType
+  credentialStatus: RequestCredentialLevel
+}
+
 /**
  * An outgoing response to an incoming request to the Cypress web server.
  */