Drtii 1691 use live view data for api v1 requests

build.sbt

@@ -1,6 +1,6 @@
 import sbt.Keys.resolvers
 
-lazy val drtLibVersion = "v972"
+lazy val drtLibVersion = "v976"
 lazy val drtCiriumVersion = "203"
 lazy val akkaHttpVersion = "10.5.3"
 lazy val akkaVersion = "2.8.5"

src/main/scala/uk/gov/homeoffice/drt/Server.scala

@@ -1,5 +1,6 @@
 package uk.gov.homeoffice.drt
 
+import akka.NotUsed
 import akka.actor.Cancellable
 import akka.actor.typed.scaladsl.AskPattern.{Askable, schedulerFromActorSystem}
 import akka.actor.typed.scaladsl.{ActorContext, Behaviors}
@@ -11,21 +12,25 @@ import akka.http.scaladsl.server.Directives.{concat, getFromResource, pathPrefix
 import akka.http.scaladsl.server.Route
 import akka.http.scaladsl.settings.ConnectionPoolSettings
 import akka.stream.Materializer
+import akka.stream.scaladsl.Source
 import akka.util.Timeout
 import org.slf4j.LoggerFactory
+import uk.gov.homeoffice.drt.arrivals.ApiFlightWithSplits
 import uk.gov.homeoffice.drt.db._
-import uk.gov.homeoffice.drt.db.dao.UserFeedbackDao
+import uk.gov.homeoffice.drt.db.dao.{FlightDao, QueueSlotDao, UserFeedbackDao}
 import uk.gov.homeoffice.drt.healthchecks._
 import uk.gov.homeoffice.drt.keycloak.KeyCloakAuth
+import uk.gov.homeoffice.drt.model.CrunchMinute
 import uk.gov.homeoffice.drt.notifications._
 import uk.gov.homeoffice.drt.persistence.{ExportPersistenceImpl, ScheduledHealthCheckPausePersistenceImpl}
 import uk.gov.homeoffice.drt.ports.Terminals.Terminal
-import uk.gov.homeoffice.drt.ports.{PortCode, PortRegion}
+import uk.gov.homeoffice.drt.ports._
 import uk.gov.homeoffice.drt.routes._
 import uk.gov.homeoffice.drt.routes.api.v1.{AuthApiV1Routes, FlightApiV1Routes, QueueApiV1Routes}
+import uk.gov.homeoffice.drt.services.api.v1.{FlightExport, QueueExport}
 import uk.gov.homeoffice.drt.services.s3.S3Service
 import uk.gov.homeoffice.drt.services.{PassengerSummaryStreams, UserRequestService, UserService}
-import uk.gov.homeoffice.drt.time.SDate
+import uk.gov.homeoffice.drt.time.{LocalDate, SDate, UtcDate}
 import uk.gov.homeoffice.drt.uploadTraining.FeatureGuideService
 
 import scala.concurrent.duration.DurationInt
@@ -88,6 +93,26 @@ object Server {
     ArrivalLandingTimesHealthCheck(windowLength = 2.hours, buffer = 20, minimumFlights = 3, passThresholdPercentage = 50, SDate.now),
   )
 
+  private val nonMlPaxPorts = Set("ABZ", "EXT", "HUY", "INV", "LHR", "MME", "NQY", "NWI", "PIK", "SEN")
+
+  val paxFeedSourceOrder: PortCode => List[FeedSource] =
+    portCode => if (!nonMlPaxPorts.contains(portCode.iata)) List(
+      ScenarioSimulationSource,
+      LiveFeedSource,
+      ApiFeedSource,
+      MlFeedSource,
+      ForecastFeedSource,
+      HistoricApiFeedSource,
+      AclFeedSource,
+    ) else List(
+      ScenarioSimulationSource,
+      LiveFeedSource,
+      ApiFeedSource,
+      ForecastFeedSource,
+      HistoricApiFeedSource,
+      AclFeedSource,
+    )
+
   def apply(config: ServerConfig,
             notifications: EmailNotifications,
             emailClient: EmailClient,
@@ -101,6 +126,7 @@ object Server {
 
       val now = () => SDate.now()
 
+      val defaultQueueSlotMinutes = 15
       val urls = Urls(config.rootDomain, config.useHttps)
       val userRequestService = UserRequestService(UserAccessRequestDao(ProdDatabase.db))
       val userService = UserService(UserDao(ProdDatabase.db))
@@ -128,13 +154,28 @@ object Server {
 
       val keyCloakAuth = KeyCloakAuth(config.keycloakTokenUrl, config.keycloakClientId, config.keycloakClientSecret, sendHttpRequest)
 
+      val queuesForPortAndDatesAndSlotSize: (PortCode, Terminal, LocalDate, LocalDate) => Source[CrunchMinute, NotUsed] = {
+        (port, terminal, start, end) =>
+          QueueSlotDao()
+            .queueSlotsForDateRange(port, defaultQueueSlotMinutes, db.run)(start, end, Seq(terminal))
+            .map(_._2)
+            .mapConcat(identity)
+      }
+
+      val flightsForDatesAndTerminals: (PortCode, List[FeedSource], LocalDate, LocalDate, Seq[Terminal]) => Source[ApiFlightWithSplits, NotUsed] =
+        (portCode, sourceOrder, start, end, terminals) =>
+          FlightDao()
+            .flightsForPcpDateRange(portCode, sourceOrder, db.run)(start, end, terminals)
+            .map(_._2)
+            .mapConcat(identity)
+
       val routes: Route = concat(
         pathPrefix("api") {
           concat(
             pathPrefix("v1") {
               concat(
-                QueueApiV1Routes(httpClient, config.enabledPorts),
-                FlightApiV1Routes(httpClient, config.enabledPorts),
+                QueueApiV1Routes(config.enabledPorts, QueueExport.queues(queuesForPortAndDatesAndSlotSize)),
+                FlightApiV1Routes(config.enabledPorts, FlightExport.flights(flightsForDatesAndTerminals)),
                 AuthApiV1Routes(keyCloakAuth.getToken),
               )
             },

src/main/scala/uk/gov/homeoffice/drt/json/ExportJsonFormats.scala

@@ -3,7 +3,7 @@ package uk.gov.homeoffice.drt.json
 import spray.json.{DefaultJsonProtocol, JsString, JsValue, RootJsonFormat}
 import uk.gov.homeoffice.drt.exports._
 import uk.gov.homeoffice.drt.json.LocalDateJsonFormats.LocalDateJsonFormat
-import uk.gov.homeoffice.drt.json.SDateLikeJsonFormats.SDateLikeJsonFormat
+import uk.gov.homeoffice.drt.json.SDateLikeJsonFormats.SDateLikeTimestampJsonFormat
 import uk.gov.homeoffice.drt.models.Export
 import uk.gov.homeoffice.drt.routes.ExportRoutes.ExportRequest
 

src/main/scala/uk/gov/homeoffice/drt/json/SDateLikeJsonFormats.scala

@@ -1,11 +1,11 @@
 package uk.gov.homeoffice.drt.json
 
-import spray.json.{DefaultJsonProtocol, JsNumber, JsValue, RootJsonFormat, enrichAny}
+import spray.json.{DefaultJsonProtocol, JsNumber, JsString, JsValue, RootJsonFormat, enrichAny}
 import uk.gov.homeoffice.drt.time.{SDate, SDateLike}
 
 object SDateLikeJsonFormats extends DefaultJsonProtocol {
 
-  implicit object SDateLikeJsonFormat extends RootJsonFormat[SDateLike] {
+  implicit object SDateLikeTimestampJsonFormat extends RootJsonFormat[SDateLike] {
     override def read(json: JsValue): SDateLike = json match {
       case JsNumber(ts) => SDate(ts.toLong)
     }

src/main/scala/uk/gov/homeoffice/drt/routes/api/v1/AuthApiV1Routes.scala

@@ -14,14 +14,6 @@ import scala.util.{Failure, Success}
 object AuthApiV1Routes extends db.UserAccessRequestJsonSupport with KeyCloakAuthTokenParserProtocol {
   val log: Logger = LoggerFactory.getLogger(getClass)
 
-  trait JsonResponse {
-    def startTime: String
-
-    def endTime: String
-
-    def ports: Seq[String]
-  }
-
   case class Credentials(username: String, password: String)
 
   implicit val credentialsJsonFormat: RootJsonFormat[Credentials] = jsonFormat2(Credentials)

src/main/scala/uk/gov/homeoffice/drt/routes/api/v1/FlightApiV1Routes.scala

@@ -1,49 +1,48 @@
 package uk.gov.homeoffice.drt.routes.api.v1
 
+import akka.http.scaladsl.model.StatusCodes.InternalServerError
 import akka.http.scaladsl.server.Directives._
 import akka.http.scaladsl.server.Route
-import akka.stream.Materializer
+import org.slf4j.LoggerFactory
 import spray.json._
 import uk.gov.homeoffice.drt.auth.Roles.ApiFlightAccess
+import uk.gov.homeoffice.drt.authentication.User
 import uk.gov.homeoffice.drt.ports.PortCode
-import uk.gov.homeoffice.drt.routes.api.v1.AuthApiV1Routes.JsonResponse
 import uk.gov.homeoffice.drt.routes.services.AuthByRole
-import uk.gov.homeoffice.drt.{Dashboard, HttpClient}
+import uk.gov.homeoffice.drt.services.api.v1.FlightExport.PortFlightsJson
+import uk.gov.homeoffice.drt.services.api.v1.serialiser.FlightApiV1JsonFormats
+import uk.gov.homeoffice.drt.time.{SDate, SDateLike}
 
-import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+import scala.util.{Failure, Success}
 
 
-trait FlightApiV1JsonFormats extends DefaultJsonProtocol {
-  implicit object jsonResponseFormat extends RootJsonFormat[JsonResponse] {
+object FlightApiV1Routes extends DefaultJsonProtocol with FlightApiV1JsonFormats {
+  private val log = LoggerFactory.getLogger(getClass)
 
-    override def write(obj: JsonResponse): JsValue = JsObject(Map(
-      "startTime" -> obj.startTime.toJson,
-      "endTime" -> obj.endTime.toJson,
-      "ports" -> JsArray(obj.ports.map(_.parseJson).toVector),
-    ))
+  case class FlightJsonResponse(startTime: SDateLike, endTime: SDateLike, ports: Seq[PortFlightsJson])
 
-    override def read(json: JsValue): JsonResponse = throw new Exception("Not implemented")
-  }
-}
-
-object FlightApiV1Routes extends DefaultJsonProtocol with ApiV1Routes with FlightApiV1JsonFormats {
-
-  case class FlightJsonResponse(startTime: String, endTime: String, ports: Seq[String]) extends JsonResponse
-
-  def apply(httpClient: HttpClient, enabledPorts: Iterable[PortCode])
-           (implicit ec: ExecutionContext, mat: Materializer): Route =
+  def apply(enabledPorts: Iterable[PortCode],
+            dateRangeJsonForPorts: Seq[PortCode] => (SDateLike, SDateLike) => Future[FlightJsonResponse]): Route =
     AuthByRole(ApiFlightAccess) {
       (get & path("flights")) {
         pathEnd(
           headerValueByName("X-Forwarded-Email") { email =>
             headerValueByName("X-Forwarded-Groups") { groups =>
               parameters("start", "end") { (startStr, endStr) =>
-                val portUri: PortCode => String =
-                  portCode => s"${Dashboard.drtInternalUriForPortCode(portCode)}/api/v1/flights?start=$startStr&end=$endStr"
-                val jsonResponse: (String, String, Seq[String]) => JsonResponse =
-                  (startTime, endTime, ports) => FlightJsonResponse(startTime, endTime, ports)
-
-                multiPortResponse(httpClient, enabledPorts, email, groups, portUri, jsonResponse, startStr, endStr)
+                val user = User.fromRoles(email, groups)
+                val ports = enabledPorts.filter(user.accessiblePorts.contains(_)).toList
+                val dateRangeJson = dateRangeJsonForPorts(ports)
+
+                val start = SDate(startStr)
+                val end = SDate(endStr)
+
+                onComplete(dateRangeJson(start, end)) {
+                  case Success(value) => complete(value.toJson.compactPrint)
+                  case Failure(t) =>
+                    log.error(s"Failed to get export: ${t.getMessage}")
+                    complete(InternalServerError)
+                }
               }
             }
           }

src/main/scala/uk/gov/homeoffice/drt/routes/api/v1/QueueApiV1Routes.scala

@@ -1,58 +1,54 @@
 package uk.gov.homeoffice.drt.routes.api.v1
 
+import akka.http.scaladsl.model.StatusCodes.InternalServerError
 import akka.http.scaladsl.server.Directives._
 import akka.http.scaladsl.server.Route
-import akka.stream.Materializer
+import org.slf4j.LoggerFactory
 import spray.json._
 import uk.gov.homeoffice.drt.auth.Roles.ApiQueueAccess
+import uk.gov.homeoffice.drt.authentication.User
 import uk.gov.homeoffice.drt.ports.PortCode
-import uk.gov.homeoffice.drt.routes.api.v1.AuthApiV1Routes.JsonResponse
-import uk.gov.homeoffice.drt.routes.api.v1.QueueApiV1Routes.QueueJsonResponse
 import uk.gov.homeoffice.drt.routes.services.AuthByRole
-import uk.gov.homeoffice.drt.{Dashboard, HttpClient}
+import uk.gov.homeoffice.drt.services.api.v1.QueueExport.PortQueuesJson
+import uk.gov.homeoffice.drt.services.api.v1.serialiser.QueueApiV1JsonFormats
+import uk.gov.homeoffice.drt.time.{SDate, SDateLike}
 
-import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+import scala.util.{Failure, Success}
 
-trait QueueApiV1JsonFormats extends DefaultJsonProtocol {
-  implicit object jsonResponseFormat extends RootJsonFormat[JsonResponse] {
 
-    override def write(obj: JsonResponse): JsValue = obj match {
-      case obj: QueueJsonResponse => JsObject(Map(
-        "startTime" -> obj.startTime.toJson,
-        "endTime" -> obj.endTime.toJson,
-        "periodLengthMinutes" -> obj.slotSizeMinutes.toJson,
-        "ports" -> JsArray(obj.ports.map(_.parseJson).toVector),
-      ))
-    }
-
-    override def read(json: JsValue): JsonResponse = throw new Exception("Not implemented")
-  }
-}
+object QueueApiV1Routes extends DefaultJsonProtocol with QueueApiV1JsonFormats {
+  private val log = LoggerFactory.getLogger(getClass)
 
-object QueueApiV1Routes extends DefaultJsonProtocol with ApiV1Routes with QueueApiV1JsonFormats {
-  case class QueueJsonResponse(startTime: String, endTime: String, slotSizeMinutes: Int, ports: Seq[String]) extends JsonResponse
+  case class QueueJsonResponse(startTime: SDateLike, endTime: SDateLike, slotSizeMinutes: Int, ports: Seq[PortQueuesJson])
 
-  def apply(httpClient: HttpClient, enabledPorts: Iterable[PortCode])
-           (implicit ec: ExecutionContext, mat: Materializer): Route =
+  def apply(enabledPorts: Iterable[PortCode],
+            dateRangeJsonForPortsAndSlotSize: (Seq[PortCode], Int) => (SDateLike, SDateLike) => Future[QueueJsonResponse]): Route =
     AuthByRole(ApiQueueAccess) {
       (get & path("queues")) {
-        pathEnd {
+        pathEnd(
           headerValueByName("X-Forwarded-Email") { email =>
             headerValueByName("X-Forwarded-Groups") { groups =>
-              val defaultSlotSizeMinutes = 15
-
-              parameters("start", "end", "slot-size-minutes".as[Int].withDefault(defaultSlotSizeMinutes)) { (startStr, endStr, slotSizeMinutes) =>
-                val portUri: PortCode => String =
-                  portCode => s"${Dashboard.drtInternalUriForPortCode(portCode)}/api/v1/queues?start=$startStr&end=$endStr&period-minutes=$slotSizeMinutes"
-
-                val jsonResponse: (String, String, Seq[String]) => QueueJsonResponse =
-                  (startTime, endTime, ports) => QueueJsonResponse(startTime, endTime, slotSizeMinutes, ports)
-
-                multiPortResponse(httpClient, enabledPorts, email, groups, portUri, jsonResponse, startStr, endStr)
+              parameters("start", "end", "period-minutes".optional) { (startStr, endStr, maybePeriodMinutes) =>
+                val defaultSlotSizeMinutes = 15
+                val slotSize = maybePeriodMinutes.map(_.toInt).getOrElse(defaultSlotSizeMinutes)
+                val user = User.fromRoles(email, groups)
+                val ports = enabledPorts.filter(user.accessiblePorts.contains(_)).toList
+                val dateRangeJson = dateRangeJsonForPortsAndSlotSize(ports, slotSize)
+
+                val start = SDate(startStr)
+                val end = SDate(endStr)
+
+                onComplete(dateRangeJson(start, end)) {
+                  case Success(value) => complete(value.toJson.compactPrint)
+                  case Failure(t) =>
+                    log.error(s"Failed to get export: ${t.getMessage}", t)
+                    complete(InternalServerError)
+                }
               }
             }
           }
-        }
+        )
       }
     }
 }