diff --git a/build.gradle b/build.gradle index 9a2f92e141..06235e7cf4 100644 --- a/build.gradle +++ b/build.gradle @@ -1,21 +1,39 @@ allprojects { apply plugin: 'java-library' + apply plugin: 'kotlin' apply plugin: 'maven' sourceCompatibility = 1.7 targetCompatibility = 1.7 - version 'v0.19.5' + //version 'v0.19.5' + version 'LOCAL_SNAPSHOT' group 'com.github.TeamNewPipe' repositories { jcenter() + mavenCentral() maven { url "https://jitpack.io" } } + + compileKotlin { + kotlinOptions { + jvmTarget = "1.6" + } + } + compileTestKotlin { + kotlinOptions { + jvmTarget = "1.6" + } + } + + dependencies { + implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" + } } dependencies { - implementation project(':extractor') + api project(':extractor') implementation project(':timeago-parser') } @@ -55,3 +73,14 @@ task aggregatedJavadocs(type: Javadoc, group: 'Documentation') { } } } + +buildscript { + ext.kotlin_version = '1.3.72' + repositories { + mavenCentral() + } + dependencies { + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" + } +} + diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java index 83d8522f50..3193d021f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/ServiceList.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor; +import org.schabi.newpipe.extractor.services.bbc_sounds.BSService; import org.schabi.newpipe.extractor.services.media_ccc.MediaCCCService; import org.schabi.newpipe.extractor.services.peertube.PeertubeService; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudService; @@ -39,6 +40,7 @@ private ServiceList() { public static final SoundcloudService SoundCloud; public static final MediaCCCService MediaCCC; public static final PeertubeService PeerTube; + public static final BSService BBC_SOUNDS; /** * When creating a new service, put this service in the end of this list, @@ -49,7 +51,8 @@ private ServiceList() { YouTube = new YoutubeService(0), SoundCloud = new SoundcloudService(1), MediaCCC = new MediaCCCService(2), - PeerTube = new PeertubeService(3) + PeerTube = new PeertubeService(3), + BBC_SOUNDS = new BSService(4) )); /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/BSDashMpdParser.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/BSDashMpdParser.kt new file mode 100644 index 0000000000..6721594547 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/BSDashMpdParser.kt @@ -0,0 +1,42 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds + +import org.schabi.newpipe.extractor.MediaFormat +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.stream.AudioStream +import org.schabi.newpipe.extractor.stream.DashMpdParser +import org.schabi.newpipe.extractor.stream.DeliveryFormat +import org.w3c.dom.Element +import java.io.ByteArrayInputStream +import java.util.* +import javax.xml.parsers.DocumentBuilderFactory + +internal object BSDashMpdParser : DashMpdParser() { + + override fun getStreams(manifestUrl: String): Result { + val manifest = NewPipe.getDownloader().get(manifestUrl).responseBody() + val builder = DocumentBuilderFactory.newInstance().newDocumentBuilder() + val dashDoc = builder.parse(ByteArrayInputStream(manifest.toByteArray())) + val representationList = dashDoc.getElementsByTagName("Representation") + + val audioStreams: MutableList = ArrayList() + + for (i in 0 until representationList.length) { + val representation = representationList.item(i) as? Element + representation?.let { + + val adaptationSet = representation.parentNode as Element + val mimeType = adaptationSet.getAttribute("mimeType") + val mediaFormat = MediaFormat.getFromMimeType(mimeType) + val abr = it.getAttribute("bandwidth").toInt() + val deliveryFormat = DeliveryFormat.manualDASH(manifestUrl, manualDashFromRepresentation(dashDoc, it)) + val stream = AudioStream(deliveryFormat, mediaFormat, abr) + if (!AudioStream.containSimilarStream(stream, audioStreams)) { + audioStreams.add(stream) + } + } + } + + return Result(emptyList(), emptyList(), audioStreams) + } + +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/BSParsingHelper.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/BSParsingHelper.kt new file mode 100644 index 0000000000..405384b1d9 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/BSParsingHelper.kt @@ -0,0 +1,36 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds + +import org.schabi.newpipe.extractor.exceptions.ParsingException +import org.schabi.newpipe.extractor.utils.Parser +import org.schabi.newpipe.extractor.utils.Parser.RegexException +import java.text.ParseException +import java.text.SimpleDateFormat +import java.util.* + +internal object BSParsingHelper { + + @Throws(ParsingException::class) + fun parseDate(textualDate: String): Calendar { + return try { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'") + .apply { timeZone = TimeZone.getTimeZone("GMT") } + .parse(textualDate).let { Calendar.getInstance().apply { time = it } } + } catch (e: ParseException) { + throw ParsingException("Could not parse date: $textualDate", e) + } + } + + fun getNextPageUrl(prevPageUrl: String, limit: Int, total: Int): String { + val prevOffset = try { + Parser.matchGroup1("offset=(\\d*)", prevPageUrl) + } catch (e: RegexException) { + return "" + } + val nextOffset = prevOffset.toInt() + limit + return if (nextOffset < total) { + prevPageUrl.replace("offset=$prevOffset", "offset=$nextOffset") + } else { + "" + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/BSService.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/BSService.kt new file mode 100644 index 0000000000..01f004e96e --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/BSService.kt @@ -0,0 +1,94 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds + +import org.schabi.newpipe.extractor.StreamingService +import org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.AUDIO +import org.schabi.newpipe.extractor.StreamingService.ServiceInfo.MediaCapability.LIVE +import org.schabi.newpipe.extractor.channel.ChannelExtractor +import org.schabi.newpipe.extractor.comments.CommentsExtractor +import org.schabi.newpipe.extractor.kiosk.KioskList +import org.schabi.newpipe.extractor.kiosk.KioskList.KioskExtractorFactory +import org.schabi.newpipe.extractor.linkhandler.* +import org.schabi.newpipe.extractor.playlist.PlaylistExtractor +import org.schabi.newpipe.extractor.search.SearchExtractor +import org.schabi.newpipe.extractor.services.bbc_sounds.extractors.BSExtractorHelper +import org.schabi.newpipe.extractor.services.bbc_sounds.extractors.BSKioskExtractor +import org.schabi.newpipe.extractor.services.bbc_sounds.extractors.BSSearchExtractor +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSChannelLinkHandlerFactory +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSKioskLinkHandlerFactory +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSSearchQueryHandlerFactory +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSStreamLinkHandlerFactory +import org.schabi.newpipe.extractor.stream.StreamExtractor +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor +import org.schabi.newpipe.extractor.suggestion.SuggestionExtractor + +class BSService(id: Int) : StreamingService(id, "BBC Sounds", listOf(AUDIO, LIVE)) { + override fun getBaseUrl(): String { + return "https://bbc.co.uk/sounds" + } + + override fun getStreamLHFactory(): LinkHandlerFactory { + return BSStreamLinkHandlerFactory + } + + override fun getChannelLHFactory(): ListLinkHandlerFactory { + return BSChannelLinkHandlerFactory + } + + override fun getPlaylistLHFactory(): ListLinkHandlerFactory? { + return null + } + + override fun getSearchQHFactory(): SearchQueryHandlerFactory { + return BSSearchQueryHandlerFactory + } + + override fun getCommentsLHFactory(): ListLinkHandlerFactory? { + return null + } + + override fun getSearchExtractor(queryHandler: SearchQueryHandler): SearchExtractor { + // TODO fix this to return brands,series also + return BSSearchExtractor(this, queryHandler) + } + + override fun getSuggestionExtractor(): SuggestionExtractor? { + return null + } + + override fun getSubscriptionExtractor(): SubscriptionExtractor? { + return null + } + + override fun getKioskList(): KioskList { + val kioskLHF = BSKioskLinkHandlerFactory + + val kioskFactory = KioskExtractorFactory { service, _, id -> + BSKioskExtractor(service, + kioskLHF.fromId(id), id) + } + + val list = KioskList(this) + BSKioskLinkHandlerFactory.kiosks.forEach { + list.addKioskEntry(kioskFactory, kioskLHF, it.key) + } + list.setDefaultKiosk(BSKioskLinkHandlerFactory.DEFAULT_KIOSK_ID) + + return list + } + + override fun getChannelExtractor(linkHandler: ListLinkHandler): ChannelExtractor { + return BSExtractorHelper.getChannelExtractor(this, linkHandler) + } + + override fun getPlaylistExtractor(linkHandler: ListLinkHandler?): PlaylistExtractor? { + return null + } + + override fun getStreamExtractor(linkHandler: LinkHandler): StreamExtractor { + return BSExtractorHelper.getStreamExtractor(this, linkHandler) + } + + override fun getCommentsExtractor(linkHandler: ListLinkHandler?): CommentsExtractor? { + return null + } +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSChannelExtractor.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSChannelExtractor.kt new file mode 100644 index 0000000000..237762ef22 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSChannelExtractor.kt @@ -0,0 +1,96 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import com.grack.nanojson.JsonObject +import com.grack.nanojson.JsonParser +import org.schabi.newpipe.extractor.StreamingService +import org.schabi.newpipe.extractor.channel.ChannelExtractor +import org.schabi.newpipe.extractor.downloader.Downloader +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSChannelLinkHandlerFactory +import org.schabi.newpipe.extractor.stream.StreamInfoItem + +class BSChannelExtractor(service: StreamingService, linkHandler: ListLinkHandler) : ChannelExtractor(service, linkHandler) { + + private lateinit var initPage: InfoItemsPage + private var container: JsonObject? = null + private var network: JsonObject? = null + + override fun onFetchPage(downloader: Downloader) { + initPage = getPage(url) + } + + override fun getInitialPage(): InfoItemsPage { + super.fetchPage() + return initPage + } + + override fun getNextPageUrl(): String { + super.fetchPage() + return initPage.nextPageUrl + } + + override fun getPage(pageUrl: String): InfoItemsPage { + val response = JsonParser.`object`().from(downloader.get(pageUrl).responseBody()) + if(network == null) { + network = response.getArray("data")?.get(0)?.let { it as? JsonObject }?.getObject("network") + } + if(container == null) { + container = response.getArray("data")?.get(0)?.let { it as? JsonObject }?.getObject("container") + } + return BSExtractorHelper.parsePage(this, pageUrl, response) + } + + private fun getDescription(synopses: JsonObject): String { + synopses.getString("long")?.let { return it } + synopses.getString("medium")?.let { return it } + synopses.getString("short")?.let { return it } + return "" + } + + override fun getSubscriberCount(): Long { + return -1 + } + + override fun getName(): String { + return container?.getString("title") ?: "" + } + + override fun getAvatarUrl(): String { + return initPage.items?.get(0)?.thumbnailUrl ?: "" + } + + override fun getBannerUrl(): String { + return "" + } + + override fun getFeedUrl(): String { + return "" + } + + override fun getDescription(): String { + return container?.getObject("synopses")?.let { getDescription(it) } ?: "" + } + + override fun getParentChannelName(): String { + return network?.getString("short_title") ?: "" + } + + override fun getParentChannelUrl(): String { + return network?.getString("id")?.let { + val id = BSChannelLinkHandlerFactory.NETWORK_ID_PREFIX + it + BSChannelLinkHandlerFactory.fromNetworkId(id).url + } ?: "" + } + + override fun getParentChannelAvatarUrl(): String { + return network?.getString("logo_url") + ?.replace("{type}", "colour") + ?.replace("{size}", "default") + ?.replace("{format}", "svg") ?: "" + + } + + override fun getOriginalUrl(): String { + return "${service.baseUrl}/brand/$id" + } +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSExtractorHelper.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSExtractorHelper.kt new file mode 100644 index 0000000000..f5d19e7725 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSExtractorHelper.kt @@ -0,0 +1,100 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import com.grack.nanojson.JsonObject +import com.grack.nanojson.JsonParser +import org.schabi.newpipe.extractor.Extractor +import org.schabi.newpipe.extractor.ListExtractor +import org.schabi.newpipe.extractor.channel.ChannelExtractor +import org.schabi.newpipe.extractor.downloader.Downloader +import org.schabi.newpipe.extractor.linkhandler.LinkHandler +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler +import org.schabi.newpipe.extractor.services.bbc_sounds.BSDashMpdParser +import org.schabi.newpipe.extractor.services.bbc_sounds.BSParsingHelper +import org.schabi.newpipe.extractor.services.bbc_sounds.BSService +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSChannelLinkHandlerFactory +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSStreamLinkHandlerFactory +import org.schabi.newpipe.extractor.stream.* + +internal object BSExtractorHelper { + + fun getStreamInfoItemExtractor(data: JsonObject): StreamInfoItemExtractor { + val id = data.getString("urn") + return if(BSStreamLinkHandlerFactory.onAcceptNetworkId(id)) { + BSNetworkStreamInfoItemExtractor(data) + } else { + BSStreamInfoItemExtractor(data) + } + } + + fun getChannelExtractor(service: BSService, linkHandler: ListLinkHandler): ChannelExtractor { + if(BSChannelLinkHandlerFactory.onAcceptNetworkUrl(linkHandler.url)) { + return BSNetworkExtractor(service, linkHandler) + } + return BSChannelExtractor(service, linkHandler) + } + + fun getStreamExtractor(service: BSService, linkHandler: LinkHandler): StreamExtractor { + if(BSStreamLinkHandlerFactory.onAcceptNetworkUrl(linkHandler.url)) { + return BSNetworkStreamExtractor(service, linkHandler) + } + return BSStreamExtractor(service, linkHandler) + } + + fun fetchStreams(downloader: Downloader, data: JsonObject): List { + val audioStreams: MutableList = ArrayList() + //fetch stream links + var streamData: JsonObject? = null + MEDIASELECTOR_URLS.forEach { + kotlin.runCatching { + val vpId = data.getString("id") + val response = downloader.get(it.format(vpId))?.responseBody() + JsonParser.`object`().from(response)?.takeIf { it.containsKey("media") }.apply { + streamData = this + return@forEach + } + } + } + + streamData?.getArray("media")?.forEach { media -> + (media as? JsonObject)?.takeIf { it.getString("kind") == "audio" }?.let { + val abr = it.getString("bitrate").toInt() + it.getArray("connection")?.forEach { connection -> + (connection as? JsonObject)?.takeIf { it.getString("protocol") == "https" && it.getString("href") != null }?.apply { + val transferFormat = this.getString("transferFormat") + val url = this.getString("href") + when (transferFormat) { + "hls", "hds" -> { + // don't bother atm, dash works perfect + } + "dash" -> kotlin.runCatching { + BSDashMpdParser.getStreams(url).audioStreams?.forEach { stream -> + if (!AudioStream.containSimilarStream(stream, audioStreams)) { + audioStreams.add(stream) + } + } + } + } + } + } + } + } + return audioStreams + } + + fun parsePage(extractor: Extractor, pageUrl: String, pageContent: JsonObject): ListExtractor.InfoItemsPage { + val collector = StreamInfoItemsCollector(extractor.serviceId) + pageContent.getArray("data")?.forEach {item -> + (item as? JsonObject)?.apply { + collector.commit(getStreamInfoItemExtractor(this)) + } + } + val limit = pageContent.getInt("limit") + val total = pageContent.getInt("total") + return ListExtractor.InfoItemsPage(collector, BSParsingHelper.getNextPageUrl(pageUrl, limit, total)) + } + + + private val MEDIASELECTOR_URLS = listOf( + "https://open.live.bbc.co.uk/mediaselector/6/select/version/2.0/mediaset/pc/vpid/%s" + ) +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSKioskExtractor.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSKioskExtractor.kt new file mode 100644 index 0000000000..6e0b9fb580 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSKioskExtractor.kt @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import com.grack.nanojson.JsonParser +import org.schabi.newpipe.extractor.StreamingService +import org.schabi.newpipe.extractor.downloader.Downloader +import org.schabi.newpipe.extractor.kiosk.KioskExtractor +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler +import org.schabi.newpipe.extractor.stream.StreamInfoItem + +class BSKioskExtractor(streamingService: StreamingService?, linkHandler: ListLinkHandler?, kioskId: String?) : KioskExtractor(streamingService, linkHandler, kioskId) { + + private lateinit var initPage: InfoItemsPage + + override fun getName(): String { + return id + } + + override fun onFetchPage(downloader: Downloader) { + initPage = getPage(url) + } + + override fun getInitialPage(): InfoItemsPage { + super.fetchPage() + return initPage + } + + override fun getNextPageUrl(): String { + super.fetchPage() + return initPage.nextPageUrl + } + + override fun getPage(pageUrl: String): InfoItemsPage { + val response = JsonParser.`object`().from(downloader.get(pageUrl).responseBody()) + return BSExtractorHelper.parsePage(this, pageUrl, response) + } +} + diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkExtractor.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkExtractor.kt new file mode 100644 index 0000000000..fc65b0d06d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkExtractor.kt @@ -0,0 +1,83 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import com.grack.nanojson.JsonObject +import com.grack.nanojson.JsonParser +import org.schabi.newpipe.extractor.StreamingService +import org.schabi.newpipe.extractor.channel.ChannelExtractor +import org.schabi.newpipe.extractor.downloader.Downloader +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler +import org.schabi.newpipe.extractor.stream.StreamInfoItem + +class BSNetworkExtractor(service: StreamingService, linkHandler: ListLinkHandler) : ChannelExtractor(service, linkHandler) { + + private lateinit var initPage: InfoItemsPage + private var network: JsonObject? = null + + override fun onFetchPage(downloader: Downloader) { + initPage = getPage(url) + } + + override fun getInitialPage(): InfoItemsPage { + super.fetchPage() + return initPage + } + + override fun getNextPageUrl(): String { + super.fetchPage() + return initPage.nextPageUrl + } + + override fun getPage(pageUrl: String): InfoItemsPage { + val response = JsonParser.`object`().from(downloader.get(pageUrl).responseBody()) + if(network == null) { + network = response.getArray("data")?.get(0)?.let { it as? JsonObject }?.getObject("network") + } + return BSExtractorHelper.parsePage(this, pageUrl, response) + } + + override fun getSubscriberCount(): Long { + return -1 + } + + override fun getName(): String { + return network?.getString("short_title") ?: "" + } + + override fun getAvatarUrl(): String { + return network?.getString("logo_url") + ?.replace("{type}", "colour") + ?.replace("{size}", "default") + ?.replace("{format}", "svg") ?: "" + } + + override fun getBannerUrl(): String { + return "" + } + + override fun getFeedUrl(): String { + return "" + } + + override fun getDescription(): String { + return "" + } + + override fun getParentChannelName(): String { + return "" + } + + override fun getParentChannelUrl(): String { + return "" + } + + override fun getParentChannelAvatarUrl(): String { + return "" + } + + override fun getOriginalUrl(): String { + return network?.getString("key")?.let { + "https://www.bbc.co.uk/$it" + } ?: "" + } + +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkStreamExtractor.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkStreamExtractor.kt new file mode 100644 index 0000000000..3a31aab2bb --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkStreamExtractor.kt @@ -0,0 +1,188 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import com.grack.nanojson.JsonObject +import com.grack.nanojson.JsonParser +import com.grack.nanojson.JsonParserException +import org.schabi.newpipe.extractor.MediaFormat +import org.schabi.newpipe.extractor.StreamingService +import org.schabi.newpipe.extractor.downloader.Downloader +import org.schabi.newpipe.extractor.exceptions.ExtractionException +import org.schabi.newpipe.extractor.linkhandler.LinkHandler +import org.schabi.newpipe.extractor.localization.DateWrapper +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSChannelLinkHandlerFactory +import org.schabi.newpipe.extractor.stream.* +import java.util.* + +class BSNetworkStreamExtractor(service: StreamingService?, linkHandler: LinkHandler?) : StreamExtractor(service, linkHandler) { + private lateinit var data: JsonObject + private lateinit var basicInfoExtractor: StreamInfoItemExtractor + private lateinit var audioStreams: List + + override fun onFetchPage(downloader: Downloader) { + try { + data = JsonParser.`object`().from(downloader.get(linkHandler.url).responseBody()) + } catch (e: JsonParserException) { + throw ExtractionException("could not parse data from ${linkHandler.url}", e) + } + + basicInfoExtractor = BSExtractorHelper.getStreamInfoItemExtractor(data) + audioStreams = BSExtractorHelper.fetchStreams(downloader, data) + } + + override fun getUploadDate(): DateWrapper? { + return basicInfoExtractor.uploadDate + } + + override fun getDescription(): Description { + return data.getObject("synopses")?.getString("long").let { Description(it, Description.PLAIN_TEXT) } + } + + override fun getAgeLimit(): Int { + return 0 + } + + override fun getLength(): Long { + return basicInfoExtractor.duration + } + + override fun getTimeStamp(): Long { + return 0L + } + + override fun getDashMpdUrl(): String { + // selects best dash format + return audioStreams.filter { it.deliveryFormat is DeliveryFormat.ManualDASH } + .maxBy { it.averageBitrate } + ?.let { it.deliveryFormat as DeliveryFormat.ManualDASH } + ?.baseUrl ?: "" + } + + override fun getVideoOnlyStreams(): MutableList? { + return Collections.emptyList() + } + + override fun getSubtitles(format: MediaFormat?): MutableList { + return Collections.emptyList() + } + + override fun getStreamType(): StreamType { + return basicInfoExtractor.streamType + } + + override fun getRelatedStreams(): StreamInfoItemsCollector? { + return null + } + + override fun getHost(): String { + return "" + } + + override fun getCategory(): String { + return "" + } + + override fun getLicence(): String { + return "" + } + + override fun getName(): String { + return basicInfoExtractor.name + } + + override fun getTextualUploadDate(): String? { + return basicInfoExtractor.textualUploadDate + } + + override fun getThumbnailUrl(): String { + return data.getString("image_url")?.replace("{recipe}", "1280x720") ?: "" + } + + override fun getViewCount(): Long { + return basicInfoExtractor.viewCount + } + + override fun getLikeCount(): Long { + return -1 + } + + override fun getDislikeCount(): Long { + return -1 + } + + override fun getUploaderUrl(): String { + return basicInfoExtractor.uploaderUrl + } + + override fun getUploaderAvatarUrl(): String { + return data.getObject("network") + ?.getString("logo_url") + ?.replace("{type}", "colour") + ?.replace("{size}", "default") + ?.replace("{format}", "svg") ?: "" + } + + override fun getUploaderName(): String { + return basicInfoExtractor.uploaderName + } + + override fun getSubChannelUrl(): String { + val id = data.getObject("container")?.getString("id") + return id?.let { BSChannelLinkHandlerFactory.fromId(id).url } ?: "" + } + + override fun getSubChannelAvatarUrl(): String { + return data.getString("image_url")?.replace("{recipe}", "320x320") ?: "" + } + + override fun getSubChannelName(): String { + return data.getObject("container")?.getString("title") ?: "" + } + + override fun getHlsUrl(): String { + return "" + } + + override fun getAudioStreams(): List { + assertPageFetched() + return audioStreams + } + + override fun getVideoStreams(): MutableList { + return Collections.emptyList() + } + + override fun getSubtitlesDefault(): MutableList { + return Collections.emptyList() + } + + override fun getNextStream(): StreamInfoItem? { + return null + } + + override fun getErrorMessage(): String { + return "" + } + + override fun getPrivacy(): String { + return "" + } + + override fun getLanguageInfo(): Locale? { + return null + } + + override fun getTags(): MutableList { + return Collections.emptyList() + } + + override fun getSupportInfo(): String { + return "" + } + + override fun getOriginalUrl(): String { + return data.getObject("network")?.getString("key")?.let { + "https://www.bbc.co.uk/$it" + } ?: "" + } + +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkStreamInfoItemExtractor.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkStreamInfoItemExtractor.kt new file mode 100644 index 0000000000..7eb97eb019 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkStreamInfoItemExtractor.kt @@ -0,0 +1,68 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import com.grack.nanojson.JsonObject +import org.schabi.newpipe.extractor.localization.DateWrapper +import org.schabi.newpipe.extractor.services.bbc_sounds.BSParsingHelper.parseDate +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSChannelLinkHandlerFactory +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSStreamLinkHandlerFactory +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor +import org.schabi.newpipe.extractor.stream.StreamType + +internal class BSNetworkStreamInfoItemExtractor(val data: JsonObject): StreamInfoItemExtractor { + + init { + val id = data.getString("urn") + if(!BSStreamLinkHandlerFactory.onAcceptNetworkId(id)) { + throw IllegalArgumentException("only network streams are allowed") + } + } + + override fun getUrl(): String { + val id = data.getString("urn") + return BSStreamLinkHandlerFactory.fromNetworkId(id).url + } + + override fun getDuration(): Long { + return -1 + } + + override fun getName(): String { + val networkName = data.getObject("network")?.getString("short_title") + val showName = data.getObject("titles")?.getString("primary") + return "$networkName | $showName" + } + + override fun getThumbnailUrl(): String? { + return data.getString("image_url")?.replace("{recipe}", "640x360") ?: "" + } + + override fun getStreamType(): StreamType { + return StreamType.AUDIO_LIVE_STREAM + } + + override fun isAd(): Boolean { + return false + } + + override fun getTextualUploadDate(): String? { + return data.getObject("release")?.getString("date") + } + + override fun getUploadDate(): DateWrapper? { + return textualUploadDate?.let { parseDate(it) }?.let { DateWrapper(it) } + } + + override fun getUploaderName(): String { + return data.getObject("network")?.getString("short_title") ?: "" + } + + override fun getUploaderUrl(): String { + val id = BSChannelLinkHandlerFactory.NETWORK_ID_PREFIX + data.getObject("network").getString("id") + return BSChannelLinkHandlerFactory.fromNetworkId(id).url + } + + override fun getViewCount(): Long { + return -1 + } + +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSSearchExtractor.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSSearchExtractor.kt new file mode 100644 index 0000000000..154a9e8343 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSSearchExtractor.kt @@ -0,0 +1,53 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import com.grack.nanojson.JsonObject +import com.grack.nanojson.JsonParser +import org.schabi.newpipe.extractor.InfoItem +import org.schabi.newpipe.extractor.StreamingService +import org.schabi.newpipe.extractor.downloader.Downloader +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler +import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector +import org.schabi.newpipe.extractor.search.SearchExtractor + +class BSSearchExtractor(service: StreamingService, linkHandler: SearchQueryHandler) : SearchExtractor(service, linkHandler) { + + private lateinit var initPage: InfoItemsPage + + override fun onFetchPage(downloader: Downloader) { + initPage = getPage(url) + } + + override fun getInitialPage(): InfoItemsPage { + super.fetchPage() + return initPage + } + + override fun getNextPageUrl(): String { + super.fetchPage() + return initPage.nextPageUrl + } + + override fun getPage(pageUrl: String): InfoItemsPage { + val collector = InfoItemsSearchCollector(serviceId) + val response = JsonParser.`object`().from(downloader.get(pageUrl).responseBody()) + response.getArray("data")?.firstOrNull { + (it as? JsonObject)?.getString("id") == "playable_search" + }?.let { + (it as? JsonObject)?.getArray("data")?.forEach { item -> + (item as? JsonObject)?.apply { + collector.commit(BSExtractorHelper.getStreamInfoItemExtractor(this)) + } + } + } + return InfoItemsPage(collector, "") + } + + override fun getSearchSuggestion(): String { + return "" + } + + override fun isCorrectedSearch(): Boolean { + return false + } + +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSStreamExtractor.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSStreamExtractor.kt new file mode 100644 index 0000000000..69678d3d37 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSStreamExtractor.kt @@ -0,0 +1,200 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import com.grack.nanojson.JsonObject +import com.grack.nanojson.JsonParser +import com.grack.nanojson.JsonParserException +import org.schabi.newpipe.extractor.MediaFormat +import org.schabi.newpipe.extractor.StreamingService +import org.schabi.newpipe.extractor.downloader.Downloader +import org.schabi.newpipe.extractor.exceptions.ExtractionException +import org.schabi.newpipe.extractor.linkhandler.LinkHandler +import org.schabi.newpipe.extractor.localization.DateWrapper +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSChannelLinkHandlerFactory +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSStreamLinkHandlerFactory +import org.schabi.newpipe.extractor.stream.* +import java.util.* + +class BSStreamExtractor(service: StreamingService?, linkHandler: LinkHandler?) : StreamExtractor(service, linkHandler) { + + private lateinit var data: JsonObject + private lateinit var basicInfoExtractor: StreamInfoItemExtractor + private lateinit var audioStreams: List + + override fun onFetchPage(downloader: Downloader) { + try { + data = JsonParser.`object`().from(downloader.get(linkHandler.url).responseBody()) + } catch (e: JsonParserException) { + throw ExtractionException("could not parse data from ${linkHandler.url}", e) + } + + basicInfoExtractor = BSExtractorHelper.getStreamInfoItemExtractor(data) + audioStreams = BSExtractorHelper.fetchStreams(downloader, data) + } + + override fun getUploadDate(): DateWrapper? { + return basicInfoExtractor.uploadDate + } + + override fun getDescription(): Description { + return data.getObject("synopses")?.getString("long").let { Description(it, Description.PLAIN_TEXT) } + } + + override fun getAgeLimit(): Int { + return 0 + } + + override fun getLength(): Long { + return basicInfoExtractor.duration + } + + override fun getTimeStamp(): Long { + return 0L + } + + override fun getDashMpdUrl(): String { + // selects best dash format + return audioStreams.filter { it.deliveryFormat is DeliveryFormat.ManualDASH } + .maxBy { it.averageBitrate } + ?.let { it.deliveryFormat as DeliveryFormat.ManualDASH } + ?.baseUrl ?: "" + } + + override fun getVideoOnlyStreams(): MutableList? { + return Collections.emptyList() + } + + override fun getSubtitles(format: MediaFormat?): MutableList { + return Collections.emptyList() + } + + override fun getStreamType(): StreamType { + return basicInfoExtractor.streamType + } + + override fun getRelatedStreams(): StreamInfoItemsCollector? { + val collector = StreamInfoItemsCollector(serviceId) + try { + val response = JsonParser.`object`().from(downloader.get((service.streamLHFactory as BSStreamLinkHandlerFactory).getRelatedStreamsUrl(id)).responseBody()) + response?.let { + it.getArray("data")?.forEach { item -> + (item as? JsonObject)?.apply { + collector.commit(BSExtractorHelper.getStreamInfoItemExtractor(this)) + } + } + } + } catch (e: Exception) { + } + return collector + } + + override fun getHost(): String { + return "" + } + + override fun getCategory(): String { + return "" + } + + override fun getLicence(): String { + return "" + } + + override fun getName(): String { + return basicInfoExtractor.name + } + + override fun getTextualUploadDate(): String? { + return basicInfoExtractor.textualUploadDate + } + + override fun getThumbnailUrl(): String { + return data.getString("image_url")?.replace("{recipe}", "1280x720") ?: "" + } + + override fun getViewCount(): Long { + return basicInfoExtractor.viewCount + } + + override fun getLikeCount(): Long { + return -1 + } + + override fun getDislikeCount(): Long { + return -1 + } + + override fun getUploaderUrl(): String { + return basicInfoExtractor.uploaderUrl + } + + override fun getUploaderAvatarUrl(): String { + return data.getObject("network") + ?.getString("logo_url") + ?.replace("{type}", "colour") + ?.replace("{size}", "default") + ?.replace("{format}", "svg") ?: "" + } + + override fun getUploaderName(): String { + return basicInfoExtractor.uploaderName + } + + override fun getSubChannelUrl(): String { + val id = data.getObject("container")?.getString("id") + return id?.let { BSChannelLinkHandlerFactory.fromId(id).url } ?: "" + } + + override fun getSubChannelAvatarUrl(): String { + return data.getString("image_url")?.replace("{recipe}", "320x320") ?: "" + } + + override fun getSubChannelName(): String { + return data.getObject("container")?.getString("title") ?: "" + } + + override fun getHlsUrl(): String { + return "" + } + + override fun getAudioStreams(): List { + assertPageFetched() + return audioStreams + } + + override fun getVideoStreams(): MutableList { + return Collections.emptyList() + } + + override fun getSubtitlesDefault(): MutableList { + return Collections.emptyList() + } + + override fun getNextStream(): StreamInfoItem? { + return null + } + + override fun getErrorMessage(): String { + return "" + } + + override fun getPrivacy(): String { + return "" + } + + override fun getLanguageInfo(): Locale? { + return null + } + + override fun getTags(): MutableList { + return Collections.emptyList() + } + + override fun getSupportInfo(): String { + return "" + } + + override fun getOriginalUrl(): String { + return "${service.baseUrl}/play/$id" + } + +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSStreamInfoItemExtractor.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSStreamInfoItemExtractor.kt new file mode 100644 index 0000000000..b539611e7c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSStreamInfoItemExtractor.kt @@ -0,0 +1,70 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import com.grack.nanojson.JsonObject +import org.schabi.newpipe.extractor.localization.DateWrapper +import org.schabi.newpipe.extractor.services.bbc_sounds.BSParsingHelper.parseDate +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSChannelLinkHandlerFactory +import org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler.BSStreamLinkHandlerFactory +import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor +import org.schabi.newpipe.extractor.stream.StreamType + +class BSStreamInfoItemExtractor(val data: JsonObject) : StreamInfoItemExtractor { + + init { + val id = data.getString("urn") + if(BSStreamLinkHandlerFactory.onAcceptNetworkId(id)) { + throw IllegalArgumentException("network streams not allowed") + } + } + + override fun getUrl(): String { + val id = data.getString("urn") + return BSStreamLinkHandlerFactory.fromId(id).url + } + + override fun getDuration(): Long { + return data.getObject("duration")?.getLong("value") ?: 0L + } + + override fun getName(): String { + var name = data.getObject("titles").getString("primary") + data.getObject("titles").getString("secondary")?.run { name = "$name | $this" } + return name + } + + override fun getThumbnailUrl(): String { + return data.getString("image_url")?.replace("{recipe}", "640x360") ?: "" + } + + override fun getStreamType(): StreamType { + if("live".equals(data.getObject("availability")?.getString("label"), true)){ + return StreamType.AUDIO_LIVE_STREAM + } + return StreamType.AUDIO_STREAM + } + + override fun isAd(): Boolean { + return false + } + + override fun getViewCount(): Long { + return -1 + } + + override fun getUploaderName(): String { + return data.getObject("network")?.getString("short_title") ?: "" + } + + override fun getUploaderUrl(): String { + val id = BSChannelLinkHandlerFactory.NETWORK_ID_PREFIX + data.getObject("network").getString("id") + return BSChannelLinkHandlerFactory.fromNetworkId(id).url + } + + override fun getTextualUploadDate(): String? { + return data.getObject("availability")?.getString("from") + } + + override fun getUploadDate(): DateWrapper? { + return textualUploadDate?.let { parseDate(it) }?.let { DateWrapper(it) } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSChannelLinkHandlerFactory.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSChannelLinkHandlerFactory.kt new file mode 100644 index 0000000000..6be20cbcef --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSChannelLinkHandlerFactory.kt @@ -0,0 +1,70 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler + +import org.schabi.newpipe.extractor.exceptions.ParsingException +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory +import org.schabi.newpipe.extractor.utils.Parser + +object BSChannelLinkHandlerFactory : ListLinkHandlerFactory() { + override fun getUrl(id: String, contentFilter: MutableList?, sortFilter: String?): String { + if (NETWORK_LHF.onAcceptId(id)) { + return NETWORK_LHF.getUrl(id, contentFilter, sortFilter) + } + return CHANNEL_API_URL.format(id.split(":").last()) + } + + override fun getId(url: String): String { + if (NETWORK_LHF.onAcceptUrl(url)) { + return NETWORK_LHF.getId(url) + } + return kotlin.runCatching { Parser.matchGroup(ID_PATTERN, url, 2) }.getOrNull() + ?: Parser.matchGroup1(ID_PATTERN_API, url) + } + + override fun onAcceptUrl(url: String?): Boolean { + return Parser.isMatch(ID_PATTERN, url) || Parser.isMatch(ID_PATTERN_API, url) || NETWORK_LHF.onAcceptUrl(url) + } + + + private const val ID_PATTERN = "/sounds/(brand|series)/([^/?&#]*)" + private const val ID_PATTERN_API = "/programmes/playable\\?container=([^/?&#]*)" + private const val CHANNEL_API_URL = "https://rms.api.bbc.co.uk/v2/programmes/playable?container=%s&offset=0" + const val NETWORK_ID_PREFIX = "network:" + private val NETWORK_LHF = BSNetworkChannelLinkHandlerFactory + + fun onAcceptNetworkUrl(url: String): Boolean { + return NETWORK_LHF.onAcceptUrl(url) + } + + @Throws(ParsingException::class) + fun fromNetworkId(id: String): ListLinkHandler { + if (NETWORK_LHF.onAcceptId(id)) { + return NETWORK_LHF.fromId(id) + } else { + throw ParsingException("id is not of type network") + } + } + + private object BSNetworkChannelLinkHandlerFactory : ListLinkHandlerFactory() { + override fun getUrl(id: String, contentFilter: MutableList?, sortFilter: String?): String { + return NETWORK_API_URL.format(id.split(":").last()) + } + + override fun onAcceptUrl(url: String?): Boolean { + return Parser.isMatch(ID_PATTERN_NETWORK_API, url) + } + + override fun getId(url: String?): String { + val id = Parser.matchGroup1(ID_PATTERN_NETWORK_API, url) + // prefix to distinguish from normal channel + return "$NETWORK_ID_PREFIX$id" + } + + fun onAcceptId(id: String): Boolean { + return id.startsWith(NETWORK_ID_PREFIX) + } + + private const val ID_PATTERN_NETWORK_API = "/programmes/playable\\?network=([^/?&#]*)" + private const val NETWORK_API_URL = "https://rms.api.bbc.co.uk/v2/programmes/playable?network=%s&offset=0" + } +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSKioskLinkHandlerFactory.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSKioskLinkHandlerFactory.kt new file mode 100644 index 0000000000..b83ab5ce10 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSKioskLinkHandlerFactory.kt @@ -0,0 +1,30 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler + +import org.schabi.newpipe.extractor.exceptions.ParsingException +import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory + +object BSKioskLinkHandlerFactory : ListLinkHandlerFactory() { + + override fun getUrl(id: String, contentFilter: MutableList?, sortFilter: String?): String { + return kiosks.getOrElse(id) { throw ParsingException("Unsupported id") } + } + + override fun getId(url: String): String { + return reversedKiosks.getOrElse(url) { throw ParsingException("Unsupported url") } + } + + override fun onAcceptUrl(url: String): Boolean { + return reversedKiosks.containsKey(url) + } + + const val DEFAULT_KIOSK_ID = "Radio (Live)" + val kiosks = mapOf( + "Radio (Live)" to "https://rms.api.bbc.co.uk/v2/networks/playable", + "Mixes" to "https://rms.api.bbc.co.uk/v2/programmes/playable?category=mixes&offset=0", + "Podcasts" to "https://rms.api.bbc.co.uk/v2/programmes/playable?category=podcasts&offset=0", + "News" to "https://rms.api.bbc.co.uk/v2/programmes/playable?category=news&offset=0", + "Sports" to "https://rms.api.bbc.co.uk/v2/programmes/playable?category=sport&offset=0", + "History" to "https://rms.api.bbc.co.uk/v2/programmes/playable?category=factual-history&offset=0" + ) + private val reversedKiosks = kiosks.entries.associate { (k, v) -> v to k } +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSSearchQueryHandlerFactory.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSSearchQueryHandlerFactory.kt new file mode 100644 index 0000000000..f92ed13143 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSSearchQueryHandlerFactory.kt @@ -0,0 +1,11 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler + +import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory + +object BSSearchQueryHandlerFactory : SearchQueryHandlerFactory() { + override fun getUrl(querry: String?, contentFilter: MutableList?, sortFilter: String?): String { + return API_URL.format(querry) + } + + private const val API_URL = "https://rms.api.bbc.co.uk/v2/experience/inline/search?q=%s" +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSStreamLinkHandlerFactory.kt b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSStreamLinkHandlerFactory.kt new file mode 100644 index 0000000000..fe6d4f533d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSStreamLinkHandlerFactory.kt @@ -0,0 +1,85 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler + +import org.schabi.newpipe.extractor.exceptions.ParsingException +import org.schabi.newpipe.extractor.linkhandler.LinkHandler +import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory +import org.schabi.newpipe.extractor.utils.Parser + +object BSStreamLinkHandlerFactory : LinkHandlerFactory() { + override fun getId(url: String): String { + if (NETWORK_STREAM_LHF.onAcceptUrl(url)) { + return NETWORK_STREAM_LHF.getId(url) + } + return kotlin.runCatching { Parser.matchGroup1(ID_PATTERN, url) }.getOrNull() + ?: Parser.matchGroup1(ID_PATTERN_API, url) + } + + override fun getUrl(id: String): String { + if (NETWORK_STREAM_LHF.onAcceptId(id)) { + return NETWORK_STREAM_LHF.getUrl(id) + } + return STREAM_API_URL.format(id.split(":").last()) + } + + override fun onAcceptUrl(url: String?): Boolean { + return Parser.isMatch(ID_PATTERN, url) || Parser.isMatch(ID_PATTERN_API, url) || NETWORK_STREAM_LHF.onAcceptUrl(url) + } + + fun getRelatedStreamsUrl(id: String): String { + if (NETWORK_STREAM_LHF.onAcceptId(id)) { + return NETWORK_STREAM_LHF.getRelatedStreamsUrl(id) + } + return RELATED_STREAMS_API_URL.format(id) + } + + fun onAcceptNetworkUrl(url: String): Boolean { + return NETWORK_STREAM_LHF.onAcceptUrl(url) + } + + fun onAcceptNetworkId(id: String): Boolean { + return NETWORK_STREAM_LHF.onAcceptId(id) + } + + @Throws(ParsingException::class) + fun fromNetworkId(id: String): LinkHandler { + if(NETWORK_STREAM_LHF.onAcceptId(id)) { + return NETWORK_STREAM_LHF.fromId(id) + } else { + throw ParsingException("id is not of type network") + } + } + + private val NETWORK_STREAM_LHF = BSNetworkStreamLinkHandlerFactory + private const val ID_PATTERN = "/sounds/play/([^/?&#:]*)" + private const val ID_PATTERN_API = "/programmes/([^/?&#]*)/playable" + private const val STREAM_API_URL = "https://rms.api.bbc.co.uk/v2/programmes/%s/playable" + private const val RELATED_STREAMS_API_URL = "https://rms.api.bbc.co.uk/v2/programmes/playqueue/%s" + + private object BSNetworkStreamLinkHandlerFactory : LinkHandlerFactory() { + override fun getUrl(id: String): String { + return NETWORK_STREAM_API_URL.format(id.split(":").last()) + } + + override fun onAcceptUrl(url: String?): Boolean { + return Parser.isMatch(ID_PATTERN_NETWORK_STREAM_API, url) + } + + override fun getId(url: String?): String { + val id = Parser.matchGroup1(ID_PATTERN_NETWORK_STREAM_API, url) + // prefix to distinguish from normal stream + return "$NETWORK_STREAM_ID_PREFIX$id" + } + + fun onAcceptId(id: String): Boolean { + return id.startsWith(NETWORK_STREAM_ID_PREFIX) + } + + fun getRelatedStreamsUrl(id: String): String { + return "" + } + + private const val ID_PATTERN_NETWORK_STREAM_API = "/networks/([^/?&#]*)/playable" + private const val NETWORK_STREAM_API_URL = "https://rms.api.bbc.co.uk/v2/networks/%s/playable" + private const val NETWORK_STREAM_ID_PREFIX = "urn:bbc:radio:network:" + } +} \ No newline at end of file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java index 93772608ef..2c090435b5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryFormat; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -160,7 +161,8 @@ public List getAudioStreams() throws ExtractionException { throw new ExtractionException("Unknown media format: " + mimeType); } - audioStreams.add(new AudioStream(recording.getString("recording_url"), + audioStreams.add(new AudioStream( + DeliveryFormat.direct(recording.getString("recording_url")), mediaFormat, -1)); } } @@ -186,7 +188,8 @@ public List getVideoStreams() throws ExtractionException { throw new ExtractionException("Unknown media format: " + mimeType); } - videoStreams.add(new VideoStream(recording.getString("recording_url"), + videoStreams.add(new VideoStream( + DeliveryFormat.direct(recording.getString("recording_url")), mediaFormat, recording.getInt("height") + "p")); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java index 72b5bea93d..a75c5796ad 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java @@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper; import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryFormat; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamExtractor; @@ -209,7 +210,8 @@ public List getVideoStreams() throws IOException, ExtractionExcepti String resolution = JsonUtils.getString(stream, "resolution.label"); String extension = url.substring(url.lastIndexOf(".") + 1); MediaFormat format = MediaFormat.getFromSuffix(extension); - VideoStream videoStream = new VideoStream(url, torrentUrl, format, resolution); + VideoStream videoStream = new VideoStream(torrentUrl, DeliveryFormat.direct(url), + format, resolution, false); if (!Stream.containSimilarStream(videoStream, videoStreams)) { videoStreams.add(videoStream); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 431baff94e..e4c6652b1a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -17,6 +17,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryFormat; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -206,7 +207,8 @@ public List getAudioStreams() throws IOException, ExtractionExcepti try { JsonObject mp3UrlObject = JsonParser.object().from(res); // Links in this file are also only valid for a short period. - audioStreams.add(new AudioStream(mp3UrlObject.getString("url"), + audioStreams.add(new AudioStream( + DeliveryFormat.direct(mp3UrlObject.getString("url")), MediaFormat.MP3, 128)); } catch (JsonParserException e) { throw new ParsingException("Could not parse streamable url", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashMpdParser.java new file mode 100644 index 0000000000..a3fbd1846a --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDashMpdParser.java @@ -0,0 +1,172 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DashMpdParser; +import org.schabi.newpipe.extractor.stream.DeliveryFormat; +import org.schabi.newpipe.extractor.stream.Stream; +import org.schabi.newpipe.extractor.stream.StreamInfo; +import org.schabi.newpipe.extractor.stream.VideoStream; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import edu.umd.cs.findbugs.annotations.NonNull; + +/* + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * DashMpdParser.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class YoutubeDashMpdParser extends DashMpdParser { + + public static YoutubeDashMpdParser INSTANCE = new YoutubeDashMpdParser(); + + private YoutubeDashMpdParser() { + } + + // TODO: Make this class generic and decouple from YouTube's ItagItem class. + + /** + * Will try to download and parse the DASH manifest (using {@link StreamInfo#getDashMpdUrl()}), + * adding items that are listed in the {@link ItagItem} class. + *

+ * It has video, video only and audio streams and will only add to the list if it don't + * find a similar stream in the respective lists (calling {@link Stream#equalStats}). + *

+ * Info about dash MPD can be found here + * + * @param manifestUrl manifest url of dash stream + * @see www.brendanlog.com + */ + public Result getStreams(final String manifestUrl) + throws DashMpdParsingException, ReCaptchaException { + final String dashDoc; + final Downloader downloader = NewPipe.getDownloader(); + try { + dashDoc = downloader.get(manifestUrl).responseBody(); + } catch (IOException e) { + throw new DashMpdParsingException("Could not fetch DASH manifest: " + + manifestUrl, e); + } + + try { + final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + final DocumentBuilder builder = factory.newDocumentBuilder(); + final InputStream stream = new ByteArrayInputStream(dashDoc.getBytes()); + + final Document doc = builder.parse(stream); + final NodeList representationList = doc.getElementsByTagName("Representation"); + + final List videoStreams = new ArrayList<>(); + final List audioStreams = new ArrayList<>(); + final List videoOnlyStreams = new ArrayList<>(); + + for (int i = 0; i < representationList.getLength(); i++) { + final Element representation = (Element) representationList.item(i); + try { + final String mimeType = ((Element) representation.getParentNode()) + .getAttribute("mimeType"); + final String id = representation.getAttribute("id"); + final String url = representation.getElementsByTagName("BaseURL") + .item(0).getTextContent(); + final ItagItem itag = ItagItem.getItag(Integer.parseInt(id)); + final Element segmentationList = (Element) representation + .getElementsByTagName("SegmentList").item(0); + + if (segmentationList == null) { + continue; + } + + boolean isUrlRangeBased = false; + boolean isUrlSegmentsBased = false; + final Element initialization = (Element) segmentationList + .getElementsByTagName("Initialization").item(0); + + if (initialization != null && initialization.hasAttributes()) { + final Node sourceURLNode = initialization.getAttributes() + .getNamedItem("sourceURL"); + if (sourceURLNode != null) { + final String initializationSourceUrl = sourceURLNode.getNodeValue(); + + isUrlRangeBased = initializationSourceUrl != null && + initializationSourceUrl.startsWith("range/"); + isUrlSegmentsBased = initializationSourceUrl != null && + initializationSourceUrl.startsWith("sq/"); + } + } + + final DeliveryFormat deliveryFormat; + if (isUrlRangeBased) { + deliveryFormat = DeliveryFormat.direct(url); + } else if (isUrlSegmentsBased) { + deliveryFormat = DeliveryFormat.manualDASH(url, + manualDashFromRepresentation(doc, representation)); + } else { + continue; + } + + final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); + + if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { + final AudioStream audioStream = new AudioStream( + deliveryFormat, mediaFormat, itag.avgBitrate); + audioStreams.add(audioStream); + } else { + boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY); + final VideoStream videoStream = new VideoStream( + deliveryFormat, mediaFormat, + itag.resolutionString, isVideoOnly); + + if (isVideoOnly) { + videoOnlyStreams.add(videoStream); + } else { + videoStreams.add(videoStream); + } + } + } catch (Exception ignored) { + } + } + return new Result(videoStreams, videoOnlyStreams, audioStreams); + } catch (Exception e) { + throw new DashMpdParsingException("Could not parse Dash mpd", e); + } + } + + +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index bdc2a10f3e..b4d1f6de3c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -21,9 +21,12 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; +import org.schabi.newpipe.extractor.services.youtube.YoutubeDashMpdParser; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DashMpdParser; +import org.schabi.newpipe.extractor.stream.DeliveryFormat; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Frameset; import org.schabi.newpipe.extractor.stream.Stream; @@ -52,7 +55,10 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -100,6 +106,9 @@ public class DecryptException extends ParsingException { private JsonObject videoSecondaryInfoRenderer; private int ageLimit; + @Nullable + private DashMpdParser.Result dashStreams; + @Nonnull private List subtitlesInfos = new ArrayList<>(); @@ -422,10 +431,11 @@ public List getAudioStreams() throws ExtractionException { assertPageFetched(); List audioStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) { + for (Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) { ItagItem itag = entry.getValue(); - AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate); + AudioStream audioStream = new AudioStream(entry.getKey(), + itag.getMediaFormat(), itag.avgBitrate); if (!Stream.containSimilarStream(audioStream, audioStreams)) { audioStreams.add(audioStream); } @@ -434,6 +444,10 @@ public List getAudioStreams() throws ExtractionException { throw new ParsingException("Could not get audio streams", e); } + if(dashStreams != null && dashStreams.getAudioStreams() != null) { + appendDashStreams(audioStreams, dashStreams.getAudioStreams()); + } + return audioStreams; } @@ -442,10 +456,11 @@ public List getVideoStreams() throws ExtractionException { assertPageFetched(); List videoStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { + for (Map.Entry entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) { ItagItem itag = entry.getValue(); - VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString); + VideoStream videoStream = new VideoStream(entry.getKey(), + itag.getMediaFormat(), itag.resolutionString); if (!Stream.containSimilarStream(videoStream, videoStreams)) { videoStreams.add(videoStream); } @@ -454,6 +469,9 @@ public List getVideoStreams() throws ExtractionException { throw new ParsingException("Could not get video streams", e); } + if(dashStreams != null && dashStreams.getVideoStreams() != null) { + appendDashStreams(videoStreams, dashStreams.getVideoStreams()); + } return videoStreams; } @@ -462,10 +480,11 @@ public List getVideoOnlyStreams() throws ExtractionException { assertPageFetched(); List videoOnlyStreams = new ArrayList<>(); try { - for (Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { + for (Map.Entry entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) { ItagItem itag = entry.getValue(); - VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true); + VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), + itag.resolutionString, true); if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) { videoOnlyStreams.add(videoStream); } @@ -474,9 +493,21 @@ public List getVideoOnlyStreams() throws ExtractionException { throw new ParsingException("Could not get video only streams", e); } + if(dashStreams != null && dashStreams.getVideoOnlyStreams() != null) { + appendDashStreams(videoOnlyStreams, dashStreams.getVideoOnlyStreams()); + } + return videoOnlyStreams; } + private void appendDashStreams(List streams, List dashStreams) { + for (T stream : dashStreams) { + if (!Stream.containSimilarStream(stream, streams)) { + streams.add(stream); + } + } + } + @Override @Nonnull public List getSubtitlesDefault() { @@ -636,6 +667,27 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr if (subtitlesInfos.isEmpty()) { subtitlesInfos.addAll(getAvailableSubtitlesInfo()); } + + dashStreams = fetchStreamsFromDash(); + } + + @Nullable + private DashMpdParser.Result fetchStreamsFromDash() { + String dashMpdUrl = null; + try { + dashMpdUrl = getDashMpdUrl(); + } catch (ParsingException e) { + // ignore + } + if (!isNullOrEmpty(dashMpdUrl)) { + try { + return YoutubeDashMpdParser.INSTANCE.getStreams(dashMpdUrl); + } catch (Exception e) { + // Sometimes we receive 403 (forbidden) error when trying to download the + // manifest (similar to what happens with youtube-dl), + } + } + return null; } private JsonObject getPlayerArgs(JsonObject playerConfig) throws ParsingException { @@ -909,8 +961,8 @@ private static String getVideoInfoUrl(final String id, final String sts) { "&sts=" + sts + "&ps=default&gl=US&hl=en"; } - private Map getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { - Map urlAndItags = new LinkedHashMap<>(); + private Map getItags(String streamingDataKey, ItagItem.ItagType itagTypeWanted) throws ParsingException { + Map urlAndItags = new LinkedHashMap<>(); JsonObject streamingData = playerResponse.getObject("streamingData"); if (!streamingData.has(streamingDataKey)) { return urlAndItags; @@ -926,6 +978,14 @@ private Map getItags(String streamingDataKey, ItagItem.ItagTyp ItagItem itagItem = ItagItem.getItag(itag); if (itagItem.itagType == itagTypeWanted) { String streamUrl; + + // Ignore streams that are delivered using YouTube's OTF format, + // as they will generally be available in when extracting the MPD. + if (formatData.getString("type", EMPTY_STRING) + .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) { + continue; + } + if (formatData.has("url")) { streamUrl = formatData.getString("url"); } else { @@ -938,7 +998,7 @@ private Map getItags(String streamingDataKey, ItagItem.ItagTyp + decryptSignature(cipher.get("s"), decryptionCode); } - urlAndItags.put(streamUrl, itagItem); + urlAndItags.put(DeliveryFormat.direct(streamUrl), itagItem); } } catch (UnsupportedEncodingException ignored) {} } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java index 98fc994274..28eb729409 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java @@ -27,12 +27,13 @@ public class AudioStream extends Stream { /** * Create a new audio stream - * @param url the url + * + * @param deliveryFormat how this stream is delivered * @param format the format * @param averageBitrate the average bitrate */ - public AudioStream(String url, MediaFormat format, int averageBitrate) { - super(url, format); + public AudioStream(DeliveryFormat deliveryFormat, MediaFormat format, int averageBitrate) { + super(deliveryFormat, format); this.average_bitrate = averageBitrate; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/DashMpdParser.java new file mode 100644 index 0000000000..56dcb85d96 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/DashMpdParser.java @@ -0,0 +1,95 @@ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.io.StringWriter; +import java.util.List; + +import javax.annotation.Nonnull; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import edu.umd.cs.findbugs.annotations.NonNull; + +public abstract class DashMpdParser { + + public static class Result { + private final List videoStreams; + private final List videoOnlyStreams; + private final List audioStreams; + + + public Result(List videoStreams, + List videoOnlyStreams, + List audioStreams) { + this.videoStreams = videoStreams; + this.videoOnlyStreams = videoOnlyStreams; + this.audioStreams = audioStreams; + } + + public List getVideoStreams() { + return videoStreams; + } + + public List getVideoOnlyStreams() { + return videoOnlyStreams; + } + + public List getAudioStreams() { + return audioStreams; + } + } + + public static class DashMpdParsingException extends ParsingException { + public DashMpdParsingException(String message, Exception e) { + super(message, e); + } + } + + public abstract Result getStreams(@Nonnull final String manifestUrl) throws DashMpdParsingException, ReCaptchaException; + + @NonNull + protected String manualDashFromRepresentation(Document document, Element representation) + throws TransformerException { + + final Element mpdElement = (Element) document.getElementsByTagName("MPD").item(0); + + // Clone element so we can freely modify it + final Element adaptationSet = (Element) representation.getParentNode(); + final Element adaptationSetClone = (Element) adaptationSet.cloneNode(true); + + // Remove other representations from the adaptation set + final NodeList representations = adaptationSetClone.getElementsByTagName("Representation"); + for (int i = representations.getLength() - 1; i >= 0; i--) { + final Node item = representations.item(i); + if (!item.isEqualNode(representation)) { + adaptationSetClone.removeChild(item); + } + } + + final Element newMpdRootElement = (Element) mpdElement.cloneNode(false); + final Element periodElement = newMpdRootElement.getOwnerDocument().createElement("Period"); + periodElement.appendChild(adaptationSetClone); + newMpdRootElement.appendChild(periodElement); + + return "\n" + + nodeToString(newMpdRootElement); + } + + private String nodeToString(Node node) throws TransformerException { + final StringWriter result = new StringWriter(); + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + transformer.transform(new DOMSource(node), new StreamResult(result)); + return result.toString(); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/DeliveryFormat.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/DeliveryFormat.java new file mode 100644 index 0000000000..90036b1a54 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/DeliveryFormat.java @@ -0,0 +1,134 @@ +package org.schabi.newpipe.extractor.stream; + +import java.io.Serializable; + +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * A class that is used to represent the way that a streaming service deliver their streams. + */ +public abstract class DeliveryFormat implements Serializable { + + public static Direct direct(@NonNull String url) { + return new Direct(url); + } + + public static HLS hls(@NonNull String url) { + return new HLS(url); + } + + public static ManualDASH manualDASH(@NonNull String baseUrl, + @NonNull String manualDashManifest) { + return new ManualDASH(baseUrl, manualDashManifest); + } + + /** + * Used when a service offer a direct link to their streams. + */ + public static class Direct extends DeliveryFormat { + private final String url; + + private Direct(String url) { + this.url = url; + } + + public String getUrl() { + return url; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Direct direct = (Direct) o; + + //noinspection EqualsReplaceableByObjectsCall + return url != null ? url.equals(direct.url) : direct.url == null; + } + + @Override + public int hashCode() { + return url != null ? url.hashCode() : 0; + } + } + + /** + * Used when a service uses HLS playlists for delivering their content. + */ + public static class HLS extends DeliveryFormat { + private final String url; + + private HLS(String url) { + this.url = url; + } + + public String getUrl() { + return url; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + HLS hls = (HLS) o; + + //noinspection EqualsReplaceableByObjectsCall + return url != null ? url.equals(hls.url) : hls.url == null; + } + + @Override + public int hashCode() { + return url != null ? url.hashCode() : 0; + } + } + + /** + * Used when a service uses DASH Manifests to deliver their streams. + *

+ * This is useful for extracting a specific stream from the entire manifest. + */ + public static class ManualDASH extends DeliveryFormat { + private final String baseUrl; + private final String manualDashManifest; + + private ManualDASH(String baseUrl, String manualDashManifest) { + this.baseUrl = baseUrl; + this.manualDashManifest = manualDashManifest; + } + + public String getBaseUrl() { + return baseUrl; + } + + public String getManualDashManifest() { + return manualDashManifest; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ManualDASH that = (ManualDASH) o; + + //noinspection EqualsReplaceableByObjectsCall + if (baseUrl != null ? !baseUrl.equals(that.baseUrl) : that.baseUrl != null) { + return false; + } + + //noinspection EqualsReplaceableByObjectsCall + return manualDashManifest != null + ? manualDashManifest.equals(that.manualDashManifest) + : that.manualDashManifest == null; + } + + @Override + public int hashCode() { + int result = baseUrl != null ? baseUrl.hashCode() : 0; + result = 31 * result + (manualDashManifest != null ? manualDashManifest.hashCode() : 0); + return result; + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index 7e99081678..5b9f41f32b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -12,8 +12,8 @@ */ public abstract class Stream implements Serializable { private final MediaFormat mediaFormat; - public final String url; public final String torrentUrl; + private final DeliveryFormat deliveryFormat; /** * @deprecated Use {@link #getFormat()} or {@link #getFormatId()} @@ -24,23 +24,23 @@ public abstract class Stream implements Serializable { /** * Instantiates a new stream object. * - * @param url the url + * @param deliveryFormat how this stream is delivered * @param format the format */ - public Stream(String url, MediaFormat format) { - this(url, null, format); + public Stream(DeliveryFormat deliveryFormat, MediaFormat format) { + this(null, deliveryFormat, format); } /** * Instantiates a new stream object. - * - * @param url the url * @param torrentUrl the url to torrent file, example https://webtorrent.io/torrents/big-buck-bunny.torrent + * @param deliveryFormat how this stream is delivered * @param format the format */ - public Stream(String url, String torrentUrl, MediaFormat format) { - this.url = url; + public Stream(String torrentUrl, + DeliveryFormat deliveryFormat, MediaFormat format) { this.torrentUrl = torrentUrl; + this.deliveryFormat = deliveryFormat; this.format = format.id; this.mediaFormat = format; } @@ -56,7 +56,7 @@ public boolean equalStats(Stream cmp) { * Reveals whether two Streams are equal */ public boolean equals(Stream cmp) { - return equalStats(cmp) && url.equals(cmp.url); + return equalStats(cmp) && deliveryFormat.equals(cmp.deliveryFormat); } /** @@ -70,15 +70,6 @@ public static boolean containSimilarStream(Stream stream, List return false; } - /** - * Gets the url. - * - * @return the url - */ - public String getUrl() { - return url; - } - /** * Gets the torrent url. * @@ -88,6 +79,13 @@ public String getTorrentUrl() { return torrentUrl; } + /** + * @return how this stream is delivered by a service. + */ + public DeliveryFormat getDeliveryFormat() { + return deliveryFormat; + } + /** * Gets the format. * diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 3878e593a4..4f98af1daa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.localization.DateWrapper; -import org.schabi.newpipe.extractor.utils.DashMpdParser; import org.schabi.newpipe.extractor.utils.ExtractorHelper; import java.io.IOException; @@ -160,37 +159,9 @@ private static StreamInfo extractStreams(StreamInfo streamInfo, StreamExtractor if (streamInfo.getAudioStreams() == null) streamInfo.setAudioStreams(new ArrayList()); - Exception dashMpdError = null; - if (!isNullOrEmpty(streamInfo.getDashMpdUrl())) { - try { - DashMpdParser.ParserResult result = DashMpdParser.getStreams(streamInfo); - streamInfo.getVideoOnlyStreams().addAll(result.getVideoOnlyStreams()); - streamInfo.getAudioStreams().addAll(result.getAudioStreams()); - streamInfo.getVideoStreams().addAll(result.getVideoStreams()); - streamInfo.segmentedVideoOnlyStreams = result.getSegmentedVideoOnlyStreams(); - streamInfo.segmentedAudioStreams = result.getSegmentedAudioStreams(); - streamInfo.segmentedVideoStreams = result.getSegmentedVideoStreams(); - } catch (Exception e) { - // Sometimes we receive 403 (forbidden) error when trying to download the - // manifest (similar to what happens with youtube-dl), - // just skip the exception (but store it somewhere), as we later check if we - // have streams anyway. - dashMpdError = e; - } - } - // Either audio or video has to be available, otherwise we didn't get a stream // (since videoOnly are optional, they don't count). if ((streamInfo.videoStreams.isEmpty()) && (streamInfo.audioStreams.isEmpty())) { - - if (dashMpdError != null) { - // If we don't have any video or audio and the dashMpd 'errored', add it to the - // error list - // (it's optional and it don't get added automatically, but it's good to have - // some additional error context) - streamInfo.addError(dashMpdError); - } - throw new StreamExtractException("Could not get any stream. See error variable to get further details."); } @@ -565,30 +536,6 @@ public void setDashMpdUrl(String dashMpdUrl) { this.dashMpdUrl = dashMpdUrl; } - public List getSegmentedVideoStreams() { - return segmentedVideoStreams; - } - - public void setSegmentedVideoStreams(List segmentedVideoStreams) { - this.segmentedVideoStreams = segmentedVideoStreams; - } - - public List getSegmentedAudioStreams() { - return segmentedAudioStreams; - } - - public void setSegmentedAudioStreams(List segmentedAudioStreams) { - this.segmentedAudioStreams = segmentedAudioStreams; - } - - public List getSegmentedVideoOnlyStreams() { - return segmentedVideoOnlyStreams; - } - - public void setSegmentedVideoOnlyStreams(List segmentedVideoOnlyStreams) { - this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams; - } - public String getHlsUrl() { return hlsUrl; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java index 8f16942e33..46e743d2cf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SubtitlesStream.java @@ -8,12 +8,11 @@ public class SubtitlesStream extends Stream implements Serializable { private final MediaFormat format; private final Locale locale; - private final String url; private final boolean autoGenerated; private final String code; public SubtitlesStream(MediaFormat format, String languageCode, String url, boolean autoGenerated) { - super(url, format); + super(DeliveryFormat.direct(url), format); /* * Locale.forLanguageTag only for API >= 21 @@ -34,7 +33,6 @@ public SubtitlesStream(MediaFormat format, String languageCode, String url, bool } this.code = languageCode; this.format = format; - this.url = url; this.autoGenerated = autoGenerated; } @@ -42,8 +40,8 @@ public String getExtension() { return format.suffix; } - public String getURL() { - return url; + public String getUrl() { + return ((DeliveryFormat.Direct) getDeliveryFormat()).getUrl(); } public boolean isAutoGenerated() { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java index 67219240f9..b16ffc8ddc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/VideoStream.java @@ -26,23 +26,18 @@ public class VideoStream extends Stream { public final String resolution; public final boolean isVideoOnly; - - public VideoStream(String url, MediaFormat format, String resolution) { - this(url, format, resolution, false); - } - - public VideoStream(String url, MediaFormat format, String resolution, boolean isVideoOnly) { - super(url, format); - this.resolution = resolution; - this.isVideoOnly = isVideoOnly; + public VideoStream(DeliveryFormat deliveryFormat, MediaFormat format, String resolution) { + this(null, deliveryFormat, format, resolution, false); } - public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution) { - this(url, torrentUrl, format, resolution, false); + public VideoStream(DeliveryFormat deliveryFormat, MediaFormat format, + String resolution, boolean isVideoOnly) { + this(null, deliveryFormat, format, resolution, isVideoOnly); } - public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution, boolean isVideoOnly) { - super(url, torrentUrl, format); + public VideoStream(String torrentUrl, DeliveryFormat deliveryFormat, + MediaFormat format, String resolution, boolean isVideoOnly) { + super(torrentUrl, deliveryFormat, format); this.resolution = resolution; this.isVideoOnly = isVideoOnly; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java deleted file mode 100644 index b8b191ef7f..0000000000 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/DashMpdParser.java +++ /dev/null @@ -1,216 +0,0 @@ -package org.schabi.newpipe.extractor.utils; - -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.stream.AudioStream; -import org.schabi.newpipe.extractor.stream.Stream; -import org.schabi.newpipe.extractor.stream.StreamInfo; -import org.schabi.newpipe.extractor.stream.VideoStream; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -/* - * Created by Christian Schabesberger on 02.02.16. - * - * Copyright (C) Christian Schabesberger 2016 - * DashMpdParser.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - -public class DashMpdParser { - - private DashMpdParser() { - } - - public static class DashMpdParsingException extends ParsingException { - DashMpdParsingException(String message, Exception e) { - super(message, e); - } - } - - public static class ParserResult { - private final List videoStreams; - private final List audioStreams; - private final List videoOnlyStreams; - - private final List segmentedVideoStreams; - private final List segmentedAudioStreams; - private final List segmentedVideoOnlyStreams; - - - public ParserResult(List videoStreams, - List audioStreams, - List videoOnlyStreams, - List segmentedVideoStreams, - List segmentedAudioStreams, - List segmentedVideoOnlyStreams) { - this.videoStreams = videoStreams; - this.audioStreams = audioStreams; - this.videoOnlyStreams = videoOnlyStreams; - this.segmentedVideoStreams = segmentedVideoStreams; - this.segmentedAudioStreams = segmentedAudioStreams; - this.segmentedVideoOnlyStreams = segmentedVideoOnlyStreams; - } - - public List getVideoStreams() { - return videoStreams; - } - - public List getAudioStreams() { - return audioStreams; - } - - public List getVideoOnlyStreams() { - return videoOnlyStreams; - } - - public List getSegmentedVideoStreams() { - return segmentedVideoStreams; - } - - public List getSegmentedAudioStreams() { - return segmentedAudioStreams; - } - - public List getSegmentedVideoOnlyStreams() { - return segmentedVideoOnlyStreams; - } - } - - /** - * Will try to download (using {@link StreamInfo#getDashMpdUrl()}) and parse the dash manifest, - * then it will search for any stream that the ItagItem has (by the id). - *

- * It has video, video only and audio streams and will only add to the list if it don't - * find a similar stream in the respective lists (calling {@link Stream#equalStats}). - *

- * Info about dash MPD can be found here - * - * @param streamInfo where the parsed streams will be added - * @see www.brendanlog.com - */ - public static ParserResult getStreams(final StreamInfo streamInfo) - throws DashMpdParsingException, ReCaptchaException { - String dashDoc; - Downloader downloader = NewPipe.getDownloader(); - try { - dashDoc = downloader.get(streamInfo.getDashMpdUrl()).responseBody(); - } catch (IOException ioe) { - throw new DashMpdParsingException("Could not get dash mpd: " + streamInfo.getDashMpdUrl(), ioe); - } - - try { - final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - final DocumentBuilder builder = factory.newDocumentBuilder(); - final InputStream stream = new ByteArrayInputStream(dashDoc.getBytes()); - - final Document doc = builder.parse(stream); - final NodeList representationList = doc.getElementsByTagName("Representation"); - - final List videoStreams = new ArrayList<>(); - final List audioStreams = new ArrayList<>(); - final List videoOnlyStreams = new ArrayList<>(); - - final List segmentedVideoStreams = new ArrayList<>(); - final List segmentedAudioStreams = new ArrayList<>(); - final List segmentedVideoOnlyStreams = new ArrayList<>(); - - for (int i = 0; i < representationList.getLength(); i++) { - final Element representation = (Element) representationList.item(i); - try { - final String mimeType = ((Element) representation.getParentNode()).getAttribute("mimeType"); - final String id = representation.getAttribute("id"); - final String url = representation.getElementsByTagName("BaseURL").item(0).getTextContent(); - final ItagItem itag = ItagItem.getItag(Integer.parseInt(id)); - final Node segmentationList = representation.getElementsByTagName("SegmentList").item(0); - - // if SegmentList is not null this means that BaseUrl is not representing the url to the stream. - // instead we need to add the "media=" value from the tags inside the - // tag in order to get a full working url. However each of these is just pointing to a part of the - // video, so we can not return a URL with a working stream here. - // Instead of putting those streams into the list of regular stream urls wie put them in a - // for example "segmentedVideoStreams" list. - if (itag != null) { - final MediaFormat mediaFormat = MediaFormat.getFromMimeType(mimeType); - - if (itag.itagType.equals(ItagItem.ItagType.AUDIO)) { - if (segmentationList == null) { - final AudioStream audioStream = new AudioStream(url, mediaFormat, itag.avgBitrate); - if (!Stream.containSimilarStream(audioStream, streamInfo.getAudioStreams())) { - audioStreams.add(audioStream); - } - } else { - segmentedAudioStreams.add( - new AudioStream(id, mediaFormat, itag.avgBitrate)); - } - } else { - boolean isVideoOnly = itag.itagType.equals(ItagItem.ItagType.VIDEO_ONLY); - - if (segmentationList == null) { - final VideoStream videoStream = new VideoStream(url, - mediaFormat, - itag.resolutionString, - isVideoOnly); - - if (isVideoOnly) { - if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoOnlyStreams())) { - videoOnlyStreams.add(videoStream); - } - } else if (!Stream.containSimilarStream(videoStream, streamInfo.getVideoStreams())) { - videoStreams.add(videoStream); - } - } else { - final VideoStream videoStream = new VideoStream(id, - mediaFormat, - itag.resolutionString, - isVideoOnly); - - if (isVideoOnly) { - segmentedVideoOnlyStreams.add(videoStream); - } else { - segmentedVideoStreams.add(videoStream); - } - } - } - } - } catch (Exception ignored) { - } - } - return new ParserResult( - videoStreams, - audioStreams, - videoOnlyStreams, - segmentedVideoStreams, - segmentedAudioStreams, - segmentedVideoOnlyStreams); - } catch (Exception e) { - throw new DashMpdParsingException("Could not parse Dash mpd", e); - } - } -} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSChannelExtractorTest.kt b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSChannelExtractorTest.kt new file mode 100644 index 0000000000..9783946193 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSChannelExtractorTest.kt @@ -0,0 +1,107 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import org.junit.Assert +import org.junit.BeforeClass +import org.junit.Test +import org.schabi.newpipe.DownloaderTestImpl +import org.schabi.newpipe.extractor.ExtractorAsserts +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.channel.ChannelExtractor +import org.schabi.newpipe.extractor.exceptions.ParsingException +import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest +import org.schabi.newpipe.extractor.services.DefaultTests + +@Suppress("unused") +class BSChannelExtractorTest { + + class BSDefaultChannelExtractorTest : BaseChannelExtractorTest { + + companion object { + @JvmStatic + private lateinit var extractor: ChannelExtractor + + @JvmStatic + @BeforeClass + @Throws(Exception::class) + fun setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()) + extractor = ServiceList.BBC_SOUNDS.getChannelExtractor("https://www.bbc.co.uk/sounds/series/p02nrsln") + extractor.fetchPage() + } + } + + + @Test + override fun testServiceId() { + Assert.assertEquals(ServiceList.BBC_SOUNDS.serviceId.toLong(), extractor.serviceId.toLong()) + } + + @Test + @Throws(Exception::class) + override fun testName() { + Assert.assertEquals("Football Daily", extractor.name) + } + + @Test + @Throws(Exception::class) + override fun testId() { + Assert.assertEquals("p02nrsln", extractor.id) + } + + @Test + @Throws(ParsingException::class) + override fun testUrl() { + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/playable?container=p02nrsln&offset=0", extractor.url) + } + + @Test + @Throws(ParsingException::class) + override fun testOriginalUrl() { + Assert.assertEquals("https://bbc.co.uk/sounds/brand/p02nrsln", extractor.originalUrl) + } + + @Test + @Throws(Exception::class) + override fun testRelatedItems() { + DefaultTests.defaultTestRelatedItems(extractor) + } + + @Test + override fun testMoreRelatedItems() { + DefaultTests.defaultTestMoreItems(extractor) + } + + @Test + @Throws(Exception::class) + override fun testDescription() { + val description = extractor.description + Assert.assertTrue(description, description.contains("latest football news")) + } + + @Test + @Throws(Exception::class) + override fun testAvatarUrl() { + val avatarUrl = extractor.avatarUrl + ExtractorAsserts.assertIsSecureUrl(avatarUrl) + } + + override fun testBannerUrl() { + Assert.assertEquals("", extractor.bannerUrl) + } + + @Test + @Throws(Exception::class) + override fun testFeedUrl() { + Assert.assertEquals("", extractor.feedUrl) + } + + @Test + @Throws(Exception::class) + override fun testSubscriberCount() { + val subscribers = extractor.subscriberCount + Assert.assertEquals(-1, subscribers) + } + } + +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSKioskExtractorTest.kt b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSKioskExtractorTest.kt new file mode 100644 index 0000000000..87b7dcd027 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSKioskExtractorTest.kt @@ -0,0 +1,73 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import org.junit.Assert +import org.junit.BeforeClass +import org.junit.Test +import org.schabi.newpipe.DownloaderTestImpl +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.exceptions.ParsingException +import org.schabi.newpipe.extractor.services.BaseListExtractorTest +import org.schabi.newpipe.extractor.services.DefaultTests + +@Suppress("unused") +class BSKioskExtractorTest { + + class BSRadioKioskTest : BaseListExtractorTest { + + companion object { + @JvmStatic + private lateinit var extractor: BSKioskExtractor + + @JvmStatic + @BeforeClass + @Throws(Exception::class) + fun setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()) + extractor = ServiceList.BBC_SOUNDS.kioskList.getExtractorById("Radio (Live)", null) as BSKioskExtractor + extractor.fetchPage() + } + } + + + @Test + override fun testServiceId() { + Assert.assertEquals(ServiceList.BBC_SOUNDS.serviceId.toLong(), extractor.serviceId.toLong()) + } + + @Test + @Throws(Exception::class) + override fun testName() { + Assert.assertEquals("Radio (Live)", extractor.name) + } + + @Test + @Throws(Exception::class) + override fun testId() { + Assert.assertEquals("Radio (Live)", extractor.id) + } + + @Test + @Throws(ParsingException::class) + override fun testUrl() { + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/networks/playable", extractor.url) + } + + @Test + @Throws(ParsingException::class) + override fun testOriginalUrl() { + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/networks/playable", extractor.originalUrl) + } + + @Test + @Throws(Exception::class) + override fun testRelatedItems() { + DefaultTests.defaultTestRelatedItems(extractor) + } + + @Test + override fun testMoreRelatedItems() { + Assert.assertEquals("", extractor.nextPageUrl) + } + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkExtractorTest.kt b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkExtractorTest.kt new file mode 100644 index 0000000000..9b4ab46de1 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkExtractorTest.kt @@ -0,0 +1,107 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import org.junit.Assert +import org.junit.BeforeClass +import org.junit.Test +import org.schabi.newpipe.DownloaderTestImpl +import org.schabi.newpipe.extractor.ExtractorAsserts +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.channel.ChannelExtractor +import org.schabi.newpipe.extractor.exceptions.ParsingException +import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest +import org.schabi.newpipe.extractor.services.DefaultTests + +@Suppress("unused") +class BSNetworkExtractorTest { + + class BSBBC1ExtractorTest: BaseChannelExtractorTest { + + companion object { + @JvmStatic + private lateinit var extractor: ChannelExtractor + + @JvmStatic + @BeforeClass + @Throws(Exception::class) + fun setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()) + extractor = ServiceList.BBC_SOUNDS.getChannelExtractor("https://rms.api.bbc.co.uk/v2/programmes/playable?network=bbc_radio_one&offset=0") + extractor.fetchPage() + } + } + + + @Test + override fun testServiceId() { + Assert.assertEquals(ServiceList.BBC_SOUNDS.serviceId.toLong(), extractor.serviceId.toLong()) + } + + @Test + @Throws(Exception::class) + override fun testName() { + Assert.assertEquals("Radio 1", extractor.name) + } + + @Test + @Throws(Exception::class) + override fun testId() { + Assert.assertEquals("network:bbc_radio_one", extractor.id) + } + + @Test + @Throws(ParsingException::class) + override fun testUrl() { + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/playable?network=bbc_radio_one&offset=0", extractor.url) + } + + @Test + @Throws(ParsingException::class) + override fun testOriginalUrl() { + Assert.assertEquals("https://www.bbc.co.uk/radio1", extractor.originalUrl) + } + + @Test + @Throws(Exception::class) + override fun testRelatedItems() { + DefaultTests.defaultTestRelatedItems(extractor) + } + + @Test + override fun testMoreRelatedItems() { + DefaultTests.defaultTestMoreItems(extractor) + } + + @Test + @Throws(Exception::class) + override fun testDescription() { + Assert.assertEquals("", extractor.description) + } + + @Test + @Throws(Exception::class) + override fun testAvatarUrl() { + val avatarUrl = extractor.avatarUrl + ExtractorAsserts.assertIsSecureUrl(avatarUrl) + Assert.assertEquals("https://sounds.files.bbci.co.uk/2.2.4/networks/bbc_radio_one/colour_default.svg", avatarUrl) + } + + override fun testBannerUrl() { + Assert.assertEquals("", extractor.bannerUrl) + } + + @Test + @Throws(Exception::class) + override fun testFeedUrl() { + Assert.assertEquals("", extractor.feedUrl) + } + + @Test + @Throws(Exception::class) + override fun testSubscriberCount() { + val subscribers = extractor.subscriberCount + Assert.assertEquals(-1, subscribers) + } + } + +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkStreamExtractorTest.kt b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkStreamExtractorTest.kt new file mode 100644 index 0000000000..3ca3c056df --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSNetworkStreamExtractorTest.kt @@ -0,0 +1,107 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import org.junit.Assert +import org.junit.BeforeClass +import org.junit.Test +import org.schabi.newpipe.DownloaderTestImpl +import org.schabi.newpipe.extractor.ExtractorAsserts +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.exceptions.ParsingException +import org.schabi.newpipe.extractor.stream.StreamExtractor +import org.schabi.newpipe.extractor.stream.StreamType + +@Suppress("unused") +class BSNetworkStreamExtractorTest { + + class BSDefaultNetworkStreamExtractorTest { + companion object { + @JvmStatic + private lateinit var extractor: StreamExtractor + + @JvmStatic + @BeforeClass + @Throws(Exception::class) + fun setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()) + extractor = ServiceList.BBC_SOUNDS.getStreamExtractor("https://rms.api.bbc.co.uk/v2/networks/bbc_radio_one/playable") + extractor.fetchPage() + } + } + + @Test + fun testCounts() { + Assert.assertEquals(-1, extractor.viewCount) + Assert.assertEquals(-1, extractor.likeCount) + Assert.assertEquals(-1, extractor.dislikeCount) + Assert.assertEquals(0, extractor.ageLimit) + } + + @Test + @Throws(ParsingException::class) + fun testGetInvalidTimeStamp() { + Assert.assertTrue(extractor.timeStamp.toString() + "", + extractor.timeStamp <= 0) + } + + @Test + @Throws(ParsingException::class) + fun testGetTitle() { + Assert.assertTrue(extractor.name.startsWith("Radio 1")) + } + + @Test + @Throws(ParsingException::class) + fun testGetDescription() { + Assert.assertNotNull("description is missing", extractor.description.content) + } + + @Test + @Throws(ParsingException::class) + fun testGetUploaderName() { + Assert.assertEquals("Radio 1", extractor.uploaderName) + } + + @Test + @Throws(ParsingException::class) + fun testGetUploaderUrl() { + ExtractorAsserts.assertIsSecureUrl(extractor.uploaderUrl) + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/playable?network=bbc_radio_one&offset=0", extractor.uploaderUrl) + + } + + @Test + @Throws(ParsingException::class) + fun testGetUploaderAvatarUrl() { + ExtractorAsserts.assertIsSecureUrl(extractor.uploaderAvatarUrl) + } + + @Test + @Throws(ParsingException::class) + fun testGetLength() { + Assert.assertEquals(-1, extractor.length) + } + + @Test + @Throws(ParsingException::class) + fun testGetThumbnailUrl() { + ExtractorAsserts.assertIsSecureUrl(extractor.thumbnailUrl) + } + + @Test + @Throws(ParsingException::class) + fun testStreamType() { + Assert.assertTrue(extractor.streamType == StreamType.AUDIO_LIVE_STREAM) + } + + @Test + fun testGetAudioStreams() { + Assert.assertTrue(extractor.audioStreams.isNotEmpty() && extractor.audioStreams[0].deliveryFormat != null) + } + + @Test + fun testGetDashMpdUrl() { + Assert.assertFalse(extractor.dashMpdUrl.isBlank()) + } + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSSearchExtractorTest.kt b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSSearchExtractorTest.kt new file mode 100644 index 0000000000..610f225286 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSSearchExtractorTest.kt @@ -0,0 +1,65 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import org.junit.BeforeClass +import org.schabi.newpipe.DownloaderTestImpl +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.StreamingService +import org.schabi.newpipe.extractor.search.SearchExtractor +import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest + +@Suppress("unused") +class BSSearchExtractorTest { + class BSDefaultSearchExtractorTest : DefaultSearchExtractorTest() { + override fun extractor(): SearchExtractor { + return extractor + } + + override fun expectedService(): StreamingService { + return ServiceList.BBC_SOUNDS + } + + override fun expectedName(): String { + return QUERY + } + + override fun expectedId(): String { + return QUERY + } + + override fun expectedUrlContains(): String { + return "/search?q=$QUERY" + } + + override fun expectedOriginalUrlContains(): String { + return "/search?q=$QUERY" + } + + override fun expectedSearchString(): String { + return QUERY + } + + override fun expectedSearchSuggestion(): String? { + return null + } + + override fun expectedHasMoreItems(): Boolean { + return false + } + + companion object { + @JvmStatic + private lateinit var extractor: SearchExtractor + private const val QUERY = "breakfast" + + @JvmStatic + @BeforeClass + @Throws(Exception::class) + fun setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()) + extractor = ServiceList.BBC_SOUNDS.getSearchExtractor(QUERY) + extractor.fetchPage() + } + } + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSStreamExtractorTest.kt b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSStreamExtractorTest.kt new file mode 100644 index 0000000000..b4b3d60f2c --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/extractors/BSStreamExtractorTest.kt @@ -0,0 +1,128 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.extractors + +import org.junit.Assert +import org.junit.BeforeClass +import org.junit.Test +import org.schabi.newpipe.DownloaderTestImpl +import org.schabi.newpipe.extractor.ExtractorAsserts +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.exceptions.ParsingException +import org.schabi.newpipe.extractor.stream.StreamExtractor +import org.schabi.newpipe.extractor.stream.StreamType +import java.text.ParseException +import java.text.SimpleDateFormat +import java.util.* + +@Suppress("unused") +class BSStreamExtractorTest { + + class BSDefaultStreamExtractorTest { + companion object { + @JvmStatic + private lateinit var extractor: StreamExtractor + + @JvmStatic + @BeforeClass + @Throws(Exception::class) + fun setUp() { + NewPipe.init(DownloaderTestImpl.getInstance()) + extractor = ServiceList.BBC_SOUNDS.getStreamExtractor("https://www.bbc.co.uk/sounds/play/p00cbxy3") + extractor.fetchPage() + } + } + + @Test + fun testCounts() { + Assert.assertEquals(-1, extractor.viewCount) + Assert.assertEquals(-1, extractor.likeCount) + Assert.assertEquals(-1, extractor.dislikeCount) + Assert.assertEquals(0, extractor.ageLimit) + } + + @Test + @Throws(ParsingException::class, ParseException::class) + fun testGetUploadDate() { + val instance = Calendar.getInstance() + val sdf = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'") + sdf.timeZone = TimeZone.getTimeZone("GMT") + instance.time = sdf.parse("2010-11-24T16:51:21Z") + Assert.assertEquals(instance, extractor.uploadDate!!.date()) + } + + @Test + fun testGetTextualUploadDate() { + Assert.assertEquals("2010-11-24T16:51:21Z", extractor.textualUploadDate) + } + + @Test + @Throws(ParsingException::class) + fun testGetInvalidTimeStamp() { + Assert.assertTrue(extractor.timeStamp.toString() + "", + extractor.timeStamp <= 0) + } + + @Test + @Throws(ParsingException::class) + fun testGetTitle() { + Assert.assertEquals("The Roman Way | 4. Filling the Mind", extractor.name) + } + + @Test + @Throws(ParsingException::class) + fun testGetDescription() { + Assert.assertNotNull("description is missing", extractor.description.content) + } + + @Test + @Throws(ParsingException::class) + fun testGetUploaderName() { + Assert.assertEquals("Radio 4", extractor.uploaderName) + } + + @Test + @Throws(ParsingException::class) + fun testGetUploaderUrl() { + ExtractorAsserts.assertIsSecureUrl(extractor.uploaderUrl) + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/playable?network=bbc_radio_four&offset=0", extractor.uploaderUrl) + + } + + @Test + @Throws(ParsingException::class) + fun testGetUploaderAvatarUrl() { + ExtractorAsserts.assertIsSecureUrl(extractor.uploaderAvatarUrl) + } + + @Test + @Throws(ParsingException::class) + fun testGetLength() { + Assert.assertEquals(1800, extractor.length) + } + + @Test + @Throws(ParsingException::class) + fun testGetThumbnailUrl() { + ExtractorAsserts.assertIsSecureUrl(extractor.thumbnailUrl) + } + + @Test + @Throws(ParsingException::class) + fun testStreamType() { + Assert.assertTrue(extractor.streamType == StreamType.AUDIO_STREAM) + } + + @Test + fun testGetAudioStreams() { + Assert.assertTrue(extractor.audioStreams.isNotEmpty() && extractor.audioStreams[0].deliveryFormat != null) + } + + @Test + fun testGetSubChannelStuff() { + Assert.assertEquals("The Roman Way", extractor.subChannelName) + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/playable?container=b00wnmlf&offset=0", extractor.subChannelUrl) + Assert.assertEquals("https://ichef.bbci.co.uk/images/ic/320x320/p01l672f.jpg", extractor.subChannelAvatarUrl) + } + } + +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSChannelLHFTest.kt b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSChannelLHFTest.kt new file mode 100644 index 0000000000..653fb14e16 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSChannelLHFTest.kt @@ -0,0 +1,45 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler + +import org.junit.Assert +import org.junit.BeforeClass +import org.junit.Test +import org.schabi.newpipe.extractor.exceptions.ParsingException + +class BSChannelLHFTest { + + companion object { + @JvmStatic + private lateinit var linkHandler: BSChannelLinkHandlerFactory + + @JvmStatic + @BeforeClass + fun setUp() { + linkHandler = BSChannelLinkHandlerFactory + } + } + + @Test + @Throws(ParsingException::class) + fun acceptUrlTest() { + Assert.assertTrue(linkHandler.acceptUrl("https://www.bbc.co.uk/sounds/series/p02nrsln")) + Assert.assertTrue(linkHandler.acceptUrl("https://rms.api.bbc.co.uk/v2/programmes/playable?container=p02nrsln&offset=0")) + Assert.assertTrue(linkHandler.acceptUrl("https://rms.api.bbc.co.uk/v2/programmes/playable?network=bbc_radio_one&offset=0")) + } + + @Test + @Throws(ParsingException::class) + fun getIdFromUrl() { + Assert.assertEquals("p02nrsln", linkHandler.fromUrl("https://www.bbc.co.uk/sounds/series/p02nrsln").id) + Assert.assertEquals("p02nrsln", linkHandler.fromUrl("https://rms.api.bbc.co.uk/v2/programmes/playable?container=p02nrsln&offset=0").id) + Assert.assertEquals("network:bbc_radio_one", linkHandler.fromUrl("https://rms.api.bbc.co.uk/v2/programmes/playable?network=bbc_radio_one&offset=0").id) + } + + @Test + @Throws(ParsingException::class) + fun getUrlFromId() { + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/playable?container=p02nrsln&offset=0", linkHandler.fromId("p02nrsln").url) + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/playable?container=p02nrsln&offset=0", linkHandler.fromId("p02nrsln").url) + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/playable?network=bbc_radio_one&offset=0", linkHandler.fromId("network:bbc_radio_one").url) + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/playable?network=bbc_radio_one&offset=0", linkHandler.fromNetworkId("network:bbc_radio_one").url) + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSKioskLHFTest.kt b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSKioskLHFTest.kt new file mode 100644 index 0000000000..07a495b848 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSKioskLHFTest.kt @@ -0,0 +1,41 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler + +import org.junit.Assert +import org.junit.BeforeClass +import org.junit.Test +import org.schabi.newpipe.extractor.exceptions.ParsingException + +class BSKioskLHFTest { + + companion object { + @JvmStatic + private lateinit var linkHandler: BSKioskLinkHandlerFactory + + @JvmStatic + @BeforeClass + fun setUp() { + linkHandler = BSKioskLinkHandlerFactory + } + } + + @Test + @Throws(ParsingException::class) + fun acceptUrlTest() { + Assert.assertTrue(linkHandler.acceptUrl("https://rms.api.bbc.co.uk/v2/networks/playable")) + Assert.assertTrue(linkHandler.acceptUrl("https://rms.api.bbc.co.uk/v2/programmes/playable?category=mixes&offset=0")) + } + + @Test + @Throws(ParsingException::class) + fun getIdFromUrl() { + Assert.assertEquals("Radio (Live)", linkHandler.fromUrl("https://rms.api.bbc.co.uk/v2/networks/playable").id) + Assert.assertEquals("Mixes", linkHandler.fromUrl("https://rms.api.bbc.co.uk/v2/programmes/playable?category=mixes&offset=0").id) + } + + @Test + @Throws(ParsingException::class) + fun getUrlFromId() { + Assert.assertEquals( "https://rms.api.bbc.co.uk/v2/networks/playable",linkHandler.fromId("Radio (Live)").url) + Assert.assertEquals( "https://rms.api.bbc.co.uk/v2/programmes/playable?category=mixes&offset=0", linkHandler.fromId("Mixes").url) + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSSearchQHFTest.kt b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSSearchQHFTest.kt new file mode 100644 index 0000000000..585cf41660 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSSearchQHFTest.kt @@ -0,0 +1,24 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler + +import org.junit.Assert +import org.junit.BeforeClass +import org.junit.Test + +class BSSearchQHFTest { + + companion object { + @JvmStatic + private lateinit var linkHandler: BSSearchQueryHandlerFactory + + @JvmStatic + @BeforeClass + fun setUp() { + linkHandler = BSSearchQueryHandlerFactory + } + } + + @Test + fun testSearchQueryUrl() { + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/experience/inline/search?q=breakfast", linkHandler.fromQuery("breakfast").url) + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSStreamLHFTest.kt b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSStreamLHFTest.kt new file mode 100644 index 0000000000..23db793c11 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bbc_sounds/linkHandler/BSStreamLHFTest.kt @@ -0,0 +1,45 @@ +package org.schabi.newpipe.extractor.services.bbc_sounds.linkHandler + +import org.junit.Assert +import org.junit.BeforeClass +import org.junit.Test +import org.schabi.newpipe.extractor.exceptions.ParsingException + +class BSStreamLHFTest { + + companion object { + @JvmStatic + private lateinit var linkHandler: BSStreamLinkHandlerFactory + + @JvmStatic + @BeforeClass + fun setUp() { + linkHandler = BSStreamLinkHandlerFactory + } + } + + @Test + @Throws(ParsingException::class) + fun acceptUrlTest() { + Assert.assertTrue(linkHandler.acceptUrl("https://www.bbc.co.uk/sounds/play/p00cbxy3")) + Assert.assertTrue(linkHandler.acceptUrl("https://rms.api.bbc.co.uk/v2/programmes/p00cbxy3/playable")) + Assert.assertTrue(linkHandler.acceptUrl("https://rms.api.bbc.co.uk/v2/networks/bbc_radio_one/playable")) + } + + @Test + @Throws(ParsingException::class) + fun getIdFromUrl() { + Assert.assertEquals("p00cbxy3", linkHandler.fromUrl("https://www.bbc.co.uk/sounds/play/p00cbxy3").id) + Assert.assertEquals("p00cbxy3", linkHandler.fromUrl("https://rms.api.bbc.co.uk/v2/programmes/p00cbxy3/playable").id) + Assert.assertEquals("urn:bbc:radio:network:bbc_radio_one", linkHandler.fromUrl("https://rms.api.bbc.co.uk/v2/networks/bbc_radio_one/playable").id) + } + + @Test + @Throws(ParsingException::class) + fun getUrlFromId() { + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/p00cbxy3/playable", linkHandler.fromId("p00cbxy3").url) + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/programmes/p00cbxy3/playable", linkHandler.fromId("p00cbxy3").url) + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/networks/bbc_radio_one/playable", linkHandler.fromId("urn:bbc:radio:network:bbc_radio_one").url) + Assert.assertEquals("https://rms.api.bbc.co.uk/v2/networks/bbc_radio_one/playable", linkHandler.fromNetworkId("urn:bbc:radio:network:bbc_radio_one").url) + } +} \ No newline at end of file diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java index 95c882290b..5f53e3c316 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/media_ccc/MediaCCCStreamExtractorTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.media_ccc.extractors.MediaCCCStreamExtractor; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryFormat; import org.schabi.newpipe.extractor.stream.VideoStream; import java.text.ParseException; @@ -90,7 +91,9 @@ public void testVideoStreams() throws Exception { List videoStreamList = extractor.getVideoStreams(); assertEquals(4, videoStreamList.size()); for (VideoStream stream : videoStreamList) { - assertIsSecureUrl(stream.getUrl()); + if (stream.getDeliveryFormat() instanceof DeliveryFormat.Direct) { + assertIsSecureUrl(((DeliveryFormat.Direct) stream.getDeliveryFormat()).getUrl()); + } } } @@ -99,7 +102,9 @@ public void testAudioStreams() throws Exception { List audioStreamList = extractor.getAudioStreams(); assertEquals(2, audioStreamList.size()); for (AudioStream stream : audioStreamList) { - assertIsSecureUrl(stream.getUrl()); + if (stream.getDeliveryFormat() instanceof DeliveryFormat.Direct) { + assertIsSecureUrl(((DeliveryFormat.Direct) stream.getDeliveryFormat()).getUrl()); + } } } @@ -176,7 +181,9 @@ public void testVideoStreams() throws Exception { List videoStreamList = extractor.getVideoStreams(); assertEquals(8, videoStreamList.size()); for (VideoStream stream : videoStreamList) { - assertIsSecureUrl(stream.getUrl()); + if (stream.getDeliveryFormat() instanceof DeliveryFormat.Direct) { + assertIsSecureUrl(((DeliveryFormat.Direct) stream.getDeliveryFormat()).getUrl()); + } } } @@ -185,7 +192,9 @@ public void testAudioStreams() throws Exception { List audioStreamList = extractor.getAudioStreams(); assertEquals(2, audioStreamList.size()); for (AudioStream stream : audioStreamList) { - assertIsSecureUrl(stream.getUrl()); + if (stream.getDeliveryFormat() instanceof DeliveryFormat.Direct) { + assertIsSecureUrl(((DeliveryFormat.Direct) stream.getDeliveryFormat()).getUrl()); + } } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index 3710b4ee1e..8c3df6e559 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -9,6 +9,7 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory; +import org.schabi.newpipe.extractor.stream.DeliveryFormat; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -119,8 +120,9 @@ public void testGetVideoStreams() throws IOException, ExtractionException { assertTrue(Integer.toString(streams.size()), streams.size() > 0); for (VideoStream s : streams) { - assertTrue(s.getUrl(), - s.getUrl().contains(HTTPS)); + if (s.getDeliveryFormat() instanceof DeliveryFormat.Direct) { + assertIsSecureUrl(((DeliveryFormat.Direct) s.getDeliveryFormat()).getUrl()); + } assertTrue(s.resolution.length() > 0); assertTrue(Integer.toString(s.getFormatId()), 0 <= s.getFormatId() && s.getFormatId() <= 0x100); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java index 0d36fd9a83..283b8f24d7 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.stream.DeliveryFormat; import org.schabi.newpipe.extractor.stream.Frameset; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -177,7 +178,9 @@ public void testGetAudioStreams() throws ExtractionException { @Test public void testGetVideoStreams() throws ExtractionException { for (VideoStream s : extractor.getVideoStreams()) { - assertIsSecureUrl(s.url); + if (s.getDeliveryFormat() instanceof DeliveryFormat.Direct) { + assertIsSecureUrl(((DeliveryFormat.Direct) s.getDeliveryFormat()).getUrl()); + } assertTrue(s.resolution.length() > 0); assertTrue(Integer.toString(s.getFormatId()), 0 <= s.getFormatId() && s.getFormatId() <= 0x100); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java index 0e17fe8d86..327ad9493d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import org.schabi.newpipe.extractor.stream.DeliveryFormat; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; @@ -102,7 +103,9 @@ public void testGetAudioStreams() throws ExtractionException { @Test public void testGetVideoStreams() throws ExtractionException { for (VideoStream s : extractor.getVideoStreams()) { - assertIsSecureUrl(s.url); + if (s.getDeliveryFormat() instanceof DeliveryFormat.Direct) { + assertIsSecureUrl(((DeliveryFormat.Direct) s.getDeliveryFormat()).getUrl()); + } assertTrue(s.resolution.length() > 0); assertTrue(Integer.toString(s.getFormatId()), 0 <= s.getFormatId() && s.getFormatId() <= 0x100); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java index 5dcd73ecdc..d99adb7a92 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java @@ -13,6 +13,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.stream.DeliveryFormat; import org.schabi.newpipe.extractor.utils.Utils; import java.text.ParseException; @@ -108,7 +109,9 @@ public void testGetAudioStreams() throws ExtractionException { List audioStreams = extractor.getAudioStreams(); assertFalse(audioStreams.isEmpty()); for (AudioStream s : audioStreams) { - assertIsSecureUrl(s.url); + if (s.getDeliveryFormat() instanceof DeliveryFormat.Direct) { + assertIsSecureUrl(((DeliveryFormat.Direct) s.getDeliveryFormat()).getUrl()); + } assertTrue(Integer.toString(s.getFormatId()), 0x100 <= s.getFormatId() && s.getFormatId() < 0x1000); } @@ -117,7 +120,9 @@ public void testGetAudioStreams() throws ExtractionException { @Test public void testGetVideoStreams() throws ExtractionException { for (VideoStream s : extractor.getVideoStreams()) { - assertIsSecureUrl(s.url); + if (s.getDeliveryFormat() instanceof DeliveryFormat.Direct) { + assertIsSecureUrl(((DeliveryFormat.Direct) s.getDeliveryFormat()).getUrl()); + } assertTrue(s.resolution.length() > 0); assertTrue(Integer.toString(s.getFormatId()), 0 <= s.getFormatId() && s.getFormatId() < 0x100);