diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java index a7efea9624..d1c481bb48 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchExtractor.java @@ -25,6 +25,16 @@ public String getSearchString() { return getLinkHandler().getSearchString(); } + /** + * The search suggestion provided by the service. + *

+ * This method also returns the corrected query if + * {@link SearchExtractor#isCorrectedSearch()} is true. + * + * @return a suggestion to another query, the corrected query, or an empty String. + * @throws ParsingException + */ + @Nonnull public abstract String getSearchSuggestion() throws ParsingException; @Override @@ -37,4 +47,14 @@ public SearchQueryHandler getLinkHandler() { public String getName() { return getLinkHandler().getSearchString(); } + + /** + * Tell if the search was corrected by the service (if it's not exactly the search you typed). + *

+ * Example: on YouTube, if you search for "pewdeipie", + * it will give you results for "pewdiepie", then isCorrectedSearch should return true. + * + * @return whether the results comes from a corrected query or not. + */ + public abstract boolean isCorrectedSearch() throws ParsingException; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java index eedce719eb..b2e072ccef 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/search/SearchInfo.java @@ -15,6 +15,7 @@ public class SearchInfo extends ListInfo { private String searchString; private String searchSuggestion; + private boolean isCorrectedSearch; public SearchInfo(int serviceId, SearchQueryHandler qIHandler, @@ -42,7 +43,12 @@ public static SearchInfo getInfo(SearchExtractor extractor) throws ExtractionExc info.addError(e); } try { - info.searchSuggestion = extractor.getSearchSuggestion(); + info.setSearchSuggestion(extractor.getSearchSuggestion()); + } catch (Exception e) { + info.addError(e); + } + try { + info.setIsCorrectedSearch(extractor.isCorrectedSearch()); } catch (Exception e) { info.addError(e); } @@ -64,10 +70,22 @@ public static ListExtractor.InfoItemsPage getMoreItems(StreamingServic // Getter public String getSearchString() { - return searchString; + return this.searchString; } public String getSearchSuggestion() { - return searchSuggestion; + return this.searchSuggestion; + } + + public boolean isCorrectedSearch() { + return this.isCorrectedSearch; + } + + public void setIsCorrectedSearch(boolean isCorrectedSearch) { + this.isCorrectedSearch = isCorrectedSearch; + } + + public void setSearchSuggestion(String searchSuggestion) { + this.searchSuggestion = searchSuggestion; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java index d5ced534b7..914c77497e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCSearchExtractor.java @@ -11,6 +11,7 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; @@ -42,9 +43,15 @@ public MediaCCCSearchExtractor(final StreamingService service, } } + @Nonnull @Override public String getSearchSuggestion() { - return null; + return ""; + } + + @Override + public boolean isCorrectedSearch() { + return false; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java index 61fcdd6df1..352164de30 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeSearchExtractor.java @@ -19,6 +19,7 @@ import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser.RegexException; +import javax.annotation.Nonnull; import java.io.IOException; public class PeertubeSearchExtractor extends SearchExtractor { @@ -35,9 +36,15 @@ public PeertubeSearchExtractor(StreamingService service, SearchQueryHandler link super(service, linkHandler); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - return null; + return ""; + } + + @Override + public boolean isCorrectedSearch() { + return false; } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java index 92730ec46b..24f5987ec1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java @@ -33,9 +33,15 @@ public SoundcloudSearchExtractor(StreamingService service, SearchQueryHandler li super(service, linkHandler); } + @Nonnull @Override public String getSearchSuggestion() { - return null; + return ""; + } + + @Override + public boolean isCorrectedSearch() { + return false; } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index 7e952c94bb..39a6e94d48 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -18,6 +18,7 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -124,15 +125,40 @@ public String getUrl() throws ParsingException { return super.getUrl(); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - final JsonObject didYouMeanRenderer = initialData.getObject("contents").getObject("sectionListRenderer") - .getArray("contents").getObject(0).getObject("itemSectionRenderer") - .getArray("contents").getObject(0).getObject("didYouMeanRenderer"); - if (!didYouMeanRenderer.has("correctedQuery")) { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer.isEmpty()) { + return ""; + } + + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents") + .getObject(0).getObject("didYouMeanRenderer"); + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + + if (!didYouMeanRenderer.isEmpty()) { + return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + } else if (!showingResultsForRenderer.isEmpty()) { + return JsonUtils.getString(showingResultsForRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } else { return ""; } - return getTextFromObject(didYouMeanRenderer.getObject("correctedQuery")); + } + + @Override + public boolean isCorrectedSearch() { + final JsonObject itemSectionRenderer = initialData.getObject("contents").getObject("sectionListRenderer") + .getArray("contents").getObject(0).getObject("itemSectionRenderer"); + if (itemSectionRenderer.isEmpty()) { + return false; + } + + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + return !showingResultsForRenderer.isEmpty(); } @Nonnull diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 732ef09ad9..05c27d53dc 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -2,7 +2,6 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; - import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; @@ -12,10 +11,10 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; import org.schabi.newpipe.extractor.search.SearchExtractor; - -import java.io.IOException; +import org.schabi.newpipe.extractor.utils.JsonUtils; import javax.annotation.Nonnull; +import java.io.IOException; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; @@ -62,17 +61,35 @@ public String getUrl() throws ParsingException { return super.getUrl() + "&gl=" + getExtractorContentCountry().getCountryCode(); } + @Nonnull @Override public String getSearchSuggestion() throws ParsingException { - final JsonObject showingResultsForRenderer = initialData.getObject("contents") + final JsonObject itemSectionRenderer = initialData.getObject("contents") .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") .getObject("sectionListRenderer").getArray("contents").getObject(0) - .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer"); + final JsonObject didYouMeanRenderer = itemSectionRenderer.getArray("contents").getObject(0) + .getObject("didYouMeanRenderer"); + final JsonObject showingResultsForRenderer = itemSectionRenderer.getArray("contents").getObject(0) .getObject("showingResultsForRenderer"); - if (!showingResultsForRenderer.has("correctedQuery")) { + + if (!didYouMeanRenderer.isEmpty()) { + return JsonUtils.getString(didYouMeanRenderer, "correctedQueryEndpoint.searchEndpoint.query"); + } else if (showingResultsForRenderer != null) { + return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + } else { return ""; } - return getTextFromObject(showingResultsForRenderer.getObject("correctedQuery")); + } + + @Override + public boolean isCorrectedSearch() { + final JsonObject showingResultsForRenderer = initialData.getObject("contents") + .getObject("twoColumnSearchResultsRenderer").getObject("primaryContents") + .getObject("sectionListRenderer").getArray("contents").getObject(0) + .getObject("itemSectionRenderer").getArray("contents").getObject(0) + .getObject("showingResultsForRenderer"); + return !showingResultsForRenderer.isEmpty(); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java index e82ad7d0af..ddfa27fb0d 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/BaseSearchExtractorTest.java @@ -4,4 +4,5 @@ public interface BaseSearchExtractorTest extends BaseListExtractorTest { void testSearchString() throws Exception; void testSearchSuggestion() throws Exception; + void testSearchCorrected() throws Exception; } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java index 93c4eac703..b363031d75 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultSearchExtractorTest.java @@ -15,6 +15,10 @@ public abstract class DefaultSearchExtractorTest extends DefaultListExtractorTes public abstract String expectedSearchString(); @Nullable public abstract String expectedSearchSuggestion(); + public boolean isCorrectedSearch() { + return false; + } + @Test @Override public void testSearchString() throws Exception { @@ -31,4 +35,9 @@ public void testSearchSuggestion() throws Exception { assertEquals(expectedSearchSuggestion, extractor().getSearchSuggestion()); } } + + @Test + public void testSearchCorrected() throws Exception { + assertEquals(isCorrectedSearch(), extractor().isCorrectedSearch()); + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java index 420db0adb8..eeac8c49b6 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeMusicSearchExtractorTest.java @@ -150,4 +150,28 @@ public static void setUp() throws Exception { @Nullable @Override public String expectedSearchSuggestion() { return "mega man x3"; } @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } } + + public static class CorrectedSearch extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "duo lipa"; + private static final String EXPECTED_SUGGESTION = "dua lipa"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(YoutubeSearchQueryHandlerFactory.MUSIC_SONGS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedOriginalUrlContains() { return "music.youtube.com/search?q=" + URLEncoder.encode(QUERY); } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean isCorrectedSearch() { return true; } + } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java index d5f2f1af25..84e1501b81 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/search/YoutubeSearchExtractorTest.java @@ -113,6 +113,29 @@ public static void setUp() throws Exception { } public static class Suggestion extends DefaultSearchExtractorTest { + private static SearchExtractor extractor; + private static final String QUERY = "newpip"; + private static final String EXPECTED_SUGGESTION = "newpipe"; + + @BeforeClass + public static void setUp() throws Exception { + NewPipe.init(DownloaderTestImpl.getInstance()); + extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), ""); + extractor.fetchPage(); + } + + @Override public SearchExtractor extractor() { return extractor; } + @Override public StreamingService expectedService() { return YouTube; } + @Override public String expectedName() { return QUERY; } + @Override public String expectedId() { return QUERY; } + @Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } + @Override public String expectedSearchString() { return QUERY; } + @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } + @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + } + + public static class CorrectedSearch extends DefaultSearchExtractorTest { private static SearchExtractor extractor; private static final String QUERY = "pewdeipie"; private static final String EXPECTED_SUGGESTION = "pewdiepie"; @@ -132,8 +155,8 @@ public static void setUp() throws Exception { @Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; } @Override public String expectedSearchString() { return QUERY; } @Nullable @Override public String expectedSearchSuggestion() { return EXPECTED_SUGGESTION; } - @Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; } + @Override public boolean isCorrectedSearch() { return true; } } public static class RandomQueryNoMorePages extends DefaultSearchExtractorTest {