From 90872d4cf6e61f4b9ae5eb38778abb7ccaf487e4 Mon Sep 17 00:00:00 2001 From: Jorge Cortes Date: Mon, 22 Jul 2024 15:36:55 -0500 Subject: [PATCH] Implemented new-sponsorship-detected polling source with no testing --- components/paved/common/constants.mjs | 17 +++ components/paved/common/utils.mjs | 11 ++ components/paved/package.json | 7 +- components/paved/paved.app.mjs | 101 +++++++++++++++++- components/paved/sources/common/polling.mjs | 81 ++++++++++++++ .../new-sponsorship-detected.mjs | 30 ++++++ pnpm-lock.yaml | 5 +- 7 files changed, 244 insertions(+), 8 deletions(-) create mode 100644 components/paved/common/constants.mjs create mode 100644 components/paved/common/utils.mjs create mode 100644 components/paved/sources/common/polling.mjs create mode 100644 components/paved/sources/new-sponsorship-detected/new-sponsorship-detected.mjs diff --git a/components/paved/common/constants.mjs b/components/paved/common/constants.mjs new file mode 100644 index 0000000000000..626f2e55be4c6 --- /dev/null +++ b/components/paved/common/constants.mjs @@ -0,0 +1,17 @@ +const BASE_URL = "https://api.paved.com"; +const LAST_FROM_DATE = "lastFromDate"; +const DEFAULT_LIMIT = 100; +const DEFAULT_MAX = 600; + +const API = { + PUBLISHER_PATH: "/publisher/v1", + ADVERTISER_PATH: "/advertiser/v1", +}; + +export default { + BASE_URL, + API, + DEFAULT_LIMIT, + DEFAULT_MAX, + LAST_FROM_DATE, +}; diff --git a/components/paved/common/utils.mjs b/components/paved/common/utils.mjs new file mode 100644 index 0000000000000..903b2593ed3c2 --- /dev/null +++ b/components/paved/common/utils.mjs @@ -0,0 +1,11 @@ +async function iterate(iterations) { + const items = []; + for await (const item of iterations) { + items.push(item); + } + return items; +} + +export default { + iterate, +}; diff --git a/components/paved/package.json b/components/paved/package.json index 59309fd125437..044b7e31455f3 100644 --- a/components/paved/package.json +++ b/components/paved/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/paved", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Paved Components", "main": "paved.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.0" } -} \ No newline at end of file +} diff --git a/components/paved/paved.app.mjs b/components/paved/paved.app.mjs index 2f952f79f3658..fa8f075e443b8 100644 --- a/components/paved/paved.app.mjs +++ b/components/paved/paved.app.mjs @@ -1,11 +1,102 @@ +import { axios } from "@pipedream/platform"; +import constants from "./common/constants.mjs"; +import utils from "./common/utils.mjs"; + export default { type: "app", app: "paved", - propDefinitions: {}, + propDefinitions: { + slug: { + type: "string", + label: "Newsletter Slug", + description: "Your newsletter slug.", + async options() { + const newsletters = await this.listNewsletters(); + return newsletters.map(({ + name: label, slug: value, + }) => ({ + label, + value, + })); + }, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + getUrl(path, versionPath = constants.API.PUBLISHER_PATH) { + return `${constants.BASE_URL}${versionPath}${path}`; + }, + getHeaders(headers) { + return { + ...headers, + Authorization: `Token ${this.$auth.api_key}`, + }; + }, + _makeRequest({ + $ = this, path, headers, ...args + } = {}) { + return axios($, { + ...args, + url: this.getUrl(path), + headers: this.getHeaders(headers), + }); + }, + listNewsletters(args = {}) { + return this._makeRequest({ + path: "/newsletters", + ...args, + }); + }, + listSponsorships({ + slug, ...args + } = {}) { + return this._makeRequest({ + path: `/newsletters/${slug}/sponsorships`, + ...args, + }); + }, + async *getIterations({ + resourcesFn, resourcesFnArgs, + fromDate, + max = constants.DEFAULT_MAX, + }) { + let resourcesCount = 0; + + while (true) { + const nextResources = + await resourcesFn({ + ...resourcesFnArgs, + params: { + ...resourcesFnArgs?.params, + limit: constants.DEFAULT_LIMIT, + from_date: fromDate, + }, + }); + + if (!nextResources?.length) { + console.log("No more resources found"); + return; + } + + for (const resource of nextResources) { + yield resource; + resourcesCount += 1; + + if (resourcesCount >= max) { + console.log("Reached max resources"); + return; + } + } + + if (nextResources.length < constants.DEFAULT_LIMIT) { + console.log("No next page found"); + return; + } + + fromDate = nextResources[nextResources.length - 1].run_date; + } + }, + paginate(args = {}) { + return utils.iterate(this.getIterations(args)); }, }, -}; \ No newline at end of file +}; diff --git a/components/paved/sources/common/polling.mjs b/components/paved/sources/common/polling.mjs new file mode 100644 index 0000000000000..aaf709400d743 --- /dev/null +++ b/components/paved/sources/common/polling.mjs @@ -0,0 +1,81 @@ +import { + ConfigurationError, + DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; +import app from "../../paved.app.mjs"; +import constants from "../../common/constants.mjs"; + +export default { + props: { + app, + db: "$.service.db", + timer: { + type: "$.interface.timer", + label: "Polling Schedule", + description: "How often to poll the API", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + slug: { + propDefinition: [ + app, + "slug", + ], + }, + }, + methods: { + generateMeta() { + throw new ConfigurationError("generateMeta is not implemented"); + }, + setLastFromDate(value) { + this.db.set(constants.LAST_FROM_DATE, value); + }, + getLastFromDate() { + return this.db.get(constants.LAST_FROM_DATE); + }, + getResourcesFn() { + throw new ConfigurationError("getResourcesFn is not implemented"); + }, + getResourcesFnArgs() { + throw new ConfigurationError("getResourcesFnArgs is not implemented"); + }, + processResource(resource) { + const meta = this.generateMeta(resource); + this.$emit(resource, meta); + }, + async processResources(resources) { + const { + setLastFromDate, + processResource, + } = this; + + const [ + lastResource, + ] = Array.from(resources).reverse(); + + if (lastResource?.run_date) { + setLastFromDate(lastResource.run_date); + } + + Array.from(resources) + .forEach(processResource); + }, + }, + async run() { + const { + app, + getResourcesFn, + getResourcesFnArgs, + processResources, + } = this; + + const resources = await app.paginate({ + resourcesFn: getResourcesFn(), + resourcesFnArgs: getResourcesFnArgs(), + fromDate: this.getLastFromDate(), + }); + + processResources(resources); + }, +}; diff --git a/components/paved/sources/new-sponsorship-detected/new-sponsorship-detected.mjs b/components/paved/sources/new-sponsorship-detected/new-sponsorship-detected.mjs new file mode 100644 index 0000000000000..e1b98d8244ebd --- /dev/null +++ b/components/paved/sources/new-sponsorship-detected/new-sponsorship-detected.mjs @@ -0,0 +1,30 @@ +import common from "../common/polling.mjs"; + +export default { + ...common, + key: "paved-new-sponsorship-detected", + name: "New Sponsorship Detected", + description: "Emit new event when a sponsorship is detected on a newsletter similar to yours. [See the documentation](https://docs.paved.com/list-sponsorships)", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getResourcesFn() { + return this.app.listSponsorships; + }, + getResourcesFnArgs() { + return { + debug: true, + slug: this.slug, + }; + }, + generateMeta(resource) { + return { + id: resource.id, + summary: `New Sponsorthip: ${resource.advertiser_name}`, + ts: Date.parse(resource.run_date), + }; + }, + }, +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 006314933877f..81390356db3db 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6502,7 +6502,10 @@ importers: url: 0.11.3 components/paved: - specifiers: {} + specifiers: + '@pipedream/platform': ^3.0.0 + dependencies: + '@pipedream/platform': 3.0.0 components/payhere: specifiers: