Implemented new-sponsorship-detected polling source with no testing

Author: jcortes

components/paved/common/constants.mjs

@@ -0,0 +1,17 @@
+const BASE_URL = "https://api.paved.com";
+const LAST_FROM_DATE = "lastFromDate";
+const DEFAULT_LIMIT = 100;
+const DEFAULT_MAX = 600;
+
+const API = {
+  PUBLISHER_PATH: "/publisher/v1",
+  ADVERTISER_PATH: "/advertiser/v1",
+};
+
+export default {
+  BASE_URL,
+  API,
+  DEFAULT_LIMIT,
+  DEFAULT_MAX,
+  LAST_FROM_DATE,
+};

components/paved/common/utils.mjs

@@ -0,0 +1,11 @@
+async function iterate(iterations) {
+  const items = [];
+  for await (const item of iterations) {
+    items.push(item);
+  }
+  return items;
+}
+
+export default {
+  iterate,
+};

components/paved/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/paved",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Pipedream Paved Components",
   "main": "paved.app.mjs",
   "keywords": [
@@ -11,5 +11,8 @@
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "@pipedream/platform": "^3.0.0"
   }
-}
+}

components/paved/paved.app.mjs

@@ -1,11 +1,102 @@
+import { axios } from "@pipedream/platform";
+import constants from "./common/constants.mjs";
+import utils from "./common/utils.mjs";
+
 export default {
   type: "app",
   app: "paved",
-  propDefinitions: {},
+  propDefinitions: {
+    slug: {
+      type: "string",
+      label: "Newsletter Slug",
+      description: "Your newsletter slug.",
+      async options() {
+        const newsletters = await this.listNewsletters();
+        return newsletters.map(({
+          name: label, slug: value,
+        }) => ({
+          label,
+          value,
+        }));
+      },
+    },
+  },
   methods: {
-    // this.$auth contains connected account data
-    authKeys() {
-      console.log(Object.keys(this.$auth));
+    getUrl(path, versionPath = constants.API.PUBLISHER_PATH) {
+      return `${constants.BASE_URL}${versionPath}${path}`;
+    },
+    getHeaders(headers) {
+      return {
+        ...headers,
+        Authorization: `Token ${this.$auth.api_key}`,
+      };
+    },
+    _makeRequest({
+      $ = this, path, headers, ...args
+    } = {}) {
+      return axios($, {
+        ...args,
+        url: this.getUrl(path),
+        headers: this.getHeaders(headers),
+      });
+    },
+    listNewsletters(args = {}) {
+      return this._makeRequest({
+        path: "/newsletters",
+        ...args,
+      });
+    },
+    listSponsorships({
+      slug, ...args
+    } = {}) {
+      return this._makeRequest({
+        path: `/newsletters/${slug}/sponsorships`,
+        ...args,
+      });
+    },
+    async *getIterations({
+      resourcesFn, resourcesFnArgs,
+      fromDate,
+      max = constants.DEFAULT_MAX,
+    }) {
+      let resourcesCount = 0;
+
+      while (true) {
+        const nextResources =
+          await resourcesFn({
+            ...resourcesFnArgs,
+            params: {
+              ...resourcesFnArgs?.params,
+              limit: constants.DEFAULT_LIMIT,
+              from_date: fromDate,
+            },
+          });
+
+        if (!nextResources?.length) {
+          console.log("No more resources found");
+          return;
+        }
+
+        for (const resource of nextResources) {
+          yield resource;
+          resourcesCount += 1;
+
+          if (resourcesCount >= max) {
+            console.log("Reached max resources");
+            return;
+          }
+        }
+
+        if (nextResources.length < constants.DEFAULT_LIMIT) {
+          console.log("No next page found");
+          return;
+        }
+
+        fromDate = nextResources[nextResources.length - 1].run_date;
+      }
+    },
+    paginate(args = {}) {
+      return utils.iterate(this.getIterations(args));
     },
   },
-};
+};

components/paved/sources/common/polling.mjs

@@ -0,0 +1,81 @@
+import {
+  ConfigurationError,
+  DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+} from "@pipedream/platform";
+import app from "../../paved.app.mjs";
+import constants from "../../common/constants.mjs";
+
+export default {
+  props: {
+    app,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      label: "Polling Schedule",
+      description: "How often to poll the API",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+    slug: {
+      propDefinition: [
+        app,
+        "slug",
+      ],
+    },
+  },
+  methods: {
+    generateMeta() {
+      throw new ConfigurationError("generateMeta is not implemented");
+    },
+    setLastFromDate(value) {
+      this.db.set(constants.LAST_FROM_DATE, value);
+    },
+    getLastFromDate() {
+      return this.db.get(constants.LAST_FROM_DATE);
+    },
+    getResourcesFn() {
+      throw new ConfigurationError("getResourcesFn is not implemented");
+    },
+    getResourcesFnArgs() {
+      throw new ConfigurationError("getResourcesFnArgs is not implemented");
+    },
+    processResource(resource) {
+      const meta = this.generateMeta(resource);
+      this.$emit(resource, meta);
+    },
+    async processResources(resources) {
+      const {
+        setLastFromDate,
+        processResource,
+      } = this;
+
+      const [
+        lastResource,
+      ] = Array.from(resources).reverse();
+
+      if (lastResource?.run_date) {
+        setLastFromDate(lastResource.run_date);
+      }
+
+      Array.from(resources)
+        .forEach(processResource);
+    },
+  },
+  async run() {
+    const {
+      app,
+      getResourcesFn,
+      getResourcesFnArgs,
+      processResources,
+    } = this;
+
+    const resources = await app.paginate({
+      resourcesFn: getResourcesFn(),
+      resourcesFnArgs: getResourcesFnArgs(),
+      fromDate: this.getLastFromDate(),
+    });
+
+    processResources(resources);
+  },
+};

components/paved/sources/new-sponsorship-detected/new-sponsorship-detected.mjs

@@ -0,0 +1,30 @@
+import common from "../common/polling.mjs";
+
+export default {
+  ...common,
+  key: "paved-new-sponsorship-detected",
+  name: "New Sponsorship Detected",
+  description: "Emit new event when a sponsorship is detected on a newsletter similar to yours. [See the documentation](https://docs.paved.com/list-sponsorships)",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    getResourcesFn() {
+      return this.app.listSponsorships;
+    },
+    getResourcesFnArgs() {
+      return {
+        debug: true,
+        slug: this.slug,
+      };
+    },
+    generateMeta(resource) {
+      return {
+        id: resource.id,
+        summary: `New Sponsorthip: ${resource.advertiser_name}`,
+        ts: Date.parse(resource.run_date),
+      };
+    },
+  },
+};