Skip to content

Commit

Permalink
Only scrape published if not current time
Browse files Browse the repository at this point in the history
  • Loading branch information
cjmalloy committed Jan 20, 2024
1 parent a9c8578 commit 6f61968
Showing 1 changed file with 4 additions and 0 deletions.
4 changes: 4 additions & 0 deletions src/app/component/bulk/bulk.component.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { HttpErrorResponse } from '@angular/common/http';
import { Component, HostBinding, Input, OnChanges, OnDestroy, SimpleChanges } from '@angular/core';
import { intersection, map, merge, pick, uniq } from 'lodash-es';
import { autorun, IReactionDisposer } from 'mobx';
import * as moment from 'moment';
import { catchError, concat, last, Observable, of, switchMap } from 'rxjs';
import { tap } from 'rxjs/operators';
import { Ext } from '../../model/ext';
Expand Down Expand Up @@ -244,6 +245,9 @@ export class BulkComponent implements OnChanges, OnDestroy {
scraped.title = ref.title;
}
scraped.origin = ref.origin;
if (scraped.published?.isAfter(moment().subtract(5, 'minutes'))) {
scraped.published = ref.published;
}
scraped.modifiedString = ref.modifiedString;
scraped.sources = uniq([...ref.sources || [], ...scraped.sources || []]);
scraped.alternateUrls = uniq([...ref.alternateUrls || [], ...scraped.alternateUrls || []]);
Expand Down

0 comments on commit 6f61968

Please sign in to comment.